/**
* Add the map-related parameters into the space, except the ones in the excluded set
*
* @param space the parameter space
* @param conf the configuration
* @param exclude the exclusion set
*/
private static void addEffectMapParameters(
ParameterSpace space, Configuration conf, Set<String> exclude) {
// Get the maximum memory
long taskMem = ProfileUtils.getTaskMemory(conf);
long maxMem = (long) (MAX_MEM_RATIO * taskMem);
long minMem = (long) (MIN_MEM_RATIO * taskMem);
// Add parameters that effect the map tasks
if (!exclude.contains(HadoopParameter.SORT_MB.toString()))
space.addParameterDescriptor(
new IntegerParamDescriptor(
HadoopParameter.SORT_MB,
ParamTaskEffect.EFFECT_MAP,
(int) (minMem >> 20),
(int) (maxMem >> 20)));
if (!exclude.contains(HadoopParameter.SPILL_PERC.toString()))
space.addParameterDescriptor(
new DoubleParamDescriptor(
HadoopParameter.SPILL_PERC, ParamTaskEffect.EFFECT_MAP, 0.2, 0.9));
if (!exclude.contains(HadoopParameter.SORT_REC_PERC.toString()))
space.addParameterDescriptor(
new DoubleParamDescriptor(
HadoopParameter.SORT_REC_PERC, ParamTaskEffect.EFFECT_MAP, 0.01, 0.5));
if (conf.get(MR_COMBINE_CLASS) != null
&& !exclude.contains(HadoopParameter.NUM_SPILLS_COMBINE.toString())) {
space.addParameterDescriptor(
new ListParamDescriptor(
HadoopParameter.NUM_SPILLS_COMBINE, ParamTaskEffect.EFFECT_MAP, "3", "9999"));
}
}
/**
* Adjusts the domain of some parameter descriptors based on information from the cluster, the
* configuration, and the virtual job profile. Currently, the parameters adjusted are:
*
* <ul>
* <li>io.sort.mb
* <li>mapred.job.reduce.input.buffer.percent
* <li>mapred.reduce.tasks
* </ul>
*
* @param space the parameter space
* @param cluster the cluster
* @param conf the configuration
* @param jobProfile the virtual job profile
*/
public static void adjustParameterDescriptors(
ParameterSpace space,
ClusterConfiguration cluster,
Configuration conf,
MRJobProfile jobProfile) {
long taskMemory = ProfileUtils.getTaskMemory(conf);
// Adjust the max value of io.sort.mb
if (space.containsParamDescriptor(HadoopParameter.SORT_MB)) {
adjustParamDescrSortMB(
(IntegerParamDescriptor) space.getParameterDescriptor(HadoopParameter.SORT_MB),
jobProfile,
taskMemory);
}
// Adjust the max value of mapred.job.reduce.input.buffer.percent
if (space.containsParamDescriptor(HadoopParameter.RED_IN_BUFF_PERC)) {
adjustParamDescrRedInBufferPerc(
(DoubleParamDescriptor) space.getParameterDescriptor(HadoopParameter.RED_IN_BUFF_PERC),
jobProfile,
taskMemory);
}
// Adjust the min and max number of mapred.reduce.tasks
if (space.containsParamDescriptor(HadoopParameter.RED_TASKS)) {
adjustParamDescrRedTasks(
(IntegerParamDescriptor) space.getParameterDescriptor(HadoopParameter.RED_TASKS),
jobProfile,
taskMemory,
cluster.getTotalReduceSlots());
}
}
