/**
* Add the reduce-related parameters into the space, except the ones in the excluded set
*
* @param space the parameter space
* @param conf the configuration
* @param exclude the exclusion set
*/
private static void addEffectReduceParameters(
ParameterSpace space, Configuration conf, Set<String> exclude) {
// Add parameters the effect the reduce tasks
if (!exclude.contains(HadoopParameter.RED_TASKS.toString()))
space.addParameterDescriptor(
new IntegerParamDescriptor(
HadoopParameter.RED_TASKS, ParamTaskEffect.EFFECT_REDUCE, 1, 100));
if (!exclude.contains(HadoopParameter.INMEM_MERGE.toString()))
space.addParameterDescriptor(
new IntegerParamDescriptor(
HadoopParameter.INMEM_MERGE, ParamTaskEffect.EFFECT_REDUCE, 10, 1000));
if (!exclude.contains(HadoopParameter.SHUFFLE_IN_BUFF_PERC.toString()))
space.addParameterDescriptor(
new DoubleParamDescriptor(
HadoopParameter.SHUFFLE_IN_BUFF_PERC, ParamTaskEffect.EFFECT_REDUCE, 0.2, 0.9));
if (!exclude.contains(HadoopParameter.SHUFFLE_MERGE_PERC.toString()))
space.addParameterDescriptor(
new DoubleParamDescriptor(
HadoopParameter.SHUFFLE_MERGE_PERC, ParamTaskEffect.EFFECT_REDUCE, 0.2, 0.9));
if (!exclude.contains(HadoopParameter.RED_IN_BUFF_PERC.toString()))
space.addParameterDescriptor(
new DoubleParamDescriptor(
HadoopParameter.RED_IN_BUFF_PERC, ParamTaskEffect.EFFECT_REDUCE, 0, 0.8));
if (!exclude.contains(HadoopParameter.COMPRESS_OUT.toString()))
space.addParameterDescriptor(
new BooleanParamDescriptor(HadoopParameter.COMPRESS_OUT, ParamTaskEffect.EFFECT_REDUCE));
}
/**
* Returns a parameter space with all the parameter descriptors that can effect the execution of
* the next MapReduce job (currently, the number of reducers and output compression)
*
* @param conf the job configuration
* @return the parameter space
*/
public static ParameterSpace getParamSpaceForNextJob(Configuration conf) {
Set<String> exclude = buildParamExclusionSet(conf);
ParameterSpace space = new ParameterSpace();
if (conf.getInt(MR_RED_TASKS, 1) != 0
&& !exclude.contains(HadoopParameter.RED_TASKS.toString()))
space.addParameterDescriptor(
new IntegerParamDescriptor(
HadoopParameter.RED_TASKS, ParamTaskEffect.EFFECT_REDUCE, 1, 100));
if (!exclude.contains(HadoopParameter.COMPRESS_MAP_OUT.toString()))
space.addParameterDescriptor(
new BooleanParamDescriptor(
HadoopParameter.COMPRESS_MAP_OUT, ParamTaskEffect.EFFECT_BOTH));
return space;
}
