/**
* Add the reduce-related parameters into the space, except the ones in the excluded set
*
* @param space the parameter space
* @param conf the configuration
* @param exclude the exclusion set
*/
private static void addEffectReduceParameters(
ParameterSpace space, Configuration conf, Set<String> exclude) {
// Add parameters the effect the reduce tasks
if (!exclude.contains(HadoopParameter.RED_TASKS.toString()))
space.addParameterDescriptor(
new IntegerParamDescriptor(
HadoopParameter.RED_TASKS, ParamTaskEffect.EFFECT_REDUCE, 1, 100));
if (!exclude.contains(HadoopParameter.INMEM_MERGE.toString()))
space.addParameterDescriptor(
new IntegerParamDescriptor(
HadoopParameter.INMEM_MERGE, ParamTaskEffect.EFFECT_REDUCE, 10, 1000));
if (!exclude.contains(HadoopParameter.SHUFFLE_IN_BUFF_PERC.toString()))
space.addParameterDescriptor(
new DoubleParamDescriptor(
HadoopParameter.SHUFFLE_IN_BUFF_PERC, ParamTaskEffect.EFFECT_REDUCE, 0.2, 0.9));
if (!exclude.contains(HadoopParameter.SHUFFLE_MERGE_PERC.toString()))
space.addParameterDescriptor(
new DoubleParamDescriptor(
HadoopParameter.SHUFFLE_MERGE_PERC, ParamTaskEffect.EFFECT_REDUCE, 0.2, 0.9));
if (!exclude.contains(HadoopParameter.RED_IN_BUFF_PERC.toString()))
space.addParameterDescriptor(
new DoubleParamDescriptor(
HadoopParameter.RED_IN_BUFF_PERC, ParamTaskEffect.EFFECT_REDUCE, 0, 0.8));
if (!exclude.contains(HadoopParameter.COMPRESS_OUT.toString()))
space.addParameterDescriptor(
new BooleanParamDescriptor(HadoopParameter.COMPRESS_OUT, ParamTaskEffect.EFFECT_REDUCE));
}
/**
* Returns a parameter space with all the parameter descriptors for a map-only job
*
* @param conf the job configuration
* @param exclude the exclusion set
* @return the parameter space
*/
private static ParameterSpace getMapOnlyParamSpace(Configuration conf, Set<String> exclude) {
ParameterSpace space = new ParameterSpace();
if (!exclude.contains(HadoopParameter.COMPRESS_OUT.toString()))
space.addParameterDescriptor(
new BooleanParamDescriptor(HadoopParameter.COMPRESS_OUT, ParamTaskEffect.EFFECT_MAP));
return space;
}