예제 #1
0
  /**
   * Add the reduce-related parameters into the space, except the ones in the excluded set
   *
   * @param space the parameter space
   * @param conf the configuration
   * @param exclude the exclusion set
   */
  private static void addEffectReduceParameters(
      ParameterSpace space, Configuration conf, Set<String> exclude) {

    // Add parameters the effect the reduce tasks
    if (!exclude.contains(HadoopParameter.RED_TASKS.toString()))
      space.addParameterDescriptor(
          new IntegerParamDescriptor(
              HadoopParameter.RED_TASKS, ParamTaskEffect.EFFECT_REDUCE, 1, 100));
    if (!exclude.contains(HadoopParameter.INMEM_MERGE.toString()))
      space.addParameterDescriptor(
          new IntegerParamDescriptor(
              HadoopParameter.INMEM_MERGE, ParamTaskEffect.EFFECT_REDUCE, 10, 1000));
    if (!exclude.contains(HadoopParameter.SHUFFLE_IN_BUFF_PERC.toString()))
      space.addParameterDescriptor(
          new DoubleParamDescriptor(
              HadoopParameter.SHUFFLE_IN_BUFF_PERC, ParamTaskEffect.EFFECT_REDUCE, 0.2, 0.9));
    if (!exclude.contains(HadoopParameter.SHUFFLE_MERGE_PERC.toString()))
      space.addParameterDescriptor(
          new DoubleParamDescriptor(
              HadoopParameter.SHUFFLE_MERGE_PERC, ParamTaskEffect.EFFECT_REDUCE, 0.2, 0.9));
    if (!exclude.contains(HadoopParameter.RED_IN_BUFF_PERC.toString()))
      space.addParameterDescriptor(
          new DoubleParamDescriptor(
              HadoopParameter.RED_IN_BUFF_PERC, ParamTaskEffect.EFFECT_REDUCE, 0, 0.8));
    if (!exclude.contains(HadoopParameter.COMPRESS_OUT.toString()))
      space.addParameterDescriptor(
          new BooleanParamDescriptor(HadoopParameter.COMPRESS_OUT, ParamTaskEffect.EFFECT_REDUCE));
  }
예제 #2
0
  /**
   * Returns a parameter space with all the parameter descriptors that can effect the execution of
   * the next MapReduce job (currently, the number of reducers and output compression)
   *
   * @param conf the job configuration
   * @return the parameter space
   */
  public static ParameterSpace getParamSpaceForNextJob(Configuration conf) {

    Set<String> exclude = buildParamExclusionSet(conf);
    ParameterSpace space = new ParameterSpace();

    if (conf.getInt(MR_RED_TASKS, 1) != 0
        && !exclude.contains(HadoopParameter.RED_TASKS.toString()))
      space.addParameterDescriptor(
          new IntegerParamDescriptor(
              HadoopParameter.RED_TASKS, ParamTaskEffect.EFFECT_REDUCE, 1, 100));

    if (!exclude.contains(HadoopParameter.COMPRESS_MAP_OUT.toString()))
      space.addParameterDescriptor(
          new BooleanParamDescriptor(
              HadoopParameter.COMPRESS_MAP_OUT, ParamTaskEffect.EFFECT_BOTH));

    return space;
  }