Пример #1
0
  /**
   * Add the reduce-related parameters into the space, except the ones in the excluded set
   *
   * @param space the parameter space
   * @param conf the configuration
   * @param exclude the exclusion set
   */
  private static void addEffectReduceParameters(
      ParameterSpace space, Configuration conf, Set<String> exclude) {

    // Add parameters the effect the reduce tasks
    if (!exclude.contains(HadoopParameter.RED_TASKS.toString()))
      space.addParameterDescriptor(
          new IntegerParamDescriptor(
              HadoopParameter.RED_TASKS, ParamTaskEffect.EFFECT_REDUCE, 1, 100));
    if (!exclude.contains(HadoopParameter.INMEM_MERGE.toString()))
      space.addParameterDescriptor(
          new IntegerParamDescriptor(
              HadoopParameter.INMEM_MERGE, ParamTaskEffect.EFFECT_REDUCE, 10, 1000));
    if (!exclude.contains(HadoopParameter.SHUFFLE_IN_BUFF_PERC.toString()))
      space.addParameterDescriptor(
          new DoubleParamDescriptor(
              HadoopParameter.SHUFFLE_IN_BUFF_PERC, ParamTaskEffect.EFFECT_REDUCE, 0.2, 0.9));
    if (!exclude.contains(HadoopParameter.SHUFFLE_MERGE_PERC.toString()))
      space.addParameterDescriptor(
          new DoubleParamDescriptor(
              HadoopParameter.SHUFFLE_MERGE_PERC, ParamTaskEffect.EFFECT_REDUCE, 0.2, 0.9));
    if (!exclude.contains(HadoopParameter.RED_IN_BUFF_PERC.toString()))
      space.addParameterDescriptor(
          new DoubleParamDescriptor(
              HadoopParameter.RED_IN_BUFF_PERC, ParamTaskEffect.EFFECT_REDUCE, 0, 0.8));
    if (!exclude.contains(HadoopParameter.COMPRESS_OUT.toString()))
      space.addParameterDescriptor(
          new BooleanParamDescriptor(HadoopParameter.COMPRESS_OUT, ParamTaskEffect.EFFECT_REDUCE));
  }
Пример #2
0
  /**
   * Returns a parameter space with all the parameter descriptors for a map-only job
   *
   * @param conf the job configuration
   * @param exclude the exclusion set
   * @return the parameter space
   */
  private static ParameterSpace getMapOnlyParamSpace(Configuration conf, Set<String> exclude) {
    ParameterSpace space = new ParameterSpace();

    if (!exclude.contains(HadoopParameter.COMPRESS_OUT.toString()))
      space.addParameterDescriptor(
          new BooleanParamDescriptor(HadoopParameter.COMPRESS_OUT, ParamTaskEffect.EFFECT_MAP));

    return space;
  }