示例#1
0
 @Override
 public void initialize(InputSplit genericSplit, TaskAttemptContext ctxt)
     throws IOException, InterruptedException {
   final GridmixSplit split = (GridmixSplit) genericSplit;
   final Configuration conf = ctxt.getConfiguration();
   factory =
       new ReadRecordFactory(
           split.getLength(), split.getInputRecords(), new FileQueue(split, conf), conf);
 }
示例#2
0
    @Override
    protected void setup(Context ctxt) throws IOException, InterruptedException {
      final Configuration conf = ctxt.getConfiguration();
      final GridmixSplit split = (GridmixSplit) ctxt.getInputSplit();
      final int maps = split.getMapCount();
      final long[] reduceBytes = split.getOutputBytes();
      final long[] reduceRecords = split.getOutputRecords();

      long totalRecords = 0L;
      final int nReduces = ctxt.getNumReduceTasks();
      if (nReduces > 0) {
        int idx = 0;
        int id = split.getId();
        for (int i = 0; i < nReduces; ++i) {
          final GridmixKey.Spec spec = new GridmixKey.Spec();
          if (i == id) {
            spec.bytes_out = split.getReduceBytes(idx);
            spec.rec_out = split.getReduceRecords(idx);
            ++idx;
            id += maps;
          }
          reduces.add(
              new IntermediateRecordFactory(
                  new AvgRecordFactory(reduceBytes[i], reduceRecords[i], conf),
                  i,
                  reduceRecords[i],
                  spec,
                  conf));
          totalRecords += reduceRecords[i];
        }
      } else {
        reduces.add(new AvgRecordFactory(reduceBytes[0], reduceRecords[0], conf));
        totalRecords = reduceRecords[0];
      }
      final long splitRecords = split.getInputRecords();
      final long inputRecords =
          splitRecords <= 0 && split.getLength() >= 0
              ? Math.max(1, split.getLength() / conf.getInt("gridmix.missing.rec.size", 64 * 1024))
              : splitRecords;
      ratio = totalRecords / (1.0 * inputRecords);
      acc = 0.0;
    }