@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext ctxt)
throws IOException, InterruptedException {
final GridmixSplit split = (GridmixSplit) genericSplit;
final Configuration conf = ctxt.getConfiguration();
factory =
new ReadRecordFactory(
split.getLength(), split.getInputRecords(), new FileQueue(split, conf), conf);
}
@Override
protected void setup(Context ctxt) throws IOException, InterruptedException {
final Configuration conf = ctxt.getConfiguration();
final GridmixSplit split = (GridmixSplit) ctxt.getInputSplit();
final int maps = split.getMapCount();
final long[] reduceBytes = split.getOutputBytes();
final long[] reduceRecords = split.getOutputRecords();
long totalRecords = 0L;
final int nReduces = ctxt.getNumReduceTasks();
if (nReduces > 0) {
int idx = 0;
int id = split.getId();
for (int i = 0; i < nReduces; ++i) {
final GridmixKey.Spec spec = new GridmixKey.Spec();
if (i == id) {
spec.bytes_out = split.getReduceBytes(idx);
spec.rec_out = split.getReduceRecords(idx);
++idx;
id += maps;
}
reduces.add(
new IntermediateRecordFactory(
new AvgRecordFactory(reduceBytes[i], reduceRecords[i], conf),
i,
reduceRecords[i],
spec,
conf));
totalRecords += reduceRecords[i];
}
} else {
reduces.add(new AvgRecordFactory(reduceBytes[0], reduceRecords[0], conf));
totalRecords = reduceRecords[0];
}
final long splitRecords = split.getInputRecords();
final long inputRecords =
splitRecords <= 0 && split.getLength() >= 0
? Math.max(1, split.getLength() / conf.getInt("gridmix.missing.rec.size", 64 * 1024))
: splitRecords;
ratio = totalRecords / (1.0 * inputRecords);
acc = 0.0;
}
