@Override
protected void reduce(
SortedMapWritableComparable key, Iterable<BytesWritable> values, Context context)
throws IOException, InterruptedException {
StatisticsProtos.Statistics.Builder statisticsBuilder =
StatisticsProtos.Statistics.newBuilder();
StatisticsProtos.KeyValue.Builder kvBuilder = StatisticsProtos.KeyValue.newBuilder();
for (SortedMapWritableComparable.Entry<WritableComparable, Writable> partEntry :
key.entrySet()) {
kvBuilder.clear();
Text kText = (Text) partEntry.getKey();
Text vText = (Text) partEntry.getValue();
kvBuilder.setKey(kText.toString());
kvBuilder.setValue(vText.toString());
statisticsBuilder.addPartitions(kvBuilder);
}
for (Map.Entry<String, StatisticCalculator> statisticEntry :
this.statGenConfiguration.getStatisticCalculators().entrySet()) {
kvBuilder.clear();
double result = statisticEntry.getValue().calculate(values);
kvBuilder.setKey(statisticEntry.getKey());
kvBuilder.setValue(String.valueOf(result));
statisticsBuilder.addStatistics(kvBuilder);
}
statisticsBuilder.setTimestamp(new Date().getTime());
context.write(
new Text(genKeyFromMap(key)), new BytesWritable(statisticsBuilder.build().toByteArray()));
}
@Override
protected void map(Text key, BytesWritable value, Context context)
throws IOException, InterruptedException {
StatisticsProtos.InputEntry inputEntry =
StatisticsProtos.InputEntry.parseFrom(value.copyBytes());
Set<SortedMapWritableComparable> outputKeyMaps = new HashSet<SortedMapWritableComparable>();
outputKeyMaps.add(new SortedMapWritableComparable());
Map<String, Partitioner> partitioners = statGenConfiguration.getPartitioners();
for (StatisticsProtos.KeyValue field : inputEntry.getFieldList()) {
String fieldKey = field.getKey();
if (partitioners.containsKey(fieldKey)) {
String[] partitions = partitioners.get(fieldKey).partition(field.getValue());
// the number of output values should be multiplied by number of partitions
// (partitions.lenght).
// if 0, then return nothing...
if (partitions.length == 0) {
return;
}
for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
outputKeyMap.put(new Text(fieldKey), new Text(partitions[0]));
}
// if more than 1, then copy output values
for (int i = 1; i < partitions.length; i++) {
Text mapKey = new Text(fieldKey);
Text mapValue = new Text(partitions[i]);
Set<SortedMapWritableComparable> newMaps = new HashSet<SortedMapWritableComparable>();
for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
SortedMapWritableComparable newMap = new SortedMapWritableComparable(outputKeyMap);
newMap.put(mapKey, mapValue);
newMaps.add(newMap);
}
outputKeyMaps.addAll(newMaps);
}
}
}
for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
context.write(outputKeyMap, value);
}
}
