@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
Integer sum, mean, max, min, var;
// TODO
for (TextArrayWritable val : values) {
Text[] pair = (Text[]) val.toArray();
String title = pair[0].toString();
Integer count = Integer.parseInt(pair[1].toString());
countToTitleMap.add(new Pair<Integer, String>(count, title));
if (countToTitleMap.size() > this.N) {
countToTitleMap.remove(countToTitleMap.first());
}
}
// calculate the statistics
Integer[] counts = new Integer[this.N];
int j = 0;
for (Pair<Integer, String> pair : countToTitleMap) {
counts[j++] = pair.first;
}
sum = 0;
min = Integer.MAX_VALUE;
max = Integer.MIN_VALUE;
for (int i = 0; i < this.N; i++) {
sum += counts[i];
min = Math.min(min, counts[i]);
max = Math.max(max, counts[i]);
}
mean = sum / this.N;
var = 0;
for (int i = 0; i < this.N; i++) {
var += ((counts[i] - mean) * (counts[i] - mean));
}
var /= this.N;
context.write(new Text("Mean"), new IntWritable(mean));
context.write(new Text("Sum"), new IntWritable(sum));
context.write(new Text("Min"), new IntWritable(min));
context.write(new Text("Max"), new IntWritable(max));
context.write(new Text("Var"), new IntWritable(var));
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
int sum, mean, max, min, var;
sum = mean = max = min = var = 0;
for (TextArrayWritable val : values) {
Text[] pair = (Text[]) val.toArray();
String word = pair[0].toString();
Integer count = Integer.parseInt(pair[1].toString());
sortedWordCount.add(new Pair<Integer, String>(count, word));
if (sortedWordCount.size() > N) {
sortedWordCount.remove(sortedWordCount.first());
}
}
for (Pair<Integer, String> entry : sortedWordCount) {
// Text word = new Text(entry.second);
// IntWritable value = new IntWritable(entry.first);
// context.write(word, value);
sum += entry.first;
}
mean = (sum / sortedWordCount.size());
min = sortedWordCount.first().first;
max = sortedWordCount.last().first;
for (Pair<Integer, String> entry : sortedWordCount) {
var += ((entry.first - mean) * (entry.first - mean));
}
var = var / sortedWordCount.size();
context.write(new Text("Mean"), new IntWritable(mean));
context.write(new Text("Sum"), new IntWritable(sum));
context.write(new Text("Min"), new IntWritable(min));
context.write(new Text("Max"), new IntWritable(max));
context.write(new Text("Var"), new IntWritable(var));
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
for (TextArrayWritable val : values) {
Text[] pair = (Text[]) val.toArray();
String word = pair[0].toString();
Float avg = Float.parseFloat(pair[1].toString());
/*
Text tWord = new Text(word);
FloatWritable value = new FloatWritable(avg);
context.write(tWord, value);
*/
String[] parts = word.split("-");
String newKey = parts[0] + "-" + parts[1];
String carrier = parts[2];
if (!carrierAvgDelay.containsKey(newKey)) {
TreeSet<Pair<Float, String>> avgDelayMap = new TreeSet<Pair<Float, String>>();
carrierAvgDelay.put(newKey, avgDelayMap);
}
TreeSet<Pair<Float, String>> ts = carrierAvgDelay.get(newKey);
ts.add(new Pair<Float, String>(avg, carrier));
if (ts.size() > 10) {
ts.remove(ts.last());
}
carrierAvgDelay.put(newKey, ts);
}
Iterator it = carrierAvgDelay.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry) it.next();
TreeSet<Pair<Float, String>> ts = (TreeSet<Pair<Float, String>>) pair.getValue();
for (Pair<Float, String> item : ts) {
Text word = new Text(pair.getKey() + "-" + item.second);
FloatWritable value = new FloatWritable(item.first);
context.write(word, value);
}
}
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
for (TextArrayWritable val : values) {
Text[] pair = (Text[]) val.toArray();
String word = pair[0].toString();
Integer count = Integer.parseInt(pair[1].toString());
countToWordMap.add(new Pair<Integer, String>(count, word));
if (countToWordMap.size() > N) {
countToWordMap.remove(countToWordMap.first());
}
}
for (Pair<Integer, String> item : countToWordMap) {
Text word = new Text(item.second);
IntWritable value = new IntWritable(item.first);
context.write(word, value);
}
}
