public void reduce(Text key, Iterable<TermFrequencyWritable> values, Context context)
throws IOException, InterruptedException {
int count = 0;
String id = "";
for (TermFrequencyWritable val : values) {
count++;
if (count == 1) {
id = val.getDocumentID().toString();
}
}
TermFrequencyWritable writable = new TermFrequencyWritable();
writable.set(id, count);
context.write(key, writable);
}
public void reduce(Text key, Iterable<TermFrequencyWritable> values, Context context)
throws IOException, InterruptedException {
HashMap<Text, IntWritable> map = new HashMap<Text, IntWritable>();
for (TermFrequencyWritable val : values) {
Text docID = new Text(val.getDocumentID());
int freq = val.getFreq().get();
if (map.get(docID) != null) {
map.put(docID, new IntWritable(map.get(docID).get() + freq));
} else {
map.put(docID, new IntWritable(freq));
}
}
MapWritable outputMap = new MapWritable();
outputMap.putAll(map);
context.write(key, outputMap);
}
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String valString = value.toString().replaceAll("[^a-zA-Z0-9]+", " ");
StringTokenizer itr = new StringTokenizer(valString);
FileSplit fileSplit = (FileSplit) context.getInputSplit();
String fileName = fileSplit.getPath().getName();
while (itr.hasMoreTokens()) {
term.set(itr.nextToken());
docFrequency.set(fileName, 1);
context.write(term, docFrequency);
}
}