public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String cur_file =
((FileSplit) context.getInputSplit()).getPath().getParent().getParent().getName();
String train_file = context.getConfiguration().get("train_file");
if (cur_file.equals(train_file)) {
StringTokenizer st = new StringTokenizer(value.toString());
String word = st.nextToken();
String f_id = st.nextToken();
myKey.set(word);
myVal.set(f_id);
context.write(myKey, myVal);
} else {
StringTokenizer st = new StringTokenizer(value.toString());
String word = st.nextToken();
String f_id = st.nextToken();
StringBuilder builder = new StringBuilder(dlt);
while (st.hasMoreTokens()) {
String filename = st.nextToken();
String tf_idf = st.nextToken();
builder.append(filename);
builder.append(dlt);
builder.append(tf_idf);
builder.append("\t");
}
myKey.set(word);
myVal.set(builder.toString());
context.write(myKey, myVal);
}
}
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String[] pair = new String[2];
int count = 0;
for (Text txt : values) {
pair[count] = txt.toString();
count++;
}
// word exists in training
if (count == 2) {
StringTokenizer st_one, st_two;
if (pair[0].contains(dlt)) {
st_one = new StringTokenizer(pair[1]);
st_two = new StringTokenizer(pair[0]);
} else {
st_one = new StringTokenizer(pair[0]);
st_two = new StringTokenizer(pair[1]);
}
// outputting the data
String f_id = st_one.nextToken();
StringBuilder builder = new StringBuilder(dlt);
builder.append(f_id);
builder.append(dlt);
while (st_two.hasMoreTokens()) {
String filename = st_two.nextToken();
String tf_idf = st_two.nextToken();
builder.append(filename);
builder.append(dlt);
builder.append(tf_idf);
builder.append("\t");
}
myVal.set(builder.toString());
context.write(key, myVal);
}
}