@Test
public void testWordCountMR() throws Exception {
// INIT
Path inputPath = new Path(box.getInputPath(), "wordCount");
Path resultPath = new Path("testMapReduceResult");
String[] content = new String[] {"1 2 3\n", "2 3 4\n", "4 5 6 6\n"};
DFSUtil.writeToFile(box.getFS(), inputPath, true, content);
Job job =
MRUtil.setUpJob(
box.getConf(),
WordCount.class,
WordCount.Map.class,
WordCount.Reduce.class,
Text.class,
IntWritable.class,
box.getInputPath(),
box.getOutputPath());
// ACT & ASSERT
boolean jobResult = job.waitForCompletion(true);
assertTrue(jobResult);
FileUtil.copyMerge(
box.getFS(), box.getOutputPath(), box.getFS(), resultPath, false, box.getConf(), "\n");
String result = DFSUtil.getFileContent(box.getFS(), resultPath);
Pattern p = Pattern.compile("[\\s]+");
for (String ln : result.split("\n")) {
String[] splitty = p.split(ln.trim());
if (splitty.length == 2) {
int term = new Integer(splitty[0]);
int freq = new Integer(splitty[1]);
switch (term) {
case 1:
assertEquals(freq, 1);
break;
case 2:
assertEquals(freq, 2);
break;
case 3:
assertEquals(freq, 2);
break;
case 4:
assertEquals(freq, 2);
break;
case 5:
assertEquals(freq, 1);
break;
case 6:
assertEquals(freq, 2);
break;
default:
throw new Exception("Unknown term " + term);
}
}
}
}
@Test
public void testSeqFileWriter() throws Exception {
int[] keys = {2, 3, 1};
String[] vals = {"b", "c", "a"};
Path testPath = new Path("testSeqFile");
List<MeEntry<IntWritable, Text>> list = new ArrayList<MeEntry<IntWritable, Text>>();
for (int i = 0; i < keys.length; i++) {
MeEntry<IntWritable, Text> e =
new MeEntry<IntWritable, Text>(new IntWritable(keys[i]), new Text(vals[i]));
list.add(e);
}
DFSUtil.appendItemsToSequenceFile(
box.getConf(), testPath, IntWritable.class, Text.class, list.iterator());
List<Map.Entry<IntWritable, Text>> readedVals = DFSUtil.readSeqFile(box.getConf(), testPath);
for (Map.Entry<IntWritable, Text> read : readedVals) {
int key = read.getKey().get();
String val = new String(read.getValue().getBytes());
switch (key) {
case 1:
assertEquals(val, "a");
break;
case 2:
assertEquals(val, "b");
break;
case 3:
assertEquals(val, "c");
break;
default:
throw new Exception("Unknown key: " + key);
}
}
}