示例#1
0
  @Test
  public void testWordCountMR() throws Exception {
    // INIT
    Path inputPath = new Path(box.getInputPath(), "wordCount");
    Path resultPath = new Path("testMapReduceResult");

    String[] content = new String[] {"1 2 3\n", "2 3 4\n", "4 5 6 6\n"};
    DFSUtil.writeToFile(box.getFS(), inputPath, true, content);

    Job job =
        MRUtil.setUpJob(
            box.getConf(),
            WordCount.class,
            WordCount.Map.class,
            WordCount.Reduce.class,
            Text.class,
            IntWritable.class,
            box.getInputPath(),
            box.getOutputPath());

    // ACT & ASSERT
    boolean jobResult = job.waitForCompletion(true);

    assertTrue(jobResult);
    FileUtil.copyMerge(
        box.getFS(), box.getOutputPath(), box.getFS(), resultPath, false, box.getConf(), "\n");
    String result = DFSUtil.getFileContent(box.getFS(), resultPath);

    Pattern p = Pattern.compile("[\\s]+");
    for (String ln : result.split("\n")) {
      String[] splitty = p.split(ln.trim());
      if (splitty.length == 2) {
        int term = new Integer(splitty[0]);
        int freq = new Integer(splitty[1]);
        switch (term) {
          case 1:
            assertEquals(freq, 1);
            break;
          case 2:
            assertEquals(freq, 2);
            break;
          case 3:
            assertEquals(freq, 2);
            break;
          case 4:
            assertEquals(freq, 2);
            break;
          case 5:
            assertEquals(freq, 1);
            break;
          case 6:
            assertEquals(freq, 2);
            break;
          default:
            throw new Exception("Unknown term " + term);
        }
      }
    }
  }
示例#2
0
  @Test
  public void testSeqFileWriter() throws Exception {
    int[] keys = {2, 3, 1};
    String[] vals = {"b", "c", "a"};
    Path testPath = new Path("testSeqFile");
    List<MeEntry<IntWritable, Text>> list = new ArrayList<MeEntry<IntWritable, Text>>();

    for (int i = 0; i < keys.length; i++) {
      MeEntry<IntWritable, Text> e =
          new MeEntry<IntWritable, Text>(new IntWritable(keys[i]), new Text(vals[i]));
      list.add(e);
    }

    DFSUtil.appendItemsToSequenceFile(
        box.getConf(), testPath, IntWritable.class, Text.class, list.iterator());

    List<Map.Entry<IntWritable, Text>> readedVals = DFSUtil.readSeqFile(box.getConf(), testPath);

    for (Map.Entry<IntWritable, Text> read : readedVals) {
      int key = read.getKey().get();
      String val = new String(read.getValue().getBytes());
      switch (key) {
        case 1:
          assertEquals(val, "a");
          break;
        case 2:
          assertEquals(val, "b");
          break;
        case 3:
          assertEquals(val, "c");
          break;
        default:
          throw new Exception("Unknown key: " + key);
      }
    }
  }