Ejemplo n.º 1
0
  @Test
  public void testWordCountMR() throws Exception {
    // INIT
    Path inputPath = new Path(box.getInputPath(), "wordCount");
    Path resultPath = new Path("testMapReduceResult");

    String[] content = new String[] {"1 2 3\n", "2 3 4\n", "4 5 6 6\n"};
    DFSUtil.writeToFile(box.getFS(), inputPath, true, content);

    Job job =
        MRUtil.setUpJob(
            box.getConf(),
            WordCount.class,
            WordCount.Map.class,
            WordCount.Reduce.class,
            Text.class,
            IntWritable.class,
            box.getInputPath(),
            box.getOutputPath());

    // ACT & ASSERT
    boolean jobResult = job.waitForCompletion(true);

    assertTrue(jobResult);
    FileUtil.copyMerge(
        box.getFS(), box.getOutputPath(), box.getFS(), resultPath, false, box.getConf(), "\n");
    String result = DFSUtil.getFileContent(box.getFS(), resultPath);

    Pattern p = Pattern.compile("[\\s]+");
    for (String ln : result.split("\n")) {
      String[] splitty = p.split(ln.trim());
      if (splitty.length == 2) {
        int term = new Integer(splitty[0]);
        int freq = new Integer(splitty[1]);
        switch (term) {
          case 1:
            assertEquals(freq, 1);
            break;
          case 2:
            assertEquals(freq, 2);
            break;
          case 3:
            assertEquals(freq, 2);
            break;
          case 4:
            assertEquals(freq, 2);
            break;
          case 5:
            assertEquals(freq, 1);
            break;
          case 6:
            assertEquals(freq, 2);
            break;
          default:
            throw new Exception("Unknown term " + term);
        }
      }
    }
  }
Ejemplo n.º 2
0
 @Override
 public List<Long> getSandBoxIdsForUpwardHierarchy(
     boolean includeInherited, boolean includeCurrent) {
   List<Long> ids = new ArrayList<Long>();
   if (includeCurrent) {
     ids.add(this.getId());
   }
   if (includeInherited) {
     SandBox current = this;
     while (current.getParentSandBox() != null) {
       current = current.getParentSandBox();
       ids.add(current.getId());
     }
     Collections.reverse(ids);
   }
   return ids;
 }
Ejemplo n.º 3
0
  @Override
  public String getColor() {
    if (StringUtils.isNotBlank(color)) {
      return color;
    }

    if (parentSandBox != null) {
      return parentSandBox.getColor();
    }

    return null;
  }
Ejemplo n.º 4
0
  @Test
  public void testSeqFileWriter() throws Exception {
    int[] keys = {2, 3, 1};
    String[] vals = {"b", "c", "a"};
    Path testPath = new Path("testSeqFile");
    List<MeEntry<IntWritable, Text>> list = new ArrayList<MeEntry<IntWritable, Text>>();

    for (int i = 0; i < keys.length; i++) {
      MeEntry<IntWritable, Text> e =
          new MeEntry<IntWritable, Text>(new IntWritable(keys[i]), new Text(vals[i]));
      list.add(e);
    }

    DFSUtil.appendItemsToSequenceFile(
        box.getConf(), testPath, IntWritable.class, Text.class, list.iterator());

    List<Map.Entry<IntWritable, Text>> readedVals = DFSUtil.readSeqFile(box.getConf(), testPath);

    for (Map.Entry<IntWritable, Text> read : readedVals) {
      int key = read.getKey().get();
      String val = new String(read.getValue().getBytes());
      switch (key) {
        case 1:
          assertEquals(val, "a");
          break;
        case 2:
          assertEquals(val, "b");
          break;
        case 3:
          assertEquals(val, "c");
          break;
        default:
          throw new Exception("Unknown key: " + key);
      }
    }
  }
Ejemplo n.º 5
0
 @AfterClass
 public static void drop() throws Exception {
   box.dropCluster();
 }
Ejemplo n.º 6
0
 @BeforeClass
 public static void init() throws Exception {
   box.initCluster();
 }
Ejemplo n.º 7
0
 static {
   box = box.getInstance();
 }