示例#1
0
  @Test
  public void testMap() throws IOException {
    mapDriver.withInput(new LongWritable(1), new Text("Marco-Fabio,Luca,Gordon"));
    mapDriver.withInput(new LongWritable(2), new Text("Fabio-Nathalia,Marco"));
    mapDriver.withInput(new LongWritable(3), new Text("Luca-Nathalia,Marco"));
    mapDriver.withInput(new LongWritable(4), new Text("Nathalia-Fabio,Luca"));
    mapDriver.withInput(new LongWritable(5), new Text("Gordon-Marco"));

    mapDriver.withOutput(new Text("Marco"), new TextPair("Fabio", "FRIENDS"));
    mapDriver.withOutput(new Text("Marco"), new TextPair("Luca", "FRIENDS"));
    mapDriver.withOutput(new Text("Marco"), new TextPair("Gordon", "FRIENDS"));
    mapDriver.withOutput(new Text("Luca"), new TextPair("Fabio", "Marco"));
    mapDriver.withOutput(new Text("Fabio"), new TextPair("Luca", "Marco"));
    mapDriver.withOutput(new Text("Gordon"), new TextPair("Fabio", "Marco"));
    mapDriver.withOutput(new Text("Fabio"), new TextPair("Gordon", "Marco"));
    mapDriver.withOutput(new Text("Gordon"), new TextPair("Luca", "Marco"));
    mapDriver.withOutput(new Text("Luca"), new TextPair("Gordon", "Marco"));
    mapDriver.withOutput(new Text("Fabio"), new TextPair("Nathalia", "FRIENDS"));
    mapDriver.withOutput(new Text("Fabio"), new TextPair("Marco", "FRIENDS"));
    mapDriver.withOutput(new Text("Marco"), new TextPair("Nathalia", "Fabio"));
    mapDriver.withOutput(new Text("Nathalia"), new TextPair("Marco", "Fabio"));
    mapDriver.withOutput(new Text("Luca"), new TextPair("Nathalia", "FRIENDS"));
    mapDriver.withOutput(new Text("Luca"), new TextPair("Marco", "FRIENDS"));
    mapDriver.withOutput(new Text("Marco"), new TextPair("Nathalia", "Luca"));
    mapDriver.withOutput(new Text("Nathalia"), new TextPair("Marco", "Luca"));
    mapDriver.withOutput(new Text("Nathalia"), new TextPair("Fabio", "FRIENDS"));
    mapDriver.withOutput(new Text("Nathalia"), new TextPair("Luca", "FRIENDS"));
    mapDriver.withOutput(new Text("Luca"), new TextPair("Fabio", "Nathalia"));
    mapDriver.withOutput(new Text("Fabio"), new TextPair("Luca", "Nathalia"));
    mapDriver.withOutput(new Text("Gordon"), new TextPair("Marco", "FRIENDS"));

    mapDriver.runTest();
  }
示例#2
0
  @Test
  public void processValidRecord() throws IOException {

    mapDriver.withInput(new Text("a"), new Text("ein"));
    mapDriver.withInput(new Text("a"), new Text("zwei"));
    mapDriver.withInput(new Text("c"), new Text("drei"));
    mapDriver.withOutput(new Text("a"), new Text("ein"));
    mapDriver.withOutput(new Text("a"), new Text("zwei"));
    mapDriver.withOutput(new Text("c"), new Text("drei"));
    mapDriver.runTest();
  }
  @Test
  public void testMapper() throws IOException, URISyntaxException {

    MapDriver<LongWritable, Text, Text, Text> mapDriver;
    JpylyzerMapper mapper = new JpylyzerMapper();
    String name = "B400022028241-RT1/balloon.jp2";
    mapDriver = MapDriver.newMapDriver(mapper);
    File testFolder =
        new File(Thread.currentThread().getContextClassLoader().getResource(name).toURI())
            .getParentFile()
            .getParentFile()
            .getParentFile()
            .getParentFile();
    File jpylyzerPath = new File(testFolder, "src/test/extras/jpylyzer-1.10.1/jpylyzer.py");
    mapDriver
        .getConfiguration()
        .set(
            dk.statsbiblioteket.medieplatform.autonomous.ConfigConstants.JPYLYZER_PATH,
            jpylyzerPath.getAbsolutePath());

    String testFile = getAbsolutePath(name);
    mapDriver.withInput(new LongWritable(1), new Text(testFile));
    mapDriver.withOutput(
        new Text(testFile),
        Utils.asText(JpylyzerMapper.jpylize(testFile, jpylyzerPath.getAbsolutePath())));
    mapDriver.runTest();
  }
 public static void verifyMapReduce(
     SmartMapper mapper, SmartReducer reducer, Object key, Object input) throws Exception {
   MapDriver mapDriver = new MapDriver();
   mapDriver.setMapper(mapper);
   MapReduceDriver mapReduceDriver = new MapReduceDriver();
   mapReduceDriver.setMapper(mapper);
   Object writableKey = createWritable(key, mapper.getKeyInType());
   Object writableValue = createWritable(input, mapper.getValueInType());
   mapDriver.withInput(writableKey, writableValue);
   List results = mapDriver.run();
   Collections.sort(results, PairComparer.INSTANCE);
   mapReduceDriver =
       new MapReduceDriver<LongWritable, Text, Text, LongWritable, Text, LongWritable>();
   writableKey = createWritable(key, mapper.getKeyInType());
   writableValue = createWritable(input, mapper.getValueInType());
   mapReduceDriver.withInput(writableKey, writableValue);
   mapReduceDriver.setMapper(mapper);
   mapReduceDriver.setReducer(reducer);
   List finalResults = mapReduceDriver.run();
   String text =
       String.format(
           "[%s]\r\n\r\n -> maps via %s to -> \r\n\r\n%s\r\n\r\n -> reduces via %s to -> \r\n\r\n%s",
           input,
           mapper.getClass().getSimpleName(),
           ArrayUtils.toString(results, Echo.INSTANCE),
           reducer.getClass().getSimpleName(),
           ArrayUtils.toString(finalResults, Echo.INSTANCE));
   Approvals.verify(text);
 }
 @Test
 public void testMapper() throws IOException {
   mapDriver.withInput(
       new LongWritable(),
       new Text(
           "Wed Apr 24 23:04:58 2013 [SESSION 2954383360]: Scanning path '/Users/komamitsu/Library/Caches/com.avast.registration/Cache.db' started."));
   mapDriver.withOutput(new Text("Apr 24"), new LongWritable(1));
   mapDriver.runTest();
 }
 @Test
 public void testMapper() throws IOException {
   mapperDriver.withInput(
       new LongWritable(),
       new Text("item_a_id" + "\t" + "item_b_id" + "\t" + "2" + "\t" + "timestamp"));
   mapperDriver.withOutput(
       new Text("item_a_id" + "\t" + "item_b_id"), new Text("4.0" + "\t" + "timestamp"));
   // mapperDriver.runTest();
 }
 @Test(dataProvider = "simpleTest")
 public void simpleTest(String inString, String[] splits, int lineNum, int wordCount)
     throws IOException {
   mapDriver.withInput(new LongWritable(), new Text(inString));
   for (String split : splits) {
     mapDriver.withOutput(new Text(split), new IntWritable(1));
   }
   mapDriver.runTest();
 }
  @Test
  public void testMapper() throws IOException {

    String title = "Test Title";

    String input = XmlTags.TITLE_START + title + XmlTags.TITLE_END;

    mapDriver
        .withInput(new LongWritable(), new Text(input))
        .withOutput(new Text(title), new IntWritable(2))
        .runTest();
  }
  @Test
  public void testSimplest() throws IOException {
    MapDriver<LongWritable, Text, Text, Text> mapDriver;
    JpylyzerMapper mapper = new JpylyzerMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
    mapDriver
        .getConfiguration()
        .set(dk.statsbiblioteket.medieplatform.autonomous.ConfigConstants.JPYLYZER_PATH, "echo");

    mapDriver.withInput(new LongWritable(1), new Text("ein"));
    mapDriver.withOutput(new Text("ein"), new Text("ein"));
    mapDriver.runTest();
  }
 public static void verifyMapping(SmartMapper mapper, Object key, Object input) throws Exception {
   MapDriver mapDriver = new MapDriver();
   mapDriver.setMapper(mapper);
   Object writableKey = createWritable(key, mapper.getKeyInType());
   Object writableValue = createWritable(input, mapper.getValueInType());
   mapDriver.withInput(writableKey, writableValue);
   List results = mapDriver.run();
   Collections.sort(results, PairComparer.INSTANCE);
   String header =
       String.format(
           "[%s]\r\n\r\n -> maps via %s to -> \r\n", input, mapper.getClass().getSimpleName());
   Approvals.verifyAll(header, results, Echo.INSTANCE);
 }
  /**
   * Tests quads to triples conversion
   *
   * @throws IOException
   */
  @Test
  public void triples_to_quads_mapper_01() throws IOException {
    MapDriver<LongWritable, TripleWritable, LongWritable, QuadWritable> driver =
        this.getMapDriver();

    Triple t =
        new Triple(
            NodeFactory.createURI("http://s"),
            NodeFactory.createURI("http://p"),
            NodeFactory.createLiteral("test"));
    Quad q = new Quad(t.getSubject(), t);
    driver
        .withInput(
            new Pair<LongWritable, TripleWritable>(new LongWritable(1), new TripleWritable(t)))
        .withOutput(new Pair<LongWritable, QuadWritable>(new LongWritable(1), new QuadWritable(q)));
    driver.runTest();
  }
 @Test(dataProvider = "simpleTest")
 public void countersTest(String inString, String[] splits, int lineNum, int wordCount)
     throws IOException {
   mapDriver.withInput(new LongWritable(), new Text(inString));
   for (String split : splits) {
     mapDriver.withOutput(new Text(split), new IntWritable(1));
   }
   mapDriver.runTest();
   Counters counters = mapDriver.getCounters();
   assertEquals(
       counters.findCounter(WordCountWithTestsMapper.Status.LINES_NUM).getValue(),
       lineNum,
       "Wrong lines num!");
   assertEquals(
       counters.findCounter(WordCountWithTestsMapper.Status.WORD_COUNT).getValue(),
       wordCount,
       "Wrong word count!");
 }
  @Test
  public void testMapper() throws IOException {

    String input =
        "<article>"
            + "<title>"
            + WORD_1
            + " "
            + WORD_2
            + " "
            + WORD_3
            + " "
            + WORD_4
            + "</title>"
            + "</article>";

    mapDriver
        .withInput(new LongWritable(), new Text(input))
        .withOutput(
            new Text((WORD_1 + " " + WORD_2 + " " + WORD_3).toLowerCase()), new IntWritable(1))
        .withOutput(
            new Text((WORD_2 + " " + WORD_3 + " " + WORD_4).toLowerCase()), new IntWritable(1))
        .runTest();
  }
 @Test
 public void testMapper() throws IOException {
   mapDriver.withInput(new LongWritable(0), inputFilePath);
   mapDriver.withOutput(new LongWritable(0), outputdir);
   mapDriver.runTest();
 }
示例#15
0
 @Test
 public void testMapper() throws IOException {
   mapDriver.withInput(new LongWritable(), new Text(INPUT_LINE));
   mapDriver.withOutput(new Text("ip1"), new IntWritable(40028));
   mapDriver.runTest();
 }