@Test
  public void testScale() throws IOException {

    final ByteArrayId adapterId = new ByteArrayId(ftype.getTypeName());
    final Random r = new Random(3434);
    for (int i = 0; i < 10000; i++) {
      final SimpleFeature feature =
          createTestFeature(
              "f" + i,
              new Coordinate(
                  round(30.0 + (r.nextGaussian() * 0.0001)),
                  round(30.0 + (r.nextGaussian() * 0.0001))));
      mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature.getID())), feature);
    }

    final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults =
        mapDriver.run();

    final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions =
        getReducerDataFromMapperInput(mapperResults);

    reduceDriver.addAll(partitions);

    reduceDriver
        .getConfiguration()
        .setInt(
            GeoWaveConfiguratorBase.enumToConfKey(
                NNMapReduce.class, ClusteringParameters.Clustering.MINIMUM_SIZE),
            10);

    final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();
    assertTrue(reduceResults.size() > 0);
  }
 @Test
 public void testReducer() throws IOException {
   reduceDriver.withInput(
       new Text("Apr 25"),
       Arrays.asList(new LongWritable(3), new LongWritable(5), new LongWritable(9)));
   reduceDriver.withOutput(new Text("Apr 25"), new LongWritable(17));
   reduceDriver.runTest();
 }
 @Test
 public void testReducer() throws IOException {
   List<Text> values = new ArrayList<Text>();
   values.add(outputdir);
   // TODO test multiple values
   reduceDriver.withInput(new LongWritable(0), values);
   reduceDriver.withOutput(new LongWritable(0), new Text(outputdir.toString() + "\n"));
   reduceDriver.runTest();
 }
示例#4
0
 @Test
 public void testReducer() throws IOException {
   List<IntWritable> values = new ArrayList<IntWritable>();
   values.add(new IntWritable(4028));
   values.add(new IntWritable(1024));
   reduceDriver.withInput(new Text("ip1"), values);
   reduceDriver.withOutput(new Text("ip1"), new AggregatedValue(5052, 2526));
   reduceDriver.runTest();
 }
  private void serializations() {
    final String[] strings = reduceDriver.getConfiguration().getStrings("io.serializations");
    final String[] newStrings = new String[strings.length + 1];
    System.arraycopy(strings, 0, newStrings, 0, strings.length);
    newStrings[newStrings.length - 1] = SimpleFeatureImplSerialization.class.getName();
    reduceDriver.getConfiguration().setStrings("io.serializations", newStrings);

    mapDriver.getConfiguration().setStrings("io.serializations", newStrings);
  }
示例#6
0
 @Before
 public void setUp() {
   IpMapper mapper = new IpMapper();
   IpReducer reducer = new IpReducer();
   mapDriver = MapDriver.newMapDriver(mapper);
   reduceDriver = ReduceDriver.newReduceDriver(reducer);
   mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
 }
 @Before
 public void setUp() throws Exception {
   MyMapper mapper = new MyMapper();
   MyReducer reducer = new MyReducer();
   mapDriver = MapDriver.newMapDriver(mapper);
   reduceDriver = ReduceDriver.newReduceDriver(reducer);
   mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
 }
 public static void verifyReducer(SmartReducer reducer, Object key, Object... values)
     throws Exception {
   List list = new ArrayList();
   for (Object value : values) {
     list.add(createWritable(value, reducer.getValueInType()));
   }
   ReduceDriver reduceDriver = new ReduceDriver<Text, LongWritable, Text, LongWritable>();
   reduceDriver.withInput(createWritable(key, reducer.getKeyInType()), list);
   reduceDriver.setReducer(reducer);
   List results = reduceDriver.run();
   Collections.sort(results, PairComparer.INSTANCE);
   String header =
       String.format(
           "(%s, %s)\r\n\r\n -> reduces via %s to -> \r\n",
           key, list, reducer.getClass().getSimpleName());
   Approvals.verifyAll(header, results, Echo.INSTANCE);
 }
  @Before
  public void setup() {

    TrigramMapper mapper = new TrigramMapper();
    TrigramReducer reducer = new TrigramReducer();

    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
  }
  @Before
  public void setup() throws FileNotFoundException {

    TitleWordCountMapper mapper = new TitleWordCountMapper();
    TitleWordCountReducer reducer = new TitleWordCountReducer();

    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
  }
示例#11
0
  @Before
  public void setUp() throws IOException {

    mapDriver = MapDriver.newMapDriver(nnMapper);
    reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

    mapDriver
        .getConfiguration()
        .set(
            GeoWaveConfiguratorBase.enumToConfKey(
                NNMapReduce.class, PartitionParameters.Partition.DISTANCE_THRESHOLDS),
            "10,10");

    reduceDriver
        .getConfiguration()
        .setDouble(
            GeoWaveConfiguratorBase.enumToConfKey(
                NNMapReduce.class, PartitionParameters.Partition.MAX_DISTANCE),
            10);

    ftype =
        AnalyticFeature.createGeometryFeatureAdapter(
                "centroid",
                new String[] {"extra1"},
                BasicFeatureTypes.DEFAULT_NAMESPACE,
                ClusteringUtils.CLUSTERING_CRS)
            .getType();

    reduceDriver
        .getConfiguration()
        .setClass(
            GeoWaveConfiguratorBase.enumToConfKey(
                DBScanMapReduce.class, HullParameters.Hull.PROJECTION_CLASS),
            SimpleFeatureProjection.class,
            Projection.class);

    JobContextAdapterStore.addDataAdapter(
        mapDriver.getConfiguration(), new FeatureDataAdapter(ftype));

    JobContextAdapterStore.addDataAdapter(
        reduceDriver.getConfiguration(), new FeatureDataAdapter(ftype));

    serializations();
  }
示例#12
0
  @Before
  public void setUp() throws Exception {
    PymkMapper mapper = new PymkMapper();
    mapDriver = MapDriver.newMapDriver(mapper);

    PymkReducer reducer = new PymkReducer();
    reducerDriver = ReduceDriver.newReduceDriver(reducer);

    mapRedDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
  }
  @Test
  public void testIdentityMapper() throws IOException {
    List<Pair<Text, Text>> results =
        driver
            .withInput(new Text("foo"), Arrays.asList(new Text("bar1"), new Text("bar2")))
            .withOutput(new Text("foo"), new Text("bar1"))
            .withOutput(new Text("foo"), new Text("bar2"))
            .run();

    MRUnitJUnitAsserts.assertOutputs(driver, results);
  }
  @Test
  public void testReducer() throws IOException, InterruptedException {
    ReduceDriver<IntTriple, IntVector1, IntWritable, IntVector> driver =
        new ReduceDriver<IntTriple, IntVector1, IntWritable, IntVector>();
    driver.withReducer(new MatrixReducer());

    IntWritable[] a = {
      new IntWritable(1), new IntWritable(1), new IntWritable(3), new IntWritable(1)
    };
    IntWritable[] b1 = {
      new IntWritable(2), new IntWritable(1), new IntWritable(3), new IntWritable(1)
    };
    IntWritable[] b2 = {
      new IntWritable(1),
      new IntWritable(1),
      new IntWritable(2),
      new IntWritable(1),
      new IntWritable(3),
      new IntWritable(1)
    };
    IntWritable[] b3 = {new IntWritable(1), new IntWritable(1)};
    driver.addInput(
        new IntTriple(1, 2, 1),
        Arrays.asList(
            new IntVector1(new ArrayWritable(IntWritable.class, a), new IntWritable(-1)),
            new IntVector1(new ArrayWritable(IntWritable.class, b1), new IntWritable(1)),
            new IntVector1(new ArrayWritable(IntWritable.class, b2), new IntWritable(2)),
            new IntVector1(new ArrayWritable(IntWritable.class, b3), new IntWritable(3))));
    IntWritable[] c = {
      new IntWritable(1),
      new IntWritable(1),
      new IntWritable(2),
      new IntWritable(1),
      new IntWritable(3),
      new IntWritable(1)
    };
    driver.addOutput(new IntWritable(1), new IntVector(new ArrayWritable(IntWritable.class, c)));
    driver.runTest();
  }
示例#15
0
  @Test
  public void testReducer() throws IOException {
    reducerDriver.withInput(
        new Text("Marco"),
        Arrays.asList(
            new TextPair("Fabio", "FRIENDS"),
            new TextPair("Luca", "FRIENDS"),
            new TextPair("Gordon", "FRIENDS"),
            new TextPair("Nathalia", "Fabio"),
            new TextPair("Nathalia", "Luca")));

    reducerDriver.withInput(
        new Text("Gordon"),
        Arrays.asList(
            new TextPair("Fabio", "Marco"),
            new TextPair("Marco", "FRIENDS"),
            new TextPair("Luca", "Marco")));

    reducerDriver.withInput(
        new Text("Luca"),
        Arrays.asList(
            new TextPair("Fabio", "Nathalia"),
            new TextPair("Fabio", "Marco"),
            new TextPair("Gordon", "Marco"),
            new TextPair("Nathalia", "FRIENDS"),
            new TextPair("Marco", "FRIENDS")));

    reducerDriver.withInput(
        new Text("Fabio"),
        Arrays.asList(
            new TextPair("Luca", "Marco"),
            new TextPair("Luca", "Nathalia"),
            new TextPair("Gordon", "Marco"),
            new TextPair("Nathalia", "FRIENDS"),
            new TextPair("Marco", "FRIENDS")));

    reducerDriver.withInput(
        new Text("Nathalia"),
        Arrays.asList(
            new TextPair("Marco", "Fabio"),
            new TextPair("Marco", "Luca"),
            new TextPair("Luca", "FRIENDS"),
            new TextPair("Fabio", "FRIENDS")));

    reducerDriver.withAllOutput(
        Arrays.asList(
            new Pair<>(new Text("Gordon"), new Text("Luca(1:[Marco]),Fabio(1:[Marco])")),
            new Pair<>(new Text("Luca"), new Text("Gordon(1:[Marco]),Fabio(2:[Nathalia, Marco])")),
            new Pair<>(new Text("Marco"), new Text("Nathalia(2:[Fabio, Luca])")),
            new Pair<>(new Text("Fabio"), new Text("Gordon(1:[Marco]),Luca(2:[Marco, Nathalia])")),
            new Pair<>(new Text("Nathalia"), new Text("Marco(2:[Fabio, Luca])"))));

    reducerDriver.runTest(false);
  }
 @Before
 public void setUp() {
   AudioQASettings.MAPPER_OUTPUT_DIR = "output/MigrateMp3ToWav/";
   MigrationMapper mapper = new MigrationMapper();
   MigrateMp3ToWav.MigrationReducer reducer = new MigrateMp3ToWav.MigrationReducer();
   mapDriver = new MapDriver<LongWritable, Text, LongWritable, Text>();
   mapDriver.setMapper(mapper);
   reduceDriver = new ReduceDriver<LongWritable, Text, LongWritable, Text>();
   reduceDriver.setReducer(reducer);
   mapReduceDriver =
       new MapReduceDriver<LongWritable, Text, LongWritable, Text, LongWritable, Text>();
   mapReduceDriver.setMapper(mapper);
   mapReduceDriver.setReducer(reducer);
 }
  @Test
  public void testReducer() throws IOException {

    List<IntWritable> values1 = new ArrayList<IntWritable>();
    List<IntWritable> values2 = new ArrayList<IntWritable>();

    values1.add(new IntWritable(1));
    values1.add(new IntWritable(1));

    values2.add(new IntWritable(1));

    String input1 = WORD_1 + " " + WORD_2 + " " + WORD_3,
        input2 = WORD_2 + " " + WORD_3 + " " + WORD_4;

    reduceDriver
        .withInput(new Text(input1), values1)
        .withInput(new Text(input2), values2)
        .withOutput(new Text(String.format(StringFormat.POPULARITY, 1, input1)), new IntWritable(2))
        .withOutput(new Text(String.format(StringFormat.POPULARITY, 2, input2)), new IntWritable(1))
        .runTest();
  }
  @Test
  public void testReducer() throws IOException {

    List<IntWritable> values1 = new ArrayList<IntWritable>();
    List<IntWritable> values2 = new ArrayList<IntWritable>();
    List<IntWritable> values3 = new ArrayList<IntWritable>();

    values1.add(new IntWritable(1));
    values2.add(new IntWritable(10));
    values3.add(new IntWritable(20));

    // Number of words should already be counted in mapper, no need to use titles with corresponding
    // number of words
    Text title1 = new Text("Title1");
    Text title2 = new Text("Title2");
    Text title3 = new Text("Title3");

    reduceDriver
        .withInput(title1, values1)
        .withInput(title2, values2)
        .withInput(title3, values3)
        .withOutput(new IntWritable(2), NullWritable.get())
        .runTest();
  }
示例#19
0
  @Test
  public void testReducer() throws IOException {

    final ByteArrayId adapterId = new ByteArrayId(ftype.getTypeName());

    final SimpleFeature feature1 = createTestFeature("f1", new Coordinate(30.0, 30.00000001));
    final SimpleFeature feature2 = createTestFeature("f2", new Coordinate(50.001, 50.001));
    final SimpleFeature feature3 =
        createTestFeature("f3", new Coordinate(30.00000001, 30.00000001));
    final SimpleFeature feature4 = createTestFeature("f4", new Coordinate(50.0011, 50.00105));
    final SimpleFeature feature5 = createTestFeature("f5", new Coordinate(50.00112, 50.00111));
    final SimpleFeature feature6 =
        createTestFeature("f6", new Coordinate(30.00000001, 30.00000002));
    final SimpleFeature feature7 = createTestFeature("f7", new Coordinate(50.00113, 50.00114));
    final SimpleFeature feature8 =
        createTestFeature("f8", new Coordinate(40.00000001, 40.000000002));

    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature1.getID())), feature1);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature2.getID())), feature2);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature3.getID())), feature3);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature4.getID())), feature4);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature5.getID())), feature5);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature6.getID())), feature6);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature7.getID())), feature7);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature8.getID())), feature8);

    final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults =
        mapDriver.run();
    assertNotNull(getPartitionDataFor(mapperResults, feature1.getID(), true));
    assertNotNull(getPartitionDataFor(mapperResults, feature2.getID(), true));
    assertNotNull(getPartitionDataFor(mapperResults, feature2.getID(), true));
    assertNotNull(getPartitionDataFor(mapperResults, feature3.getID(), true));

    assertEquals(
        getPartitionDataFor(mapperResults, feature1.getID(), true).getId(),
        getPartitionDataFor(mapperResults, feature3.getID(), true).getId());

    assertEquals(
        getPartitionDataFor(mapperResults, feature6.getID(), true).getId(),
        getPartitionDataFor(mapperResults, feature3.getID(), true).getId());

    assertEquals(
        getPartitionDataFor(mapperResults, feature5.getID(), true).getId(),
        getPartitionDataFor(mapperResults, feature7.getID(), true).getId());

    assertEquals(
        getPartitionDataFor(mapperResults, feature5.getID(), true).getId(),
        getPartitionDataFor(mapperResults, feature4.getID(), true).getId());

    final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions =
        getReducerDataFromMapperInput(mapperResults);

    reduceDriver.addAll(partitions);

    reduceDriver
        .getConfiguration()
        .setInt(
            GeoWaveConfiguratorBase.enumToConfKey(
                NNMapReduce.class, ClusteringParameters.Clustering.MINIMUM_SIZE),
            2);

    final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();

    assertEquals(2, reduceResults.size());

    /*
     * assertEquals( feature3.getID(), find( reduceResults,
     * feature1.getID()).toString());
     *
     * assertEquals( feature1.getID(), find( reduceResults,
     * feature3.getID()).toString());
     *
     * assertEquals( feature4.getID(), find( reduceResults,
     * feature2.getID()).toString());
     *
     * assertEquals( feature2.getID(), find( reduceResults,
     * feature4.getID()).toString());
     */
  }