@Test
public void testScale() throws IOException {
final ByteArrayId adapterId = new ByteArrayId(ftype.getTypeName());
final Random r = new Random(3434);
for (int i = 0; i < 10000; i++) {
final SimpleFeature feature =
createTestFeature(
"f" + i,
new Coordinate(
round(30.0 + (r.nextGaussian() * 0.0001)),
round(30.0 + (r.nextGaussian() * 0.0001))));
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature.getID())), feature);
}
final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults =
mapDriver.run();
final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions =
getReducerDataFromMapperInput(mapperResults);
reduceDriver.addAll(partitions);
reduceDriver
.getConfiguration()
.setInt(
GeoWaveConfiguratorBase.enumToConfKey(
NNMapReduce.class, ClusteringParameters.Clustering.MINIMUM_SIZE),
10);
final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();
assertTrue(reduceResults.size() > 0);
}
@Test
public void testReducer() throws IOException {
reduceDriver.withInput(
new Text("Apr 25"),
Arrays.asList(new LongWritable(3), new LongWritable(5), new LongWritable(9)));
reduceDriver.withOutput(new Text("Apr 25"), new LongWritable(17));
reduceDriver.runTest();
}
@Test
public void testReducer() throws IOException {
List<Text> values = new ArrayList<Text>();
values.add(outputdir);
// TODO test multiple values
reduceDriver.withInput(new LongWritable(0), values);
reduceDriver.withOutput(new LongWritable(0), new Text(outputdir.toString() + "\n"));
reduceDriver.runTest();
}
@Test
public void testReducer() throws IOException {
List<IntWritable> values = new ArrayList<IntWritable>();
values.add(new IntWritable(4028));
values.add(new IntWritable(1024));
reduceDriver.withInput(new Text("ip1"), values);
reduceDriver.withOutput(new Text("ip1"), new AggregatedValue(5052, 2526));
reduceDriver.runTest();
}
private void serializations() {
final String[] strings = reduceDriver.getConfiguration().getStrings("io.serializations");
final String[] newStrings = new String[strings.length + 1];
System.arraycopy(strings, 0, newStrings, 0, strings.length);
newStrings[newStrings.length - 1] = SimpleFeatureImplSerialization.class.getName();
reduceDriver.getConfiguration().setStrings("io.serializations", newStrings);
mapDriver.getConfiguration().setStrings("io.serializations", newStrings);
}
@Before
public void setUp() {
IpMapper mapper = new IpMapper();
IpReducer reducer = new IpReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
@Before
public void setUp() throws Exception {
MyMapper mapper = new MyMapper();
MyReducer reducer = new MyReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
public static void verifyReducer(SmartReducer reducer, Object key, Object... values)
throws Exception {
List list = new ArrayList();
for (Object value : values) {
list.add(createWritable(value, reducer.getValueInType()));
}
ReduceDriver reduceDriver = new ReduceDriver<Text, LongWritable, Text, LongWritable>();
reduceDriver.withInput(createWritable(key, reducer.getKeyInType()), list);
reduceDriver.setReducer(reducer);
List results = reduceDriver.run();
Collections.sort(results, PairComparer.INSTANCE);
String header =
String.format(
"(%s, %s)\r\n\r\n -> reduces via %s to -> \r\n",
key, list, reducer.getClass().getSimpleName());
Approvals.verifyAll(header, results, Echo.INSTANCE);
}
@Before
public void setup() {
TrigramMapper mapper = new TrigramMapper();
TrigramReducer reducer = new TrigramReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
@Before
public void setup() throws FileNotFoundException {
TitleWordCountMapper mapper = new TitleWordCountMapper();
TitleWordCountReducer reducer = new TitleWordCountReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
@Before
public void setUp() throws IOException {
mapDriver = MapDriver.newMapDriver(nnMapper);
reduceDriver = ReduceDriver.newReduceDriver(nnReducer);
mapDriver
.getConfiguration()
.set(
GeoWaveConfiguratorBase.enumToConfKey(
NNMapReduce.class, PartitionParameters.Partition.DISTANCE_THRESHOLDS),
"10,10");
reduceDriver
.getConfiguration()
.setDouble(
GeoWaveConfiguratorBase.enumToConfKey(
NNMapReduce.class, PartitionParameters.Partition.MAX_DISTANCE),
10);
ftype =
AnalyticFeature.createGeometryFeatureAdapter(
"centroid",
new String[] {"extra1"},
BasicFeatureTypes.DEFAULT_NAMESPACE,
ClusteringUtils.CLUSTERING_CRS)
.getType();
reduceDriver
.getConfiguration()
.setClass(
GeoWaveConfiguratorBase.enumToConfKey(
DBScanMapReduce.class, HullParameters.Hull.PROJECTION_CLASS),
SimpleFeatureProjection.class,
Projection.class);
JobContextAdapterStore.addDataAdapter(
mapDriver.getConfiguration(), new FeatureDataAdapter(ftype));
JobContextAdapterStore.addDataAdapter(
reduceDriver.getConfiguration(), new FeatureDataAdapter(ftype));
serializations();
}
@Before
public void setUp() throws Exception {
PymkMapper mapper = new PymkMapper();
mapDriver = MapDriver.newMapDriver(mapper);
PymkReducer reducer = new PymkReducer();
reducerDriver = ReduceDriver.newReduceDriver(reducer);
mapRedDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
@Test
public void testIdentityMapper() throws IOException {
List<Pair<Text, Text>> results =
driver
.withInput(new Text("foo"), Arrays.asList(new Text("bar1"), new Text("bar2")))
.withOutput(new Text("foo"), new Text("bar1"))
.withOutput(new Text("foo"), new Text("bar2"))
.run();
MRUnitJUnitAsserts.assertOutputs(driver, results);
}
@Test
public void testReducer() throws IOException, InterruptedException {
ReduceDriver<IntTriple, IntVector1, IntWritable, IntVector> driver =
new ReduceDriver<IntTriple, IntVector1, IntWritable, IntVector>();
driver.withReducer(new MatrixReducer());
IntWritable[] a = {
new IntWritable(1), new IntWritable(1), new IntWritable(3), new IntWritable(1)
};
IntWritable[] b1 = {
new IntWritable(2), new IntWritable(1), new IntWritable(3), new IntWritable(1)
};
IntWritable[] b2 = {
new IntWritable(1),
new IntWritable(1),
new IntWritable(2),
new IntWritable(1),
new IntWritable(3),
new IntWritable(1)
};
IntWritable[] b3 = {new IntWritable(1), new IntWritable(1)};
driver.addInput(
new IntTriple(1, 2, 1),
Arrays.asList(
new IntVector1(new ArrayWritable(IntWritable.class, a), new IntWritable(-1)),
new IntVector1(new ArrayWritable(IntWritable.class, b1), new IntWritable(1)),
new IntVector1(new ArrayWritable(IntWritable.class, b2), new IntWritable(2)),
new IntVector1(new ArrayWritable(IntWritable.class, b3), new IntWritable(3))));
IntWritable[] c = {
new IntWritable(1),
new IntWritable(1),
new IntWritable(2),
new IntWritable(1),
new IntWritable(3),
new IntWritable(1)
};
driver.addOutput(new IntWritable(1), new IntVector(new ArrayWritable(IntWritable.class, c)));
driver.runTest();
}
@Test
public void testReducer() throws IOException {
reducerDriver.withInput(
new Text("Marco"),
Arrays.asList(
new TextPair("Fabio", "FRIENDS"),
new TextPair("Luca", "FRIENDS"),
new TextPair("Gordon", "FRIENDS"),
new TextPair("Nathalia", "Fabio"),
new TextPair("Nathalia", "Luca")));
reducerDriver.withInput(
new Text("Gordon"),
Arrays.asList(
new TextPair("Fabio", "Marco"),
new TextPair("Marco", "FRIENDS"),
new TextPair("Luca", "Marco")));
reducerDriver.withInput(
new Text("Luca"),
Arrays.asList(
new TextPair("Fabio", "Nathalia"),
new TextPair("Fabio", "Marco"),
new TextPair("Gordon", "Marco"),
new TextPair("Nathalia", "FRIENDS"),
new TextPair("Marco", "FRIENDS")));
reducerDriver.withInput(
new Text("Fabio"),
Arrays.asList(
new TextPair("Luca", "Marco"),
new TextPair("Luca", "Nathalia"),
new TextPair("Gordon", "Marco"),
new TextPair("Nathalia", "FRIENDS"),
new TextPair("Marco", "FRIENDS")));
reducerDriver.withInput(
new Text("Nathalia"),
Arrays.asList(
new TextPair("Marco", "Fabio"),
new TextPair("Marco", "Luca"),
new TextPair("Luca", "FRIENDS"),
new TextPair("Fabio", "FRIENDS")));
reducerDriver.withAllOutput(
Arrays.asList(
new Pair<>(new Text("Gordon"), new Text("Luca(1:[Marco]),Fabio(1:[Marco])")),
new Pair<>(new Text("Luca"), new Text("Gordon(1:[Marco]),Fabio(2:[Nathalia, Marco])")),
new Pair<>(new Text("Marco"), new Text("Nathalia(2:[Fabio, Luca])")),
new Pair<>(new Text("Fabio"), new Text("Gordon(1:[Marco]),Luca(2:[Marco, Nathalia])")),
new Pair<>(new Text("Nathalia"), new Text("Marco(2:[Fabio, Luca])"))));
reducerDriver.runTest(false);
}
@Before
public void setUp() {
AudioQASettings.MAPPER_OUTPUT_DIR = "output/MigrateMp3ToWav/";
MigrationMapper mapper = new MigrationMapper();
MigrateMp3ToWav.MigrationReducer reducer = new MigrateMp3ToWav.MigrationReducer();
mapDriver = new MapDriver<LongWritable, Text, LongWritable, Text>();
mapDriver.setMapper(mapper);
reduceDriver = new ReduceDriver<LongWritable, Text, LongWritable, Text>();
reduceDriver.setReducer(reducer);
mapReduceDriver =
new MapReduceDriver<LongWritable, Text, LongWritable, Text, LongWritable, Text>();
mapReduceDriver.setMapper(mapper);
mapReduceDriver.setReducer(reducer);
}
@Test
public void testReducer() throws IOException {
List<IntWritable> values1 = new ArrayList<IntWritable>();
List<IntWritable> values2 = new ArrayList<IntWritable>();
values1.add(new IntWritable(1));
values1.add(new IntWritable(1));
values2.add(new IntWritable(1));
String input1 = WORD_1 + " " + WORD_2 + " " + WORD_3,
input2 = WORD_2 + " " + WORD_3 + " " + WORD_4;
reduceDriver
.withInput(new Text(input1), values1)
.withInput(new Text(input2), values2)
.withOutput(new Text(String.format(StringFormat.POPULARITY, 1, input1)), new IntWritable(2))
.withOutput(new Text(String.format(StringFormat.POPULARITY, 2, input2)), new IntWritable(1))
.runTest();
}
@Test
public void testReducer() throws IOException {
List<IntWritable> values1 = new ArrayList<IntWritable>();
List<IntWritable> values2 = new ArrayList<IntWritable>();
List<IntWritable> values3 = new ArrayList<IntWritable>();
values1.add(new IntWritable(1));
values2.add(new IntWritable(10));
values3.add(new IntWritable(20));
// Number of words should already be counted in mapper, no need to use titles with corresponding
// number of words
Text title1 = new Text("Title1");
Text title2 = new Text("Title2");
Text title3 = new Text("Title3");
reduceDriver
.withInput(title1, values1)
.withInput(title2, values2)
.withInput(title3, values3)
.withOutput(new IntWritable(2), NullWritable.get())
.runTest();
}
@Test
public void testReducer() throws IOException {
final ByteArrayId adapterId = new ByteArrayId(ftype.getTypeName());
final SimpleFeature feature1 = createTestFeature("f1", new Coordinate(30.0, 30.00000001));
final SimpleFeature feature2 = createTestFeature("f2", new Coordinate(50.001, 50.001));
final SimpleFeature feature3 =
createTestFeature("f3", new Coordinate(30.00000001, 30.00000001));
final SimpleFeature feature4 = createTestFeature("f4", new Coordinate(50.0011, 50.00105));
final SimpleFeature feature5 = createTestFeature("f5", new Coordinate(50.00112, 50.00111));
final SimpleFeature feature6 =
createTestFeature("f6", new Coordinate(30.00000001, 30.00000002));
final SimpleFeature feature7 = createTestFeature("f7", new Coordinate(50.00113, 50.00114));
final SimpleFeature feature8 =
createTestFeature("f8", new Coordinate(40.00000001, 40.000000002));
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature1.getID())), feature1);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature2.getID())), feature2);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature3.getID())), feature3);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature4.getID())), feature4);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature5.getID())), feature5);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature6.getID())), feature6);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature7.getID())), feature7);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature8.getID())), feature8);
final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults =
mapDriver.run();
assertNotNull(getPartitionDataFor(mapperResults, feature1.getID(), true));
assertNotNull(getPartitionDataFor(mapperResults, feature2.getID(), true));
assertNotNull(getPartitionDataFor(mapperResults, feature2.getID(), true));
assertNotNull(getPartitionDataFor(mapperResults, feature3.getID(), true));
assertEquals(
getPartitionDataFor(mapperResults, feature1.getID(), true).getId(),
getPartitionDataFor(mapperResults, feature3.getID(), true).getId());
assertEquals(
getPartitionDataFor(mapperResults, feature6.getID(), true).getId(),
getPartitionDataFor(mapperResults, feature3.getID(), true).getId());
assertEquals(
getPartitionDataFor(mapperResults, feature5.getID(), true).getId(),
getPartitionDataFor(mapperResults, feature7.getID(), true).getId());
assertEquals(
getPartitionDataFor(mapperResults, feature5.getID(), true).getId(),
getPartitionDataFor(mapperResults, feature4.getID(), true).getId());
final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions =
getReducerDataFromMapperInput(mapperResults);
reduceDriver.addAll(partitions);
reduceDriver
.getConfiguration()
.setInt(
GeoWaveConfiguratorBase.enumToConfKey(
NNMapReduce.class, ClusteringParameters.Clustering.MINIMUM_SIZE),
2);
final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();
assertEquals(2, reduceResults.size());
/*
* assertEquals( feature3.getID(), find( reduceResults,
* feature1.getID()).toString());
*
* assertEquals( feature1.getID(), find( reduceResults,
* feature3.getID()).toString());
*
* assertEquals( feature4.getID(), find( reduceResults,
* feature2.getID()).toString());
*
* assertEquals( feature2.getID(), find( reduceResults,
* feature4.getID()).toString());
*/
}
