public static void verifyMapReduce(
SmartMapper mapper, SmartReducer reducer, Object key, Object input) throws Exception {
MapDriver mapDriver = new MapDriver();
mapDriver.setMapper(mapper);
MapReduceDriver mapReduceDriver = new MapReduceDriver();
mapReduceDriver.setMapper(mapper);
Object writableKey = createWritable(key, mapper.getKeyInType());
Object writableValue = createWritable(input, mapper.getValueInType());
mapDriver.withInput(writableKey, writableValue);
List results = mapDriver.run();
Collections.sort(results, PairComparer.INSTANCE);
mapReduceDriver =
new MapReduceDriver<LongWritable, Text, Text, LongWritable, Text, LongWritable>();
writableKey = createWritable(key, mapper.getKeyInType());
writableValue = createWritable(input, mapper.getValueInType());
mapReduceDriver.withInput(writableKey, writableValue);
mapReduceDriver.setMapper(mapper);
mapReduceDriver.setReducer(reducer);
List finalResults = mapReduceDriver.run();
String text =
String.format(
"[%s]\r\n\r\n -> maps via %s to -> \r\n\r\n%s\r\n\r\n -> reduces via %s to -> \r\n\r\n%s",
input,
mapper.getClass().getSimpleName(),
ArrayUtils.toString(results, Echo.INSTANCE),
reducer.getClass().getSimpleName(),
ArrayUtils.toString(finalResults, Echo.INSTANCE));
Approvals.verify(text);
}
@Test
public void testScale() throws IOException {
final ByteArrayId adapterId = new ByteArrayId(ftype.getTypeName());
final Random r = new Random(3434);
for (int i = 0; i < 10000; i++) {
final SimpleFeature feature =
createTestFeature(
"f" + i,
new Coordinate(
round(30.0 + (r.nextGaussian() * 0.0001)),
round(30.0 + (r.nextGaussian() * 0.0001))));
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature.getID())), feature);
}
final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults =
mapDriver.run();
final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions =
getReducerDataFromMapperInput(mapperResults);
reduceDriver.addAll(partitions);
reduceDriver
.getConfiguration()
.setInt(
GeoWaveConfiguratorBase.enumToConfKey(
NNMapReduce.class, ClusteringParameters.Clustering.MINIMUM_SIZE),
10);
final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();
assertTrue(reduceResults.size() > 0);
}
/**
* Tests quads to triples conversion
*
* @throws IOException
*/
@Test
public void triples_to_quads_mapper_02() throws IOException {
MapDriver<LongWritable, TripleWritable, LongWritable, QuadWritable> driver =
this.getMapDriver();
this.generateData(driver, 100);
driver.runTest();
}
@Test
public void testMapper() throws IOException {
mapperDriver.withInput(
new LongWritable(),
new Text("item_a_id" + "\t" + "item_b_id" + "\t" + "2" + "\t" + "timestamp"));
mapperDriver.withOutput(
new Text("item_a_id" + "\t" + "item_b_id"), new Text("4.0" + "\t" + "timestamp"));
// mapperDriver.runTest();
}
@Test(dataProvider = "simpleTest")
public void simpleTest(String inString, String[] splits, int lineNum, int wordCount)
throws IOException {
mapDriver.withInput(new LongWritable(), new Text(inString));
for (String split : splits) {
mapDriver.withOutput(new Text(split), new IntWritable(1));
}
mapDriver.runTest();
}
@Test
public void testMapper() throws IOException {
mapDriver.withInput(
new LongWritable(),
new Text(
"Wed Apr 24 23:04:58 2013 [SESSION 2954383360]: Scanning path '/Users/komamitsu/Library/Caches/com.avast.registration/Cache.db' started."));
mapDriver.withOutput(new Text("Apr 24"), new LongWritable(1));
mapDriver.runTest();
}
@Test
public void processValidRecord() throws IOException {
mapDriver.withInput(new Text("a"), new Text("ein"));
mapDriver.withInput(new Text("a"), new Text("zwei"));
mapDriver.withInput(new Text("c"), new Text("drei"));
mapDriver.withOutput(new Text("a"), new Text("ein"));
mapDriver.withOutput(new Text("a"), new Text("zwei"));
mapDriver.withOutput(new Text("c"), new Text("drei"));
mapDriver.runTest();
}
protected void generateData(
MapDriver<LongWritable, TripleWritable, LongWritable, QuadWritable> driver, int num) {
for (int i = 0; i < num; i++) {
Triple t =
new Triple(
NodeFactory.createURI("http://subjects/" + i),
NodeFactory.createURI("http://predicate"),
NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger));
Quad q = new Quad(t.getSubject(), t);
driver.addInput(new LongWritable(i), new TripleWritable(t));
driver.addOutput(new LongWritable(i), new QuadWritable(q));
}
}
public static void verifyMapping(SmartMapper mapper, Object key, Object input) throws Exception {
MapDriver mapDriver = new MapDriver();
mapDriver.setMapper(mapper);
Object writableKey = createWritable(key, mapper.getKeyInType());
Object writableValue = createWritable(input, mapper.getValueInType());
mapDriver.withInput(writableKey, writableValue);
List results = mapDriver.run();
Collections.sort(results, PairComparer.INSTANCE);
String header =
String.format(
"[%s]\r\n\r\n -> maps via %s to -> \r\n", input, mapper.getClass().getSimpleName());
Approvals.verifyAll(header, results, Echo.INSTANCE);
}
/**
* Tests quads to triples conversion
*
* @throws IOException
*/
@Test
public void triples_to_quads_mapper_01() throws IOException {
MapDriver<LongWritable, TripleWritable, LongWritable, QuadWritable> driver =
this.getMapDriver();
Triple t =
new Triple(
NodeFactory.createURI("http://s"),
NodeFactory.createURI("http://p"),
NodeFactory.createLiteral("test"));
Quad q = new Quad(t.getSubject(), t);
driver
.withInput(
new Pair<LongWritable, TripleWritable>(new LongWritable(1), new TripleWritable(t)))
.withOutput(new Pair<LongWritable, QuadWritable>(new LongWritable(1), new QuadWritable(q)));
driver.runTest();
}
@Before
public void setUp() {
IpMapper mapper = new IpMapper();
IpReducer reducer = new IpReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
@Before
public void setUp() throws Exception {
MyMapper mapper = new MyMapper();
MyReducer reducer = new MyReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
@Before
public void setup() {
TrigramMapper mapper = new TrigramMapper();
TrigramReducer reducer = new TrigramReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
@Test(dataProvider = "simpleTest")
public void countersTest(String inString, String[] splits, int lineNum, int wordCount)
throws IOException {
mapDriver.withInput(new LongWritable(), new Text(inString));
for (String split : splits) {
mapDriver.withOutput(new Text(split), new IntWritable(1));
}
mapDriver.runTest();
Counters counters = mapDriver.getCounters();
assertEquals(
counters.findCounter(WordCountWithTestsMapper.Status.LINES_NUM).getValue(),
lineNum,
"Wrong lines num!");
assertEquals(
counters.findCounter(WordCountWithTestsMapper.Status.WORD_COUNT).getValue(),
wordCount,
"Wrong word count!");
}
private void serializations() {
final String[] strings = reduceDriver.getConfiguration().getStrings("io.serializations");
final String[] newStrings = new String[strings.length + 1];
System.arraycopy(strings, 0, newStrings, 0, strings.length);
newStrings[newStrings.length - 1] = SimpleFeatureImplSerialization.class.getName();
reduceDriver.getConfiguration().setStrings("io.serializations", newStrings);
mapDriver.getConfiguration().setStrings("io.serializations", newStrings);
}
@Before
public void setup() throws FileNotFoundException {
TitleWordCountMapper mapper = new TitleWordCountMapper();
TitleWordCountReducer reducer = new TitleWordCountReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
@Before
public void setUp() throws IOException {
mapDriver = MapDriver.newMapDriver(nnMapper);
reduceDriver = ReduceDriver.newReduceDriver(nnReducer);
mapDriver
.getConfiguration()
.set(
GeoWaveConfiguratorBase.enumToConfKey(
NNMapReduce.class, PartitionParameters.Partition.DISTANCE_THRESHOLDS),
"10,10");
reduceDriver
.getConfiguration()
.setDouble(
GeoWaveConfiguratorBase.enumToConfKey(
NNMapReduce.class, PartitionParameters.Partition.MAX_DISTANCE),
10);
ftype =
AnalyticFeature.createGeometryFeatureAdapter(
"centroid",
new String[] {"extra1"},
BasicFeatureTypes.DEFAULT_NAMESPACE,
ClusteringUtils.CLUSTERING_CRS)
.getType();
reduceDriver
.getConfiguration()
.setClass(
GeoWaveConfiguratorBase.enumToConfKey(
DBScanMapReduce.class, HullParameters.Hull.PROJECTION_CLASS),
SimpleFeatureProjection.class,
Projection.class);
JobContextAdapterStore.addDataAdapter(
mapDriver.getConfiguration(), new FeatureDataAdapter(ftype));
JobContextAdapterStore.addDataAdapter(
reduceDriver.getConfiguration(), new FeatureDataAdapter(ftype));
serializations();
}
@Before
public void setUp() throws Exception {
PymkMapper mapper = new PymkMapper();
mapDriver = MapDriver.newMapDriver(mapper);
PymkReducer reducer = new PymkReducer();
reducerDriver = ReduceDriver.newReduceDriver(reducer);
mapRedDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
@Test
public void testMapper() throws IOException {
String title = "Test Title";
String input = XmlTags.TITLE_START + title + XmlTags.TITLE_END;
mapDriver
.withInput(new LongWritable(), new Text(input))
.withOutput(new Text(title), new IntWritable(2))
.runTest();
}
@Test
public void testMapper() throws IOException, URISyntaxException {
MapDriver<LongWritable, Text, Text, Text> mapDriver;
JpylyzerMapper mapper = new JpylyzerMapper();
String name = "B400022028241-RT1/balloon.jp2";
mapDriver = MapDriver.newMapDriver(mapper);
File testFolder =
new File(Thread.currentThread().getContextClassLoader().getResource(name).toURI())
.getParentFile()
.getParentFile()
.getParentFile()
.getParentFile();
File jpylyzerPath = new File(testFolder, "src/test/extras/jpylyzer-1.10.1/jpylyzer.py");
mapDriver
.getConfiguration()
.set(
dk.statsbiblioteket.medieplatform.autonomous.ConfigConstants.JPYLYZER_PATH,
jpylyzerPath.getAbsolutePath());
String testFile = getAbsolutePath(name);
mapDriver.withInput(new LongWritable(1), new Text(testFile));
mapDriver.withOutput(
new Text(testFile),
Utils.asText(JpylyzerMapper.jpylize(testFile, jpylyzerPath.getAbsolutePath())));
mapDriver.runTest();
}
@Test
public void shouldOutputAsPropertiesWithVeryLongPropValueAndLargeNodeSizeAndHighEdgeId()
throws Exception {
driver.setConfiguration(populateConfigWithMetaDataWithLargeNodeSize());
output =
driver
.withInputKey(new Text("700000000"))
.withInputValue(
new Text(
"B C Prop1 Prop2 AndAPropWithAverryLongvalueBecauseWeKindOfNeedToFindOutIfABlockCountIsPresentAndCorrectAndAPropWithAverryLongvalueBecauseWeKindOfNeedToFindOutIfABlockCountIsPresentAndCorrect"))
.run();
FullEdgePropertiesWritable edge1 =
new FullEdgePropertiesWritable(700000000, "B", "C", 0, 0, -1, -1, 5, "Prop1");
edge1.add(6, "Prop2", 0);
FullEdgePropertiesWritable edge2 =
new FullEdgePropertiesWritable(
700000000,
"B",
"C",
1,
2,
-1,
-1,
7,
"AndAPropWithAverryLongvalueBecauseWeKindOfNeedToFindOutIfABlockCountIsPresentAndCorrectAndAPropWithAverryLongvalueBecauseWeKindOfNeedToFindOutIfABlockCountIsPresentAndCorrect");
assertThat(output.size(), is(2));
assertThat(
output.get(0).getFirst(),
equalTo(new AscLongDescLongWritable(new LongWritable(700000000L), new LongWritable(0))));
assertThat(output.get(0).getSecond(), equalTo(edge1));
assertThat(
output.get(1).getFirst(),
equalTo(new AscLongDescLongWritable(new LongWritable(700000000L), new LongWritable(1))));
assertThat(output.get(1).getSecond(), equalTo(edge2));
}
@Before
public void setUp() {
AudioQASettings.MAPPER_OUTPUT_DIR = "output/MigrateMp3ToWav/";
MigrationMapper mapper = new MigrationMapper();
MigrateMp3ToWav.MigrationReducer reducer = new MigrateMp3ToWav.MigrationReducer();
mapDriver = new MapDriver<LongWritable, Text, LongWritable, Text>();
mapDriver.setMapper(mapper);
reduceDriver = new ReduceDriver<LongWritable, Text, LongWritable, Text>();
reduceDriver.setReducer(reducer);
mapReduceDriver =
new MapReduceDriver<LongWritable, Text, LongWritable, Text, LongWritable, Text>();
mapReduceDriver.setMapper(mapper);
mapReduceDriver.setReducer(reducer);
}
@Test
public void shouldOutputAsProperties() throws Exception {
driver.setConfiguration(populateConfigWithMetaData());
output =
driver
.withInputKey(new Text("0"))
.withInputValue(
new Text(
"A B Prop3 Prop4 AndAPropWithAverryLongvalueBecauseWeKindOfNeedToFindOutIfABlockCountIsPresentAndCorrect"))
.run();
FullEdgePropertiesWritable edge1 =
new FullEdgePropertiesWritable(0, "A", "B", 0, 0, -1, -1, 5, "Prop3");
edge1.add(6, "Prop4", 0);
FullEdgePropertiesWritable edge2 =
new FullEdgePropertiesWritable(
0,
"A",
"B",
1,
1,
-1,
-1,
7,
"AndAPropWithAverryLongvalueBecauseWeKindOfNeedToFindOutIfABlockCountIsPresentAndCorrect");
assertThat(output.size(), is(2));
assertThat(
output.get(0).getFirst(),
equalTo(new AscLongDescLongWritable(new LongWritable(0), new LongWritable(0))));
assertThat(output.get(0).getSecond(), equalTo(edge1));
assertThat(
output.get(1).getFirst(),
equalTo(new AscLongDescLongWritable(new LongWritable(0), new LongWritable(1))));
assertThat(output.get(1).getSecond(), equalTo(edge2));
}
@Test
public void testSimplest() throws IOException {
MapDriver<LongWritable, Text, Text, Text> mapDriver;
JpylyzerMapper mapper = new JpylyzerMapper();
mapDriver = MapDriver.newMapDriver(mapper);
mapDriver
.getConfiguration()
.set(dk.statsbiblioteket.medieplatform.autonomous.ConfigConstants.JPYLYZER_PATH, "echo");
mapDriver.withInput(new LongWritable(1), new Text("ein"));
mapDriver.withOutput(new Text("ein"), new Text("ein"));
mapDriver.runTest();
}
@Test
public void testMapper() throws IOException {
String input =
"<article>"
+ "<title>"
+ WORD_1
+ " "
+ WORD_2
+ " "
+ WORD_3
+ " "
+ WORD_4
+ "</title>"
+ "</article>";
mapDriver
.withInput(new LongWritable(), new Text(input))
.withOutput(
new Text((WORD_1 + " " + WORD_2 + " " + WORD_3).toLowerCase()), new IntWritable(1))
.withOutput(
new Text((WORD_2 + " " + WORD_3 + " " + WORD_4).toLowerCase()), new IntWritable(1))
.runTest();
}
@Test
public void testMapper() throws IOException {
mapDriver.withInput(new LongWritable(), new Text(INPUT_LINE));
mapDriver.withOutput(new Text("ip1"), new IntWritable(40028));
mapDriver.runTest();
}
@Test
public void testMapper() throws IOException {
mapDriver.withInput(new LongWritable(0), inputFilePath);
mapDriver.withOutput(new LongWritable(0), outputdir);
mapDriver.runTest();
}
@Before
public void setup() {
WordMapper mapper = new WordMapper();
mapDriver = MapDriver.newMapDriver(mapper);
}
@Test
public void testMap() throws IOException {
mapDriver.withInput(new LongWritable(1), new Text("Marco-Fabio,Luca,Gordon"));
mapDriver.withInput(new LongWritable(2), new Text("Fabio-Nathalia,Marco"));
mapDriver.withInput(new LongWritable(3), new Text("Luca-Nathalia,Marco"));
mapDriver.withInput(new LongWritable(4), new Text("Nathalia-Fabio,Luca"));
mapDriver.withInput(new LongWritable(5), new Text("Gordon-Marco"));
mapDriver.withOutput(new Text("Marco"), new TextPair("Fabio", "FRIENDS"));
mapDriver.withOutput(new Text("Marco"), new TextPair("Luca", "FRIENDS"));
mapDriver.withOutput(new Text("Marco"), new TextPair("Gordon", "FRIENDS"));
mapDriver.withOutput(new Text("Luca"), new TextPair("Fabio", "Marco"));
mapDriver.withOutput(new Text("Fabio"), new TextPair("Luca", "Marco"));
mapDriver.withOutput(new Text("Gordon"), new TextPair("Fabio", "Marco"));
mapDriver.withOutput(new Text("Fabio"), new TextPair("Gordon", "Marco"));
mapDriver.withOutput(new Text("Gordon"), new TextPair("Luca", "Marco"));
mapDriver.withOutput(new Text("Luca"), new TextPair("Gordon", "Marco"));
mapDriver.withOutput(new Text("Fabio"), new TextPair("Nathalia", "FRIENDS"));
mapDriver.withOutput(new Text("Fabio"), new TextPair("Marco", "FRIENDS"));
mapDriver.withOutput(new Text("Marco"), new TextPair("Nathalia", "Fabio"));
mapDriver.withOutput(new Text("Nathalia"), new TextPair("Marco", "Fabio"));
mapDriver.withOutput(new Text("Luca"), new TextPair("Nathalia", "FRIENDS"));
mapDriver.withOutput(new Text("Luca"), new TextPair("Marco", "FRIENDS"));
mapDriver.withOutput(new Text("Marco"), new TextPair("Nathalia", "Luca"));
mapDriver.withOutput(new Text("Nathalia"), new TextPair("Marco", "Luca"));
mapDriver.withOutput(new Text("Nathalia"), new TextPair("Fabio", "FRIENDS"));
mapDriver.withOutput(new Text("Nathalia"), new TextPair("Luca", "FRIENDS"));
mapDriver.withOutput(new Text("Luca"), new TextPair("Fabio", "Nathalia"));
mapDriver.withOutput(new Text("Fabio"), new TextPair("Luca", "Nathalia"));
mapDriver.withOutput(new Text("Gordon"), new TextPair("Marco", "FRIENDS"));
mapDriver.runTest();
}
@Test
public void testReducer() throws IOException {
final ByteArrayId adapterId = new ByteArrayId(ftype.getTypeName());
final SimpleFeature feature1 = createTestFeature("f1", new Coordinate(30.0, 30.00000001));
final SimpleFeature feature2 = createTestFeature("f2", new Coordinate(50.001, 50.001));
final SimpleFeature feature3 =
createTestFeature("f3", new Coordinate(30.00000001, 30.00000001));
final SimpleFeature feature4 = createTestFeature("f4", new Coordinate(50.0011, 50.00105));
final SimpleFeature feature5 = createTestFeature("f5", new Coordinate(50.00112, 50.00111));
final SimpleFeature feature6 =
createTestFeature("f6", new Coordinate(30.00000001, 30.00000002));
final SimpleFeature feature7 = createTestFeature("f7", new Coordinate(50.00113, 50.00114));
final SimpleFeature feature8 =
createTestFeature("f8", new Coordinate(40.00000001, 40.000000002));
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature1.getID())), feature1);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature2.getID())), feature2);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature3.getID())), feature3);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature4.getID())), feature4);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature5.getID())), feature5);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature6.getID())), feature6);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature7.getID())), feature7);
mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArrayId(feature8.getID())), feature8);
final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults =
mapDriver.run();
assertNotNull(getPartitionDataFor(mapperResults, feature1.getID(), true));
assertNotNull(getPartitionDataFor(mapperResults, feature2.getID(), true));
assertNotNull(getPartitionDataFor(mapperResults, feature2.getID(), true));
assertNotNull(getPartitionDataFor(mapperResults, feature3.getID(), true));
assertEquals(
getPartitionDataFor(mapperResults, feature1.getID(), true).getId(),
getPartitionDataFor(mapperResults, feature3.getID(), true).getId());
assertEquals(
getPartitionDataFor(mapperResults, feature6.getID(), true).getId(),
getPartitionDataFor(mapperResults, feature3.getID(), true).getId());
assertEquals(
getPartitionDataFor(mapperResults, feature5.getID(), true).getId(),
getPartitionDataFor(mapperResults, feature7.getID(), true).getId());
assertEquals(
getPartitionDataFor(mapperResults, feature5.getID(), true).getId(),
getPartitionDataFor(mapperResults, feature4.getID(), true).getId());
final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions =
getReducerDataFromMapperInput(mapperResults);
reduceDriver.addAll(partitions);
reduceDriver
.getConfiguration()
.setInt(
GeoWaveConfiguratorBase.enumToConfKey(
NNMapReduce.class, ClusteringParameters.Clustering.MINIMUM_SIZE),
2);
final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();
assertEquals(2, reduceResults.size());
/*
* assertEquals( feature3.getID(), find( reduceResults,
* feature1.getID()).toString());
*
* assertEquals( feature1.getID(), find( reduceResults,
* feature3.getID()).toString());
*
* assertEquals( feature4.getID(), find( reduceResults,
* feature2.getID()).toString());
*
* assertEquals( feature2.getID(), find( reduceResults,
* feature4.getID()).toString());
*/
}
