/**
* Test characteristic set reduction
*
* @throws IOException
*/
@Test
public void characteristic_set_reducer_06() throws IOException {
MapReduceDriver<
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
NullWritable>
driver = this.getMapReduceDriver();
this.createSet(driver, 2, 1, "http://predicate", "http://other");
this.createSet(driver, 1, 1, "http://other");
driver.runTest(false);
driver = getMapReduceDriver();
createSet(driver, 2, 1, "http://predicate", "http://other");
createSet(driver, 1, 1, "http://other");
List<Pair<CharacteristicSetWritable, NullWritable>> results = driver.run();
for (Pair<CharacteristicSetWritable, NullWritable> pair : results) {
CharacteristicSetWritable cw = pair.getFirst();
boolean expectTwo = cw.hasCharacteristic("http://predicate");
Assert.assertEquals(expectTwo ? 2 : 1, cw.getCount().get());
}
}
@org.testng.annotations.Test(groups = {"fast"})
public void sortUsagesTest() throws ParseException {
mapReduceDriver.addInput(new Text(RES1_ID), new LongWritable(COUNT1));
mapReduceDriver.addInput(new Text(RES2_ID), new LongWritable(COUNT2));
mapReduceDriver.addInput(new Text(RES3_ID), new LongWritable(COUNT3));
MostPopularProtos.MostPopularStats.Builder statsBuilder =
MostPopularProtos.MostPopularStats.newBuilder();
statsBuilder.setTimestamp(new SimpleDateFormat("yyyy-MM-dd").parse(TEST_DATE).getTime());
MostPopularProtos.ResourceStat.Builder resourceStatBuilder =
MostPopularProtos.ResourceStat.newBuilder();
resourceStatBuilder.setCounter(COUNT3);
resourceStatBuilder.setResourceId(RES3_ID);
statsBuilder.addStat(resourceStatBuilder);
resourceStatBuilder = MostPopularProtos.ResourceStat.newBuilder();
resourceStatBuilder.setCounter(COUNT1);
resourceStatBuilder.setResourceId(RES1_ID);
statsBuilder.addStat(resourceStatBuilder);
BytesWritable expectedOutput = new BytesWritable(statsBuilder.build().toByteArray());
mapReduceDriver.addOutput(NullWritable.get(), expectedOutput);
mapReduceDriver.runTest();
}
public void setUp() {
mapReduceDriver =
new MapReduceDriver<
NullWritable, FaunusVertex, NullWritable, FaunusVertex, NullWritable, FaunusVertex>();
mapReduceDriver.setMapper(new IntervalFilterMap.Map());
mapReduceDriver.setReducer(
new Reducer<NullWritable, FaunusVertex, NullWritable, FaunusVertex>());
}
public void setUp() {
mapReduceDriver =
new MapReduceDriver<
NullWritable, FaunusVertex, LongWritable, Holder, NullWritable, FaunusVertex>();
mapReduceDriver.setMapper(new BackFilterMapReduce.Map());
mapReduceDriver.setCombiner(new BackFilterMapReduce.Combiner());
mapReduceDriver.setReducer(new BackFilterMapReduce.Reduce());
}
@Test
public void testVisitPageMR() throws Exception {
// 添加模拟数据
generateVisitData();
mrPageVisitor.getConfiguration().set("custom.period", "month");
// 开始计算
List<Pair<PageVisitKey, PageVisitOutputValue>> results = mrPageVisitor.run(true);
for (int i = 0; i < results.size(); i++) {
Pair<PageVisitKey, PageVisitOutputValue> pair = results.get(i);
System.out.println(
"Key:" + pair.getFirst().toString() + ", Value:" + pair.getSecond().toString());
}
}
@Before
public void setup() {
mrPageVisitor =
new MapReduceDriver<
LongWritable,
VisitLogEntry,
PageVisitKey,
PageVisitValue,
PageVisitKey,
PageVisitOutputValue>();
mrPageVisitor.setMapper(new PageVisitMapReduce.PageVisitMapper());
mrPageVisitor.setReducer(new PageVisitMapReduce.PageVisitReducer());
}
@Before
public void setUp() {
AudioQASettings.MAPPER_OUTPUT_DIR = "output/MigrateMp3ToWav/";
MigrationMapper mapper = new MigrationMapper();
MigrateMp3ToWav.MigrationReducer reducer = new MigrateMp3ToWav.MigrationReducer();
mapDriver = new MapDriver<LongWritable, Text, LongWritable, Text>();
mapDriver.setMapper(mapper);
reduceDriver = new ReduceDriver<LongWritable, Text, LongWritable, Text>();
reduceDriver.setReducer(reducer);
mapReduceDriver =
new MapReduceDriver<LongWritable, Text, LongWritable, Text, LongWritable, Text>();
mapReduceDriver.setMapper(mapper);
mapReduceDriver.setReducer(reducer);
}
/**
* Test characteristic set reduction
*
* @throws IOException
*/
@Test
public void characteristic_set_reducer_01() throws IOException {
MapReduceDriver<
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
NullWritable>
driver = this.getMapReduceDriver();
this.createSet(driver, 1, 1, "http://predicate");
driver.runTest(false);
}
/**
* Test class for {@linkplain WordCountJobRunner}
*
* @author Tom
*/
@RunWith(MockitoJUnitRunner.class)
public class WordCountJobRunnerTest {
private final MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable>
wordCountMapReduce = MapReduceDriver.newMapReduceDriver();
@Before
public void setUp() {
wordCountMapReduce.withMapper(new WordCountMapper());
wordCountMapReduce.withReducer(new WordCountReducer());
}
@Test
public void testMapReduce() {
wordCountMapReduce //
.withInput(new LongWritable(1), new Text("word1 word2 word2 word1")) //
.withOutput(buildExpectedOutput(new Text("word1"), new IntWritable(2))) //
.withOutput(buildExpectedOutput(new Text("word2"), new IntWritable(2))) //
.runTest();
}
@Test
public void testMapReduceWithCombiner() {
wordCountMapReduce //
.withInput(new LongWritable(1), new Text("word1 word2 word2 word1")) //
.withCombiner(new WordCountReducer()) //
.withOutput(buildExpectedOutput(new Text("word1"), new IntWritable(2))) //
.withOutput(buildExpectedOutput(new Text("word2"), new IntWritable(2))) //
.runTest();
}
}
public void testEdgesOnWeight() throws Exception {
Configuration config = IntervalFilterMap.createConfiguration(Edge.class, "weight", 0.3f, 0.45f);
mapReduceDriver.withConfiguration(config);
Map<Long, FaunusVertex> graph =
runWithGraph(
startPath(generateGraph(BaseTest.ExampleGraph.TINKERGRAPH, config), Edge.class),
mapReduceDriver);
assertEquals(graph.size(), 6);
long counter = 0;
for (FaunusVertex vertex : graph.values()) {
for (Edge edge : vertex.getEdges(Direction.BOTH)) {
if (((StandardFaunusEdge) edge).hasPaths()) {
counter = ((StandardFaunusEdge) edge).pathCount() + counter;
assertEquals(edge.getProperty("weight"), 0.4d);
}
}
}
assertEquals(counter, 4);
assertEquals(
DEFAULT_COMPAT.getCounter(mapReduceDriver, IntervalFilterMap.Counters.VERTICES_FILTERED),
0);
assertEquals(
DEFAULT_COMPAT.getCounter(mapReduceDriver, IntervalFilterMap.Counters.EDGES_FILTERED), 8);
identicalStructure(graph, ExampleGraph.TINKERGRAPH);
}
public void testVerticesWithPaths() throws Exception {
Configuration config = new Configuration();
config.setBoolean(Tokens.TITAN_HADOOP_PIPELINE_TRACK_PATHS, true);
mapReduceDriver.withConfiguration(config);
Map<Long, FaunusVertex> graph =
runWithGraph(generateGraph(BaseTest.ExampleGraph.TINKERGRAPH, config), mapReduceDriver);
assertEquals(graph.size(), 6);
for (FaunusVertex vertex : graph.values()) {
assertEquals(vertex.pathCount(), 1);
assertEquals(vertex.getPaths().get(0).size(), 1);
assertEquals(vertex.getPaths().get(0).get(0).getId(), vertex.getLongId());
for (Edge edge : vertex.getEdges(Direction.BOTH)) {
assertEquals(((StandardFaunusEdge) edge).pathCount(), 0);
}
}
assertEquals(
DEFAULT_COMPAT.getCounter(mapReduceDriver, VerticesMap.Counters.EDGES_PROCESSED), 12);
assertEquals(
DEFAULT_COMPAT.getCounter(mapReduceDriver, VerticesMap.Counters.VERTICES_PROCESSED), 6);
identicalStructure(graph, ExampleGraph.TINKERGRAPH);
}
public void testVerticesWithNoPaths() throws Exception {
Configuration config = new Configuration();
mapReduceDriver.withConfiguration(config);
Map<Long, FaunusVertex> graph =
runWithGraph(generateGraph(BaseTest.ExampleGraph.TINKERGRAPH, config), mapReduceDriver);
assertEquals(graph.size(), 6);
for (FaunusVertex vertex : graph.values()) {
assertEquals(vertex.pathCount(), 1);
for (Edge edge : vertex.getEdges(Direction.BOTH)) {
assertEquals(((StandardFaunusEdge) edge).pathCount(), 0);
}
try {
vertex.getPaths();
assertTrue(false);
} catch (IllegalStateException e) {
assertTrue(true);
}
}
assertEquals(
DEFAULT_COMPAT.getCounter(mapReduceDriver, VerticesMap.Counters.EDGES_PROCESSED), 12);
assertEquals(
DEFAULT_COMPAT.getCounter(mapReduceDriver, VerticesMap.Counters.VERTICES_PROCESSED), 6);
identicalStructure(graph, ExampleGraph.TINKERGRAPH);
}
@Test
public void testMapReduce() {
wordCountMapReduce //
.withInput(new LongWritable(1), new Text("word1 word2 word2 word1")) //
.withOutput(buildExpectedOutput(new Text("word1"), new IntWritable(2))) //
.withOutput(buildExpectedOutput(new Text("word2"), new IntWritable(2))) //
.runTest();
}
@Before
public void setUp() throws Exception {
MyMapper mapper = new MyMapper();
MyReducer reducer = new MyReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
@Before
public void setUp() {
IpMapper mapper = new IpMapper();
IpReducer reducer = new IpReducer();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
@Test
public void testMapperReducer() throws IOException {
mapReduceDriver.withInput(
new LongWritable(),
new Text(
"Wed Apr 24 23:04:58 2013 [SESSION 2954383360]: Scanning path '/Users/komamitsu/Library/Caches/com.avast.registration/Cache.db' started."));
mapReduceDriver.withInput(
new LongWritable(),
new Text(
"Wed Apr 25 23:04:58 2013 [SESSION 2954383360]: Scanning path '/Users/komamitsu/Library/Caches/com.avast.registration/Cache.db' started."));
mapReduceDriver.withInput(
new LongWritable(),
new Text(
"Wed Apr 24 23:04:58 2013 [SESSION 2954383360]: Scanning path '/Users/komamitsu/Library/Caches/com.avast.registration/Cache.db' started."));
mapReduceDriver.withOutput(new Text("Apr 24"), new LongWritable(2));
mapReduceDriver.withOutput(new Text("Apr 25"), new LongWritable(1));
mapReduceDriver.runTest();
}
@Before
public void setUp() throws Exception {
PymkMapper mapper = new PymkMapper();
mapDriver = MapDriver.newMapDriver(mapper);
PymkReducer reducer = new PymkReducer();
reducerDriver = ReduceDriver.newReduceDriver(reducer);
mapRedDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
@BeforeClass
public void beforeClass() {
Mapper m = new SortUsagesPart.SorterMap();
Reducer c = new SortUsagesPart.SorterCombine();
Reducer r = new SortUsagesPart.SorterReduce();
mapReduceDriver = new MapReduceDriver(m, r).withCombiner(c);
Configuration conf = mapReduceDriver.getConfiguration();
conf.set("NB_OF_RECORDS", RESULT_RECORDS);
conf.set("RESULT_DATE", TEST_DATE);
}
@Test
public void testMapReduce() throws Exception {
System.out.println(
"MapReduce,Begin------------------------------------------------------------------");
// 多个输入源
BufferedReader[] arr = {
new BufferedReader(new FileReader("D:\\hdfs\\warehouse\\appdetail.log"))
};
LongWritable longWritable = new LongWritable();
String line = null;
for (BufferedReader br : arr) {
while (null != (line = br.readLine())) {
mapReduceDriver.withInput(longWritable, new Text(line));
}
br.close();
}
List<Pair<OutFieldsBaseModel, OutFieldsBaseModel>> onlineList = mapReduceDriver.run();
Collections.sort(
onlineList,
new Comparator<Pair<OutFieldsBaseModel, OutFieldsBaseModel>>() {
@Override
public int compare(
Pair<OutFieldsBaseModel, OutFieldsBaseModel> o1,
Pair<OutFieldsBaseModel, OutFieldsBaseModel> o2) {
if (o1.getFirst().getSuffix().compareTo(o2.getFirst().getSuffix()) > 0) {
return -1;
}
return 0;
}
});
for (Pair<OutFieldsBaseModel, OutFieldsBaseModel> pair : onlineList) {
String lin =
pair.getFirst().toString()
+ "\t"
+ pair.getSecond().toString()
+ "\t"
+ pair.getFirst().getSuffix();
System.out.println(lin);
}
System.out.println(
"MapReduce,End------------------------------------------------------------------");
}
/**
* Test characteristic set reduction
*
* @throws IOException
*/
@Test
public void characteristic_set_reducer_02() throws IOException {
MapReduceDriver<
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
NullWritable>
driver = this.getMapReduceDriver();
this.createSet(driver, 2, 1, "http://predicate");
driver.runTest(false);
driver = getMapReduceDriver();
createSet(driver, 2, 1, "http://predicate");
List<Pair<CharacteristicSetWritable, NullWritable>> results = driver.run();
CharacteristicSetWritable cw = results.get(0).getFirst();
Assert.assertEquals(2, cw.getCount().get());
}
@Test
public void testIdentityMapper() throws IOException {
List<Pair<Text, Text>> results =
driver
.withInput(new Text("foo"), new Text("bar"))
.withInput(new Text("foo2"), new Text("bar2"))
.withOutput(new Text("foo"), new Text("bar"))
.withOutput(new Text("foo2"), new Text("bar2"))
.run();
MRUnitJUnitAsserts.assertOutputs(driver, results);
}
public static void verifyMapReduce(
SmartMapper mapper, SmartReducer reducer, Object key, Object input) throws Exception {
MapDriver mapDriver = new MapDriver();
mapDriver.setMapper(mapper);
MapReduceDriver mapReduceDriver = new MapReduceDriver();
mapReduceDriver.setMapper(mapper);
Object writableKey = createWritable(key, mapper.getKeyInType());
Object writableValue = createWritable(input, mapper.getValueInType());
mapDriver.withInput(writableKey, writableValue);
List results = mapDriver.run();
Collections.sort(results, PairComparer.INSTANCE);
mapReduceDriver =
new MapReduceDriver<LongWritable, Text, Text, LongWritable, Text, LongWritable>();
writableKey = createWritable(key, mapper.getKeyInType());
writableValue = createWritable(input, mapper.getValueInType());
mapReduceDriver.withInput(writableKey, writableValue);
mapReduceDriver.setMapper(mapper);
mapReduceDriver.setReducer(reducer);
List finalResults = mapReduceDriver.run();
String text =
String.format(
"[%s]\r\n\r\n -> maps via %s to -> \r\n\r\n%s\r\n\r\n -> reduces via %s to -> \r\n\r\n%s",
input,
mapper.getClass().getSimpleName(),
ArrayUtils.toString(results, Echo.INSTANCE),
reducer.getClass().getSimpleName(),
ArrayUtils.toString(finalResults, Echo.INSTANCE));
Approvals.verify(text);
}
public void testVerticesBiasedStart() throws Exception {
Configuration config = BackFilterMapReduce.createConfiguration(Vertex.class, 0);
mapReduceDriver.withConfiguration(config);
Map<Long, FaunusVertex> graph = generateGraph(BaseTest.ExampleGraph.TINKERGRAPH, config);
graph
.get(1l)
.addPath(
(List)
Arrays.asList(new FaunusVertex.MicroVertex(1l), new FaunusVertex.MicroVertex(1l)),
false);
graph
.get(2l)
.addPath(
(List)
Arrays.asList(new FaunusVertex.MicroVertex(1l), new FaunusVertex.MicroVertex(2l)),
false);
graph
.get(3l)
.addPath(
(List)
Arrays.asList(new FaunusVertex.MicroVertex(2l), new FaunusVertex.MicroVertex(3l)),
false);
graph
.get(4l)
.addPath(
(List)
Arrays.asList(new FaunusVertex.MicroVertex(3l), new FaunusVertex.MicroVertex(4l)),
false);
graph
.get(5l)
.addPath(
(List)
Arrays.asList(new FaunusVertex.MicroVertex(3l), new FaunusVertex.MicroVertex(5l)),
false);
graph = runWithGraph(graph, mapReduceDriver);
assertEquals(graph.size(), 6);
assertEquals(graph.get(1l).pathCount(), 2);
assertEquals(graph.get(2l).pathCount(), 1);
assertEquals(graph.get(3l).pathCount(), 2);
assertEquals(graph.get(4l).pathCount(), 0);
assertEquals(graph.get(5l).pathCount(), 0);
assertEquals(graph.get(6l).pathCount(), 0);
identicalStructure(graph, ExampleGraph.TINKERGRAPH);
}
public void testBackingUpToVerticesFromEdges() throws Exception {
Configuration config = BackFilterMapReduce.createConfiguration(Edge.class, 0);
mapReduceDriver.withConfiguration(config);
Map<Long, FaunusVertex> graph = generateGraph(BaseTest.ExampleGraph.TINKERGRAPH, config);
((StandardFaunusEdge) graph.get(1l).getEdges(Direction.OUT, "created").iterator().next())
.addPath(
(List)
Arrays.asList(
new FaunusVertex.MicroVertex(1l), new StandardFaunusEdge.MicroEdge(2l)),
false);
((StandardFaunusEdge) graph.get(6l).getEdges(Direction.OUT, "created").iterator().next())
.addPath(
(List)
Arrays.asList(
new FaunusVertex.MicroVertex(6l), new StandardFaunusEdge.MicroEdge(2l)),
false);
((StandardFaunusEdge) graph.get(6l).getEdges(Direction.OUT, "created").iterator().next())
.addPath(
(List)
Arrays.asList(
new FaunusVertex.MicroVertex(2l), new StandardFaunusEdge.MicroEdge(2l)),
false);
((StandardFaunusEdge) graph.get(6l).getEdges(Direction.OUT, "created").iterator().next())
.addPath(
(List)
Arrays.asList(
new FaunusVertex.MicroVertex(2l), new StandardFaunusEdge.MicroEdge(2l)),
false);
graph = runWithGraph(graph, mapReduceDriver);
assertEquals(graph.size(), 6);
assertEquals(graph.get(1l).pathCount(), 1);
assertEquals(graph.get(2l).pathCount(), 2);
assertEquals(graph.get(3l).pathCount(), 0);
assertEquals(graph.get(4l).pathCount(), 0);
assertEquals(graph.get(5l).pathCount(), 0);
assertEquals(graph.get(6l).pathCount(), 1);
for (Vertex vertex : graph.values()) {
for (Edge e : vertex.getEdges(Direction.BOTH)) {
assertEquals(((StandardFaunusEdge) e).pathCount(), 0);
}
}
identicalStructure(graph, ExampleGraph.TINKERGRAPH);
}
public void testBackingUpToEdgesException() throws Exception {
Configuration config = BackFilterMapReduce.createConfiguration(Vertex.class, 1);
mapReduceDriver.withConfiguration(config);
Map<Long, FaunusVertex> graph = generateGraph(BaseTest.ExampleGraph.TINKERGRAPH, config);
graph
.get(1l)
.addPath(
(List)
Arrays.asList(
new FaunusVertex.MicroVertex(1l), new StandardFaunusEdge.MicroEdge(1l)),
false);
graph
.get(2l)
.addPath(
(List)
Arrays.asList(
new FaunusVertex.MicroVertex(1l), new StandardFaunusEdge.MicroEdge(2l)),
false);
graph
.get(3l)
.addPath(
(List)
Arrays.asList(
new FaunusVertex.MicroVertex(2l), new StandardFaunusEdge.MicroEdge(3l)),
false);
graph
.get(4l)
.addPath(
(List)
Arrays.asList(
new FaunusVertex.MicroVertex(3l), new StandardFaunusEdge.MicroEdge(4l)),
false);
graph
.get(5l)
.addPath(
(List)
Arrays.asList(
new FaunusVertex.MicroVertex(3l), new StandardFaunusEdge.MicroEdge(5l)),
false);
try {
graph = runWithGraph(graph, mapReduceDriver);
assertFalse(true);
} catch (Exception e) {
assertTrue(true);
}
}
@Test
public void testPalyerOnlineHour() throws Exception {
Set<String> set = new HashSet<String>();
int count = 0;
String line = null;
LongWritable longWritable = new LongWritable();
BufferedReader reader =
new BufferedReader(new FileReader("C:\\Users\\Administrator\\Desktop\\error_pay"));
while (null != (line = reader.readLine())) {
/*String[] array = line.split(MRConstants.SEPERATOR_IN);
PaymentDayLog paymentDayLog = new PaymentDayLog(array);
set.add(paymentDayLog.getExtend().getResolution());
if("1".equals(paymentDayLog.getExtend().getResolution())){
count++;
}*/
mapReduceDriver.withInput(longWritable, new Text(line));
}
reader.close();
List<Pair<OutFieldsBaseModel, NullWritable>> list = mapReduceDriver.run();
for (Pair<OutFieldsBaseModel, NullWritable> pair : list) {
System.out.println(pair.getFirst().getSuffix() + "---" + pair.getFirst().toString());
}
}
/**
* Creates a set consisting of the given predicates
*
* @param predicates Predicates
* @return Set
*/
protected CharacteristicSetWritable createSet(
MapReduceDriver<
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
NullWritable>
driver,
int inputOccurrences,
int outputOccurrences,
String... predicates) {
CharacteristicSetWritable set = new CharacteristicSetWritable();
for (String predicateUri : predicates) {
set.add(new CharacteristicWritable(NodeFactory.createURI(predicateUri)));
}
for (int i = 1; i <= inputOccurrences; i++) {
driver.addInput(set, set);
}
for (int i = 1; i <= outputOccurrences; i++) {
driver.addOutput(set, NullWritable.get());
}
return set;
}
public void testVerticesFullStart() throws Exception {
Configuration config = BackFilterMapReduce.createConfiguration(Vertex.class, 0);
mapReduceDriver.withConfiguration(config);
Map<Long, FaunusVertex> graph =
runWithGraph(
startPath(generateGraph(BaseTest.ExampleGraph.TINKERGRAPH, config), Vertex.class),
mapReduceDriver);
assertEquals(graph.size(), 6);
assertEquals(graph.get(1l).pathCount(), 1);
assertEquals(graph.get(2l).pathCount(), 1);
assertEquals(graph.get(3l).pathCount(), 1);
assertEquals(graph.get(4l).pathCount(), 1);
assertEquals(graph.get(5l).pathCount(), 1);
assertEquals(graph.get(6l).pathCount(), 1);
identicalStructure(graph, ExampleGraph.TINKERGRAPH);
}
@Test
public void testMapReduce() throws IOException {
mapRedDriver.withInput(new LongWritable(1), new Text("Marco-Fabio,Luca,Gordon"));
mapRedDriver.withInput(new LongWritable(2), new Text("Fabio-Nathalia,Marco,Luca,Gordon"));
mapRedDriver.withInput(new LongWritable(3), new Text("Luca-Nathalia,Marco,Fabio"));
mapRedDriver.withInput(new LongWritable(4), new Text("Nathalia-Fabio,Luca"));
mapRedDriver.withInput(new LongWritable(5), new Text("Gordon-Marco,Fabio"));
mapRedDriver.withAllOutput(
Arrays.asList(
new Pair<>(new Text("Gordon"), new Text("Luca(2:[Marco, Fabio]),Nathalia(1:[Fabio])")),
new Pair<>(new Text("Luca"), new Text("Gordon(2:[Marco, Fabio])")),
new Pair<>(new Text("Marco"), new Text("Nathalia(2:[Fabio, Luca])")),
new Pair<>(
new Text("Nathalia"), new Text("Marco(2:[Fabio, Luca]),Gordon(1:[Fabio])"))));
mapRedDriver.runTest();
}
private void generateVisitData() throws Exception {
long counter = 0;
Integer[] appIds = new Integer[] {1, 2, 3, 4, 5, 6};
Long[] userIds =
new Long[] {
100011L, 100010L, 10090L, 10008L, 10007L, 10006L, 10005L, 10004L, 10003L, 10002L
};
String[] visitTime =
new String[] {
"2014-07-05 15:15:15",
"2014-07-10 15:15:15",
"2014-07-15 15:15:15",
"2014-07-20 15:15:15",
"2014-07-25 15:15:15",
"2014-07-30 15:15:15",
"2014-08-04 15:15:15",
"2014-08-09 15:15:15",
"2014-08-14 15:15:15"
};
Random rnd = new Random();
// 添加模拟的数据
for (int i = 0; i < 100; i++) {
counter++;
int order = rnd.nextInt(5);
int uOrder = rnd.nextInt(9);
String orgText =
String.format(
"auc\t%d\t%d\t1001\t3264456\trefer.page\tvisit.page\tvisit.param\t%s",
userIds[uOrder], appIds[order].intValue(), visitTime[uOrder]);
VisitLogEntry entry = new VisitLogEntry();
entry.parse(orgText);
mrPageVisitor.addInput(new LongWritable(counter), entry);
}
}