public List<Pair<K2, V2>> run() throws IOException {
// inputs starts with the user-provided inputs.
List inputs = this.inputList;
if (mapReducePipeline.size() == 0) {
LOG.warn("No Mapper or Reducer instances in pipeline; this is a trivial test.");
}
if (inputs.size() == 0) {
LOG.warn("No inputs configured to send to MapReduce pipeline; this is a trivial test.");
}
for (Pair<Mapper, Reducer> job : mapReducePipeline) {
// Create a MapReduceDriver to run this phase of the pipeline.
MapReduceDriver mrDriver = new MapReduceDriver(job.getFirst(), job.getSecond());
mrDriver.setCounters(getCounters());
// Add the inputs from the user, or from the previous stage of the pipeline.
for (Object input : inputs) {
mrDriver.addInput((Pair) input);
}
// Run the MapReduce "job". The output of this job becomes
// the input to the next job.
inputs = mrDriver.run();
}
// The last list of values stored in "inputs" is actually the outputs.
// Unfortunately, due to the variable-length list of MR passes the user
// can test, this is not type-safe.
return (List<Pair<K2, V2>>) inputs;
}
private List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>>
getReducerDataFromMapperInput(
final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults) {
final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> reducerInputSet =
new ArrayList<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>>();
for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
getListFor(pair.getFirst(), reducerInputSet).add(pair.getSecond());
}
return reducerInputSet;
}
@Test
public void testVisitPageMR() throws Exception {
// 添加模拟数据
generateVisitData();
mrPageVisitor.getConfiguration().set("custom.period", "month");
// 开始计算
List<Pair<PageVisitKey, PageVisitOutputValue>> results = mrPageVisitor.run(true);
for (int i = 0; i < results.size(); i++) {
Pair<PageVisitKey, PageVisitOutputValue> pair = results.get(i);
System.out.println(
"Key:" + pair.getFirst().toString() + ", Value:" + pair.getSecond().toString());
}
}
public List<Pair<Text, List<Text>>> getReducerPairsFromFile(String file) throws IOException {
List<Pair<Text, Text>> unsplitPairs = this.getPairsFromFile(file);
List<Pair<Text, List<Text>>> splitPairs = new LinkedList<Pair<Text, List<Text>>>();
for (Pair<Text, Text> unsplitPair : unsplitPairs) {
List<Text> splitValues = new LinkedList<Text>();
String[] splits = unsplitPair.getSecond().toString().split("(\r\n|\n)");
for (String split : splits) {
splitValues.add(new Text(split));
}
splitPairs.add(new Pair<Text, List<Text>>(unsplitPair.getFirst(), splitValues));
}
return splitPairs;
}
private PartitionData getPartitionDataFor(
final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults,
final String id,
final boolean primary) {
for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
if (((FeatureWritable) pair.getSecond().getObjectWritable().get())
.getFeature()
.getID()
.equals(id)
&& (pair.getFirst().getPartitionData().isPrimary() == primary)) {
return pair.getFirst().getPartitionData();
}
}
return null;
}
@Test
public void testMapReduce() throws Exception {
System.out.println(
"MapReduce,Begin------------------------------------------------------------------");
// 多个输入源
BufferedReader[] arr = {
new BufferedReader(new FileReader("D:\\hdfs\\warehouse\\appdetail.log"))
};
LongWritable longWritable = new LongWritable();
String line = null;
for (BufferedReader br : arr) {
while (null != (line = br.readLine())) {
mapReduceDriver.withInput(longWritable, new Text(line));
}
br.close();
}
List<Pair<OutFieldsBaseModel, OutFieldsBaseModel>> onlineList = mapReduceDriver.run();
Collections.sort(
onlineList,
new Comparator<Pair<OutFieldsBaseModel, OutFieldsBaseModel>>() {
@Override
public int compare(
Pair<OutFieldsBaseModel, OutFieldsBaseModel> o1,
Pair<OutFieldsBaseModel, OutFieldsBaseModel> o2) {
if (o1.getFirst().getSuffix().compareTo(o2.getFirst().getSuffix()) > 0) {
return -1;
}
return 0;
}
});
for (Pair<OutFieldsBaseModel, OutFieldsBaseModel> pair : onlineList) {
String lin =
pair.getFirst().toString()
+ "\t"
+ pair.getSecond().toString()
+ "\t"
+ pair.getFirst().getSuffix();
System.out.println(lin);
}
System.out.println(
"MapReduce,End------------------------------------------------------------------");
}