public List<Pair<K2, V2>> run() throws IOException {
    // inputs starts with the user-provided inputs.
    List inputs = this.inputList;

    if (mapReducePipeline.size() == 0) {
      LOG.warn("No Mapper or Reducer instances in pipeline; this is a trivial test.");
    }

    if (inputs.size() == 0) {
      LOG.warn("No inputs configured to send to MapReduce pipeline; this is a trivial test.");
    }

    for (Pair<Mapper, Reducer> job : mapReducePipeline) {
      // Create a MapReduceDriver to run this phase of the pipeline.
      MapReduceDriver mrDriver = new MapReduceDriver(job.getFirst(), job.getSecond());

      mrDriver.setCounters(getCounters());

      // Add the inputs from the user, or from the previous stage of the pipeline.
      for (Object input : inputs) {
        mrDriver.addInput((Pair) input);
      }

      // Run the MapReduce "job". The output of this job becomes
      // the input to the next job.
      inputs = mrDriver.run();
    }

    // The last list of values stored in "inputs" is actually the outputs.
    // Unfortunately, due to the variable-length list of MR passes the user
    // can test, this is not type-safe.
    return (List<Pair<K2, V2>>) inputs;
  }
 private List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>>
     getReducerDataFromMapperInput(
         final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults) {
   final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> reducerInputSet =
       new ArrayList<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>>();
   for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
     getListFor(pair.getFirst(), reducerInputSet).add(pair.getSecond());
   }
   return reducerInputSet;
 }
示例#3
0
 @Test
 public void testVisitPageMR() throws Exception {
   // 添加模拟数据
   generateVisitData();
   mrPageVisitor.getConfiguration().set("custom.period", "month");
   // 开始计算
   List<Pair<PageVisitKey, PageVisitOutputValue>> results = mrPageVisitor.run(true);
   for (int i = 0; i < results.size(); i++) {
     Pair<PageVisitKey, PageVisitOutputValue> pair = results.get(i);
     System.out.println(
         "Key:" + pair.getFirst().toString() + ", Value:" + pair.getSecond().toString());
   }
 }
 public List<Pair<Text, List<Text>>> getReducerPairsFromFile(String file) throws IOException {
   List<Pair<Text, Text>> unsplitPairs = this.getPairsFromFile(file);
   List<Pair<Text, List<Text>>> splitPairs = new LinkedList<Pair<Text, List<Text>>>();
   for (Pair<Text, Text> unsplitPair : unsplitPairs) {
     List<Text> splitValues = new LinkedList<Text>();
     String[] splits = unsplitPair.getSecond().toString().split("(\r\n|\n)");
     for (String split : splits) {
       splitValues.add(new Text(split));
     }
     splitPairs.add(new Pair<Text, List<Text>>(unsplitPair.getFirst(), splitValues));
   }
   return splitPairs;
 }
 private PartitionData getPartitionDataFor(
     final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults,
     final String id,
     final boolean primary) {
   for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
     if (((FeatureWritable) pair.getSecond().getObjectWritable().get())
             .getFeature()
             .getID()
             .equals(id)
         && (pair.getFirst().getPartitionData().isPrimary() == primary)) {
       return pair.getFirst().getPartitionData();
     }
   }
   return null;
 }
 @Test
 public void testMapReduce() throws Exception {
   System.out.println(
       "MapReduce,Begin------------------------------------------------------------------");
   // 多个输入源
   BufferedReader[] arr = {
     new BufferedReader(new FileReader("D:\\hdfs\\warehouse\\appdetail.log"))
   };
   LongWritable longWritable = new LongWritable();
   String line = null;
   for (BufferedReader br : arr) {
     while (null != (line = br.readLine())) {
       mapReduceDriver.withInput(longWritable, new Text(line));
     }
     br.close();
   }
   List<Pair<OutFieldsBaseModel, OutFieldsBaseModel>> onlineList = mapReduceDriver.run();
   Collections.sort(
       onlineList,
       new Comparator<Pair<OutFieldsBaseModel, OutFieldsBaseModel>>() {
         @Override
         public int compare(
             Pair<OutFieldsBaseModel, OutFieldsBaseModel> o1,
             Pair<OutFieldsBaseModel, OutFieldsBaseModel> o2) {
           if (o1.getFirst().getSuffix().compareTo(o2.getFirst().getSuffix()) > 0) {
             return -1;
           }
           return 0;
         }
       });
   for (Pair<OutFieldsBaseModel, OutFieldsBaseModel> pair : onlineList) {
     String lin =
         pair.getFirst().toString()
             + "\t"
             + pair.getSecond().toString()
             + "\t"
             + pair.getFirst().getSuffix();
     System.out.println(lin);
   }
   System.out.println(
       "MapReduce,End------------------------------------------------------------------");
 }