コード例 #1
0
  public List<Pair<K2, V2>> run() throws IOException {
    // inputs starts with the user-provided inputs.
    List inputs = this.inputList;

    if (mapReducePipeline.size() == 0) {
      LOG.warn("No Mapper or Reducer instances in pipeline; this is a trivial test.");
    }

    if (inputs.size() == 0) {
      LOG.warn("No inputs configured to send to MapReduce pipeline; this is a trivial test.");
    }

    for (Pair<Mapper, Reducer> job : mapReducePipeline) {
      // Create a MapReduceDriver to run this phase of the pipeline.
      MapReduceDriver mrDriver = new MapReduceDriver(job.getFirst(), job.getSecond());

      mrDriver.setCounters(getCounters());

      // Add the inputs from the user, or from the previous stage of the pipeline.
      for (Object input : inputs) {
        mrDriver.addInput((Pair) input);
      }

      // Run the MapReduce "job". The output of this job becomes
      // the input to the next job.
      inputs = mrDriver.run();
    }

    // The last list of values stored in "inputs" is actually the outputs.
    // Unfortunately, due to the variable-length list of MR passes the user
    // can test, this is not type-safe.
    return (List<Pair<K2, V2>>) inputs;
  }
コード例 #2
0
  /**
   * Test characteristic set reduction
   *
   * @throws IOException
   */
  @Test
  public void characteristic_set_reducer_06() throws IOException {
    MapReduceDriver<
            CharacteristicSetWritable,
            CharacteristicSetWritable,
            CharacteristicSetWritable,
            CharacteristicSetWritable,
            CharacteristicSetWritable,
            NullWritable>
        driver = this.getMapReduceDriver();

    this.createSet(driver, 2, 1, "http://predicate", "http://other");
    this.createSet(driver, 1, 1, "http://other");
    driver.runTest(false);

    driver = getMapReduceDriver();
    createSet(driver, 2, 1, "http://predicate", "http://other");
    createSet(driver, 1, 1, "http://other");
    List<Pair<CharacteristicSetWritable, NullWritable>> results = driver.run();
    for (Pair<CharacteristicSetWritable, NullWritable> pair : results) {
      CharacteristicSetWritable cw = pair.getFirst();
      boolean expectTwo = cw.hasCharacteristic("http://predicate");
      Assert.assertEquals(expectTwo ? 2 : 1, cw.getCount().get());
    }
  }
コード例 #3
0
 private List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>>
     getReducerDataFromMapperInput(
         final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults) {
   final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> reducerInputSet =
       new ArrayList<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>>();
   for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
     getListFor(pair.getFirst(), reducerInputSet).add(pair.getSecond());
   }
   return reducerInputSet;
 }
コード例 #4
0
 @Test
 public void testVisitPageMR() throws Exception {
   // 添加模拟数据
   generateVisitData();
   mrPageVisitor.getConfiguration().set("custom.period", "month");
   // 开始计算
   List<Pair<PageVisitKey, PageVisitOutputValue>> results = mrPageVisitor.run(true);
   for (int i = 0; i < results.size(); i++) {
     Pair<PageVisitKey, PageVisitOutputValue> pair = results.get(i);
     System.out.println(
         "Key:" + pair.getFirst().toString() + ", Value:" + pair.getSecond().toString());
   }
 }
コード例 #5
0
 public List<Pair<Text, List<Text>>> getReducerPairsFromFile(String file) throws IOException {
   List<Pair<Text, Text>> unsplitPairs = this.getPairsFromFile(file);
   List<Pair<Text, List<Text>>> splitPairs = new LinkedList<Pair<Text, List<Text>>>();
   for (Pair<Text, Text> unsplitPair : unsplitPairs) {
     List<Text> splitValues = new LinkedList<Text>();
     String[] splits = unsplitPair.getSecond().toString().split("(\r\n|\n)");
     for (String split : splits) {
       splitValues.add(new Text(split));
     }
     splitPairs.add(new Pair<Text, List<Text>>(unsplitPair.getFirst(), splitValues));
   }
   return splitPairs;
 }
コード例 #6
0
 private PartitionData getPartitionDataFor(
     final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults,
     final String id,
     final boolean primary) {
   for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
     if (((FeatureWritable) pair.getSecond().getObjectWritable().get())
             .getFeature()
             .getID()
             .equals(id)
         && (pair.getFirst().getPartitionData().isPrimary() == primary)) {
       return pair.getFirst().getPartitionData();
     }
   }
   return null;
 }
コード例 #7
0
 @Test
 public void testMapReduce() throws Exception {
   System.out.println(
       "MapReduce,Begin------------------------------------------------------------------");
   // 多个输入源
   BufferedReader[] arr = {
     new BufferedReader(new FileReader("D:\\hdfs\\warehouse\\appdetail.log"))
   };
   LongWritable longWritable = new LongWritable();
   String line = null;
   for (BufferedReader br : arr) {
     while (null != (line = br.readLine())) {
       mapReduceDriver.withInput(longWritable, new Text(line));
     }
     br.close();
   }
   List<Pair<OutFieldsBaseModel, OutFieldsBaseModel>> onlineList = mapReduceDriver.run();
   Collections.sort(
       onlineList,
       new Comparator<Pair<OutFieldsBaseModel, OutFieldsBaseModel>>() {
         @Override
         public int compare(
             Pair<OutFieldsBaseModel, OutFieldsBaseModel> o1,
             Pair<OutFieldsBaseModel, OutFieldsBaseModel> o2) {
           if (o1.getFirst().getSuffix().compareTo(o2.getFirst().getSuffix()) > 0) {
             return -1;
           }
           return 0;
         }
       });
   for (Pair<OutFieldsBaseModel, OutFieldsBaseModel> pair : onlineList) {
     String lin =
         pair.getFirst().toString()
             + "\t"
             + pair.getSecond().toString()
             + "\t"
             + pair.getFirst().getSuffix();
     System.out.println(lin);
   }
   System.out.println(
       "MapReduce,End------------------------------------------------------------------");
 }
コード例 #8
0
  @Test
  public void testPalyerOnlineHour() throws Exception {
    Set<String> set = new HashSet<String>();
    int count = 0;
    String line = null;
    LongWritable longWritable = new LongWritable();
    BufferedReader reader =
        new BufferedReader(new FileReader("C:\\Users\\Administrator\\Desktop\\error_pay"));
    while (null != (line = reader.readLine())) {
      /*String[] array = line.split(MRConstants.SEPERATOR_IN);
      PaymentDayLog paymentDayLog = new PaymentDayLog(array);
      set.add(paymentDayLog.getExtend().getResolution());
      if("1".equals(paymentDayLog.getExtend().getResolution())){
      	count++;
      }*/
      mapReduceDriver.withInput(longWritable, new Text(line));
    }
    reader.close();

    List<Pair<OutFieldsBaseModel, NullWritable>> list = mapReduceDriver.run();
    for (Pair<OutFieldsBaseModel, NullWritable> pair : list) {
      System.out.println(pair.getFirst().getSuffix() + "---" + pair.getFirst().toString());
    }
  }
コード例 #9
0
 public int compare(Pair o1, Pair o2) {
   return ((Comparable) o1.getFirst()).compareTo(o2.getFirst());
 }