コード例 #1
0
  @Test
  public void testAllReduceDriverImmutableEmpty() {
    try {
      TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
          new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>();

      List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
      TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
      MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
      context.setDriverStrategy(DriverStrategy.ALL_REDUCE);

      context.setInput1(input, typeInfo.createSerializer());
      context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());

      AllReduceDriver<Tuple2<String, Integer>> driver =
          new AllReduceDriver<Tuple2<String, Integer>>();
      driver.setup(context);
      driver.prepare();
      driver.run();
    } catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      Assert.fail(e.getMessage());
    }
  }
コード例 #2
0
  @Test
  public void testImmutableEmpty() {
    try {
      TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
          new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>(
              1024 * 1024);
      context.getTaskConfig().setRelativeMemoryDriver(0.5);

      List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
      Collections.shuffle(data);

      TupleTypeInfo<Tuple2<String, Integer>> typeInfo =
          (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
      MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();

      context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
      TypeComparator<Tuple2<String, Integer>> comparator =
          typeInfo.createComparator(new int[] {0}, new boolean[] {true}, 0, new ExecutionConfig());

      GatheringCollector<Tuple2<String, Integer>> result =
          new GatheringCollector<Tuple2<String, Integer>>(
              typeInfo.createSerializer(new ExecutionConfig()));

      context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
      context.setComparator1(comparator);
      context.setCollector(result);

      ReduceCombineDriver<Tuple2<String, Integer>> driver =
          new ReduceCombineDriver<Tuple2<String, Integer>>();
      driver.setup(context);
      driver.prepare();
      driver.run();

      Assert.assertEquals(0, result.getList().size());
    } catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      Assert.fail(e.getMessage());
    }
  }
コード例 #3
0
  @Test
  public void testAllReduceDriverImmutable() {
    try {
      {
        TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
            new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>();

        List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
        TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
        MutableObjectIterator<Tuple2<String, Integer>> input =
            new RegularToMutableObjectIterator<Tuple2<String, Integer>>(
                data.iterator(), typeInfo.createSerializer());

        GatheringCollector<Tuple2<String, Integer>> result =
            new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer());

        context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
        context.setInput1(input, typeInfo.createSerializer());
        context.setCollector(result);
        context.setUdf(new ConcatSumFirstReducer());

        AllReduceDriver<Tuple2<String, Integer>> driver =
            new AllReduceDriver<Tuple2<String, Integer>>();
        driver.setup(context);
        driver.prepare();
        driver.run();

        Tuple2<String, Integer> res = result.getList().get(0);

        char[] foundString = res.f0.toCharArray();
        Arrays.sort(foundString);

        char[] expectedString = "abcddeeeffff".toCharArray();
        Arrays.sort(expectedString);

        Assert.assertArrayEquals(expectedString, foundString);
        Assert.assertEquals(78, res.f1.intValue());
      }

      {
        TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
            new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>();

        List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
        TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
        MutableObjectIterator<Tuple2<String, Integer>> input =
            new RegularToMutableObjectIterator<Tuple2<String, Integer>>(
                data.iterator(), typeInfo.createSerializer());

        GatheringCollector<Tuple2<String, Integer>> result =
            new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer());

        context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
        context.setInput1(input, typeInfo.createSerializer());
        context.setCollector(result);
        context.setUdf(new ConcatSumSecondReducer());

        AllReduceDriver<Tuple2<String, Integer>> driver =
            new AllReduceDriver<Tuple2<String, Integer>>();
        driver.setup(context);
        driver.prepare();
        driver.run();

        Tuple2<String, Integer> res = result.getList().get(0);

        char[] foundString = res.f0.toCharArray();
        Arrays.sort(foundString);

        char[] expectedString = "abcddeeeffff".toCharArray();
        Arrays.sort(expectedString);

        Assert.assertArrayEquals(expectedString, foundString);
        Assert.assertEquals(78, res.f1.intValue());
      }
    } catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      Assert.fail(e.getMessage());
    }
  }