コード例 #1
0
  @Parameterized.Parameters
  public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {

    LinkedList<Object[]> configs = new LinkedList<Object[]>();

    ExecutionConfig withReuse = new ExecutionConfig();
    withReuse.enableObjectReuse();

    ExecutionConfig withoutReuse = new ExecutionConfig();
    withoutReuse.disableObjectReuse();

    Object[] a = {withoutReuse};
    configs.add(a);
    Object[] b = {withReuse};
    configs.add(b);

    return configs;
  }
コード例 #2
0
  @Test
  public void testJoinPlain() {
    final FlatJoinFunction<String, String, Integer> joiner =
        new FlatJoinFunction<String, String, Integer>() {

          @Override
          public void join(String first, String second, Collector<Integer> out) throws Exception {
            out.collect(first.length());
            out.collect(second.length());
          }
        };

    @SuppressWarnings({"rawtypes", "unchecked"})
    JoinOperatorBase<String, String, Integer, FlatJoinFunction<String, String, Integer>> base =
        new JoinOperatorBase(
            joiner,
            new BinaryOperatorInformation(
                BasicTypeInfo.STRING_TYPE_INFO,
                BasicTypeInfo.STRING_TYPE_INFO,
                BasicTypeInfo.INT_TYPE_INFO),
            new int[0],
            new int[0],
            "TestJoiner");

    List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
    List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
    List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));

    try {
      ExecutionConfig executionConfig = new ExecutionConfig();
      executionConfig.disableObjectReuse();
      List<Integer> resultSafe =
          base.executeOnCollections(inputData1, inputData2, null, executionConfig);
      executionConfig.enableObjectReuse();
      List<Integer> resultRegular =
          base.executeOnCollections(inputData1, inputData2, null, executionConfig);

      assertEquals(expected, resultSafe);
      assertEquals(expected, resultRegular);
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }
コード例 #3
0
  @Test
  public void testJoinRich() {
    final AtomicBoolean opened = new AtomicBoolean(false);
    final AtomicBoolean closed = new AtomicBoolean(false);
    final String taskName = "Test rich join function";

    final RichFlatJoinFunction<String, String, Integer> joiner =
        new RichFlatJoinFunction<String, String, Integer>() {
          @Override
          public void open(Configuration parameters) throws Exception {
            opened.compareAndSet(false, true);
            assertEquals(0, getRuntimeContext().getIndexOfThisSubtask());
            assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
          }

          @Override
          public void close() throws Exception {
            closed.compareAndSet(false, true);
          }

          @Override
          public void join(String first, String second, Collector<Integer> out) throws Exception {
            out.collect(first.length());
            out.collect(second.length());
          }
        };

    JoinOperatorBase<String, String, Integer, RichFlatJoinFunction<String, String, Integer>> base =
        new JoinOperatorBase<
            String, String, Integer, RichFlatJoinFunction<String, String, Integer>>(
            joiner,
            new BinaryOperatorInformation<String, String, Integer>(
                BasicTypeInfo.STRING_TYPE_INFO,
                BasicTypeInfo.STRING_TYPE_INFO,
                BasicTypeInfo.INT_TYPE_INFO),
            new int[0],
            new int[0],
            taskName);

    final List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
    final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
    final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));

    try {
      final HashMap<String, Accumulator<?, ?>> accumulatorMap =
          new HashMap<String, Accumulator<?, ?>>();
      final HashMap<String, Future<Path>> cpTasks = new HashMap<>();

      ExecutionConfig executionConfig = new ExecutionConfig();
      executionConfig.disableObjectReuse();
      List<Integer> resultSafe =
          base.executeOnCollections(
              inputData1,
              inputData2,
              new RuntimeUDFContext(taskName, 1, 0, null, executionConfig, cpTasks, accumulatorMap),
              executionConfig);
      executionConfig.enableObjectReuse();
      List<Integer> resultRegular =
          base.executeOnCollections(
              inputData1,
              inputData2,
              new RuntimeUDFContext(taskName, 1, 0, null, executionConfig, cpTasks, accumulatorMap),
              executionConfig);

      assertEquals(expected, resultSafe);
      assertEquals(expected, resultRegular);
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }

    assertTrue(opened.get());
    assertTrue(closed.get());
  }