コード例 #1
0
  /**
   * This test verifies that open() and close() are correctly called. This test also verifies that
   * timestamps of emitted elements are correct. {@link CoStreamMap} assigns the input timestamp to
   * emitted elements.
   */
  @Test
  @SuppressWarnings("unchecked")
  public void testOpenCloseAndTimestamps() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask =
        new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness =
        new TwoInputStreamTaskTestHarness<String, Integer, String>(
            coMapTask,
            BasicTypeInfo.STRING_TYPE_INFO,
            BasicTypeInfo.INT_TYPE_INFO,
            BasicTypeInfo.STRING_TYPE_INFO);

    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator =
        new CoStreamMap<String, Integer, String>(new TestOpenCloseMapFunction());
    streamConfig.setStreamOperator(coMapOperator);

    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

    testHarness.invoke();

    testHarness.processElement(new StreamRecord<String>("Hello", initialTime + 1), 0, 0);
    expectedOutput.add(new StreamRecord<String>("Hello", initialTime + 1));

    // wait until the input is processed to ensure ordering of the output
    testHarness.waitForInputProcessing();

    testHarness.processElement(new StreamRecord<Integer>(1337, initialTime + 2), 1, 0);

    expectedOutput.add(new StreamRecord<String>("1337", initialTime + 2));

    testHarness.endInput();

    testHarness.waitForTaskCompletion();

    Assert.assertTrue(
        "RichFunction methods where not called.", TestOpenCloseMapFunction.closeCalled);

    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());
  }
コード例 #2
0
  /** This test verifies that checkpoint barriers are correctly forwarded. */
  @Test
  @SuppressWarnings("unchecked")
  public void testCheckpointBarriers() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask =
        new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness =
        new TwoInputStreamTaskTestHarness<String, Integer, String>(
            coMapTask,
            2,
            2,
            new int[] {1, 2},
            BasicTypeInfo.STRING_TYPE_INFO,
            BasicTypeInfo.INT_TYPE_INFO,
            BasicTypeInfo.STRING_TYPE_INFO);

    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator =
        new CoStreamMap<String, Integer, String>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);

    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;

    testHarness.invoke();

    testHarness.processEvent(new CheckpointBarrier(0, 0), 0, 0);

    // This element should be buffered since we received a checkpoint barrier on
    // this input
    testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);

    // This one should go through
    testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 1);
    expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));

    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<Integer>(11, initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<Integer>(111, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("11", initialTime));
    expectedOutput.add(new StreamRecord<String>("111", initialTime));

    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", testHarness.getOutput(), expectedOutput);

    testHarness.processEvent(new CheckpointBarrier(0, 0), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0), 1, 1);

    testHarness.waitForInputProcessing();

    // now we should see the barrier and after that the buffered elements
    expectedOutput.add(new CheckpointBarrier(0, 0));
    expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", testHarness.getOutput(), expectedOutput);

    testHarness.endInput();

    testHarness.waitForTaskCompletion();

    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(4, resultElements.size());
  }
コード例 #3
0
  /**
   * This test verifies that checkpoint barriers and barrier buffers work correctly with concurrent
   * checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e. some inputs
   * receive barriers from an earlier checkpoint, thereby blocking, then all inputs receive barriers
   * from a later checkpoint.
   */
  @Test
  @SuppressWarnings("unchecked")
  public void testOvertakingCheckpointBarriers() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask =
        new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness =
        new TwoInputStreamTaskTestHarness<String, Integer, String>(
            coMapTask,
            2,
            2,
            new int[] {1, 2},
            BasicTypeInfo.STRING_TYPE_INFO,
            BasicTypeInfo.INT_TYPE_INFO,
            BasicTypeInfo.STRING_TYPE_INFO);

    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator =
        new CoStreamMap<String, Integer, String>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);

    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;

    testHarness.invoke();

    testHarness.processEvent(new CheckpointBarrier(0, 0), 0, 0);

    // These elements should be buffered until we receive barriers from
    // all inputs
    testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 0);

    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<Integer>(42, initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<Integer>(1337, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("42", initialTime));
    expectedOutput.add(new StreamRecord<String>("1337", initialTime));

    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());

    // Now give a later barrier to all inputs, this should unblock the first channel,
    // thereby allowing the two blocked elements through
    testHarness.processEvent(new CheckpointBarrier(1, 1), 0, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(1, 1), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1), 1, 1);

    expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
    expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));
    expectedOutput.add(new CheckpointBarrier(1, 1));

    testHarness.waitForInputProcessing();

    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());

    // Then give the earlier barrier, these should be ignored
    testHarness.processEvent(new CheckpointBarrier(0, 0), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0), 1, 1);

    testHarness.waitForInputProcessing();

    testHarness.endInput();

    testHarness.waitForTaskCompletion();

    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());
  }
コード例 #4
0
  /**
   * This test verifies that watermarks are correctly forwarded. This also checks whether watermarks
   * are forwarded only when we have received watermarks from all inputs. The forwarded watermark
   * must be the minimum of the watermarks of all inputs.
   */
  @Test
  @SuppressWarnings("unchecked")
  public void testWatermarkForwarding() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask =
        new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness =
        new TwoInputStreamTaskTestHarness<String, Integer, String>(
            coMapTask,
            2,
            2,
            new int[] {1, 2},
            BasicTypeInfo.STRING_TYPE_INFO,
            BasicTypeInfo.INT_TYPE_INFO,
            BasicTypeInfo.STRING_TYPE_INFO);

    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator =
        new CoStreamMap<String, Integer, String>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);

    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;

    testHarness.invoke();

    testHarness.processElement(new Watermark(initialTime), 0, 0);
    testHarness.processElement(new Watermark(initialTime), 0, 1);

    testHarness.processElement(new Watermark(initialTime), 1, 0);

    // now the output should still be empty
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());

    testHarness.processElement(new Watermark(initialTime), 1, 1);

    // now the watermark should have propagated, Map simply forward Watermarks
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime));
    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());

    // contrary to checkpoint barriers these elements are not blocked by watermarks
    testHarness.processElement(new StreamRecord<String>("Hello", initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<Integer>(42, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("Hello", initialTime));
    expectedOutput.add(new StreamRecord<String>("42", initialTime));

    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());

    testHarness.processElement(new Watermark(initialTime + 4), 0, 0);
    testHarness.processElement(new Watermark(initialTime + 3), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 3), 1, 0);
    testHarness.processElement(new Watermark(initialTime + 2), 1, 1);

    // check whether we get the minimum of all the watermarks, this must also only occur in
    // the output after the two StreamRecords
    expectedOutput.add(new Watermark(initialTime + 2));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());

    // advance watermark from one of the inputs, now we should get a now one since the
    // minimum increases
    testHarness.processElement(new Watermark(initialTime + 4), 1, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 3));
    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());

    // advance the other two inputs, now we should get a new one since the
    // minimum increases again
    testHarness.processElement(new Watermark(initialTime + 4), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 4), 1, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 4));
    TestHarnessUtil.assertOutputEquals(
        "Output was not correct.", expectedOutput, testHarness.getOutput());

    testHarness.endInput();

    testHarness.waitForTaskCompletion();

    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(2, resultElements.size());
  }