Ejemplo n.º 1
0
  @Test
  public void testOpenClose() throws Exception {
    KeySelector<Integer, Integer> keySelector =
        new KeySelector<Integer, Integer>() {
          @Override
          public Integer getKey(Integer value) {
            return value;
          }
        };

    StreamGroupedFold<Integer, String, Integer> operator =
        new StreamGroupedFold<>(new TestOpenCloseFoldFunction(), "init");
    operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());

    OneInputStreamOperatorTestHarness<Integer, String> testHarness =
        new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.configureForKeyedStream(keySelector, BasicTypeInfo.INT_TYPE_INFO);

    long initialTime = 0L;

    testHarness.open();

    testHarness.processElement(new StreamRecord<>(1, initialTime));
    testHarness.processElement(new StreamRecord<>(2, initialTime));

    testHarness.close();

    assertTrue("RichFunction methods where not called.", TestOpenCloseFoldFunction.closeCalled);
    assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0);
  }
  @Test
  public void testKeyValueStateInWindowFunction() {
    try {

      StatefulFunction.globalCounts.clear();

      // tumbling window that triggers every 20 milliseconds
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              new StatefulFunction(),
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              50,
              50);

      TestTimeServiceProvider timerService = new TestTimeServiceProvider();

      OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
          new KeyedOneInputStreamOperatorTestHarness<>(
              op,
              new ExecutionConfig(),
              timerService,
              identitySelector,
              BasicTypeInfo.INT_TYPE_INFO);

      testHarness.open();

      timerService.setCurrentTime(0);

      testHarness.processElement(new StreamRecord<>(1));
      testHarness.processElement(new StreamRecord<>(2));

      op.processElement(new StreamRecord<>(1));
      op.processElement(new StreamRecord<>(2));
      op.processElement(new StreamRecord<>(1));
      op.processElement(new StreamRecord<>(1));
      op.processElement(new StreamRecord<>(2));
      op.processElement(new StreamRecord<>(2));

      timerService.setCurrentTime(1000);

      List<Integer> result = extractFromStreamRecords(testHarness.getOutput());
      assertEquals(8, result.size());

      Collections.sort(result);
      assertEquals(Arrays.asList(1, 1, 1, 1, 2, 2, 2, 2), result);

      assertEquals(4, StatefulFunction.globalCounts.get(1).intValue());
      assertEquals(4, StatefulFunction.globalCounts.get(2).intValue());

      testHarness.close();
      op.dispose();
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }
  @Test
  public void checkpointRestoreWithPendingWindowSliding() {
    try {
      final int factor = 4;
      final int windowSlide = 50;
      final int windowSize = factor * windowSlide;

      TestTimeServiceProvider timerService = new TestTimeServiceProvider();

      // sliding window (200 msecs) every 50 msecs
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              windowSize,
              windowSlide);

      OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
          new OneInputStreamOperatorTestHarness<>(op, new ExecutionConfig(), timerService);

      timerService.setCurrentTime(0);

      testHarness.setup();
      testHarness.open();

      // inject some elements
      final int numElements = 1000;
      final int numElementsFirst = 700;

      for (int i = 0; i < numElementsFirst; i++) {
        testHarness.processElement(new StreamRecord<>(i));
      }

      // draw a snapshot
      List<Integer> resultAtSnapshot = extractFromStreamRecords(testHarness.getOutput());
      int beforeSnapShot = testHarness.getOutput().size();
      StreamStateHandle state = testHarness.snapshot(1L, System.currentTimeMillis());
      int afterSnapShot = testHarness.getOutput().size();
      assertEquals("operator performed computation during snapshot", beforeSnapShot, afterSnapShot);

      assertTrue(resultAtSnapshot.size() <= factor * numElementsFirst);

      // inject the remaining elements - these should not influence the snapshot
      for (int i = numElementsFirst; i < numElements; i++) {
        testHarness.processElement(new StreamRecord<>(i));
      }

      op.dispose();

      // re-create the operator and restore the state
      op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              windowSize,
              windowSlide);

      testHarness =
          new OneInputStreamOperatorTestHarness<>(op, new ExecutionConfig(), timerService);

      testHarness.setup();
      testHarness.restore(state);
      testHarness.open();

      // inject again the remaining elements
      for (int i = numElementsFirst; i < numElements; i++) {
        testHarness.processElement(new StreamRecord<>(i));
      }

      timerService.setCurrentTime(50);
      timerService.setCurrentTime(100);
      timerService.setCurrentTime(150);
      timerService.setCurrentTime(200);
      timerService.setCurrentTime(250);
      timerService.setCurrentTime(300);
      timerService.setCurrentTime(350);

      testHarness.close();
      op.dispose();

      // get and verify the result
      List<Integer> finalResult = new ArrayList<>(resultAtSnapshot);
      List<Integer> finalPartialResult = extractFromStreamRecords(testHarness.getOutput());
      finalResult.addAll(finalPartialResult);
      assertEquals(factor * numElements, finalResult.size());

      Collections.sort(finalResult);
      for (int i = 0; i < factor * numElements; i++) {
        assertEquals(i / factor, finalResult.get(i).intValue());
      }
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }