@Test public void testOpenClose() throws Exception { KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() { @Override public Integer getKey(Integer value) { return value; } }; StreamGroupedFold<Integer, String, Integer> operator = new StreamGroupedFold<>(new TestOpenCloseFoldFunction(), "init"); operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.configureForKeyedStream(keySelector, BasicTypeInfo.INT_TYPE_INFO); long initialTime = 0L; testHarness.open(); testHarness.processElement(new StreamRecord<>(1, initialTime)); testHarness.processElement(new StreamRecord<>(2, initialTime)); testHarness.close(); assertTrue("RichFunction methods where not called.", TestOpenCloseFoldFunction.closeCalled); assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0); }
@Test public void testKeyValueStateInWindowFunction() { try { StatefulFunction.globalCounts.clear(); // tumbling window that triggers every 20 milliseconds AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op = new AccumulatingProcessingTimeWindowOperator<>( new StatefulFunction(), identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, 50, 50); TestTimeServiceProvider timerService = new TestTimeServiceProvider(); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( op, new ExecutionConfig(), timerService, identitySelector, BasicTypeInfo.INT_TYPE_INFO); testHarness.open(); timerService.setCurrentTime(0); testHarness.processElement(new StreamRecord<>(1)); testHarness.processElement(new StreamRecord<>(2)); op.processElement(new StreamRecord<>(1)); op.processElement(new StreamRecord<>(2)); op.processElement(new StreamRecord<>(1)); op.processElement(new StreamRecord<>(1)); op.processElement(new StreamRecord<>(2)); op.processElement(new StreamRecord<>(2)); timerService.setCurrentTime(1000); List<Integer> result = extractFromStreamRecords(testHarness.getOutput()); assertEquals(8, result.size()); Collections.sort(result); assertEquals(Arrays.asList(1, 1, 1, 1, 2, 2, 2, 2), result); assertEquals(4, StatefulFunction.globalCounts.get(1).intValue()); assertEquals(4, StatefulFunction.globalCounts.get(2).intValue()); testHarness.close(); op.dispose(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Test public void testGroupedFold() throws Exception { KeySelector<Integer, String> keySelector = new KeySelector<Integer, String>() { @Override public String getKey(Integer value) { return value.toString(); } }; StreamGroupedFold<Integer, String, String> operator = new StreamGroupedFold<>(new MyFolder(), "100"); operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.configureForKeyedStream(keySelector, BasicTypeInfo.STRING_TYPE_INFO); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); testHarness.processElement(new StreamRecord<>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<>(1, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<>(2, initialTime + 3)); testHarness.processElement(new StreamRecord<>(2, initialTime + 4)); testHarness.processElement(new StreamRecord<>(3, initialTime + 5)); expectedOutput.add(new StreamRecord<>("1001", initialTime + 1)); expectedOutput.add(new StreamRecord<>("10011", initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<>("1002", initialTime + 3)); expectedOutput.add(new StreamRecord<>("10022", initialTime + 4)); expectedOutput.add(new StreamRecord<>("1003", initialTime + 5)); TestHarnessUtil.assertOutputEquals( "Output was not correct.", expectedOutput, testHarness.getOutput()); }
@Test public void checkpointRestoreWithPendingWindowSliding() { try { final int factor = 4; final int windowSlide = 50; final int windowSize = factor * windowSlide; TestTimeServiceProvider timerService = new TestTimeServiceProvider(); // sliding window (200 msecs) every 50 msecs AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op = new AccumulatingProcessingTimeWindowOperator<>( validatingIdentityFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSlide); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(op, new ExecutionConfig(), timerService); timerService.setCurrentTime(0); testHarness.setup(); testHarness.open(); // inject some elements final int numElements = 1000; final int numElementsFirst = 700; for (int i = 0; i < numElementsFirst; i++) { testHarness.processElement(new StreamRecord<>(i)); } // draw a snapshot List<Integer> resultAtSnapshot = extractFromStreamRecords(testHarness.getOutput()); int beforeSnapShot = testHarness.getOutput().size(); StreamStateHandle state = testHarness.snapshot(1L, System.currentTimeMillis()); int afterSnapShot = testHarness.getOutput().size(); assertEquals("operator performed computation during snapshot", beforeSnapShot, afterSnapShot); assertTrue(resultAtSnapshot.size() <= factor * numElementsFirst); // inject the remaining elements - these should not influence the snapshot for (int i = numElementsFirst; i < numElements; i++) { testHarness.processElement(new StreamRecord<>(i)); } op.dispose(); // re-create the operator and restore the state op = new AccumulatingProcessingTimeWindowOperator<>( validatingIdentityFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSlide); testHarness = new OneInputStreamOperatorTestHarness<>(op, new ExecutionConfig(), timerService); testHarness.setup(); testHarness.restore(state); testHarness.open(); // inject again the remaining elements for (int i = numElementsFirst; i < numElements; i++) { testHarness.processElement(new StreamRecord<>(i)); } timerService.setCurrentTime(50); timerService.setCurrentTime(100); timerService.setCurrentTime(150); timerService.setCurrentTime(200); timerService.setCurrentTime(250); timerService.setCurrentTime(300); timerService.setCurrentTime(350); testHarness.close(); op.dispose(); // get and verify the result List<Integer> finalResult = new ArrayList<>(resultAtSnapshot); List<Integer> finalPartialResult = extractFromStreamRecords(testHarness.getOutput()); finalResult.addAll(finalPartialResult); assertEquals(factor * numElements, finalResult.size()); Collections.sort(finalResult); for (int i = 0; i < factor * numElements; i++) { assertEquals(i / factor, finalResult.get(i).intValue()); } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Test public void checkpointRestoreWithPendingWindowTumbling() { try { final int windowSize = 200; // tumbling window that triggers every 200 milliseconds AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op = new AccumulatingProcessingTimeWindowOperator<>( validatingIdentityFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize); TestTimeServiceProvider timerService = new TestTimeServiceProvider(); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(op, new ExecutionConfig(), timerService); testHarness.setup(); testHarness.open(); timerService.setCurrentTime(0); // inject some elements final int numElementsFirst = 700; final int numElements = 1000; for (int i = 0; i < numElementsFirst; i++) { testHarness.processElement(new StreamRecord<>(i)); } // draw a snapshot and dispose the window System.out.println("GOT: " + testHarness.getOutput()); int beforeSnapShot = testHarness.getOutput().size(); StreamStateHandle state = testHarness.snapshot(1L, System.currentTimeMillis()); List<Integer> resultAtSnapshot = extractFromStreamRecords(testHarness.getOutput()); int afterSnapShot = testHarness.getOutput().size(); assertEquals("operator performed computation during snapshot", beforeSnapShot, afterSnapShot); assertTrue(afterSnapShot <= numElementsFirst); // inject some random elements, which should not show up in the state for (int i = 0; i < 300; i++) { testHarness.processElement(new StreamRecord<>(i + numElementsFirst)); } op.dispose(); // re-create the operator and restore the state op = new AccumulatingProcessingTimeWindowOperator<>( validatingIdentityFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize); testHarness = new OneInputStreamOperatorTestHarness<>(op, new ExecutionConfig(), timerService); testHarness.setup(); testHarness.restore(state); testHarness.open(); // inject some more elements for (int i = numElementsFirst; i < numElements; i++) { testHarness.processElement(new StreamRecord<>(i)); } timerService.setCurrentTime(400); // get and verify the result List<Integer> finalResult = new ArrayList<>(); finalResult.addAll(resultAtSnapshot); List<Integer> finalPartialResult = extractFromStreamRecords(testHarness.getOutput()); finalResult.addAll(finalPartialResult); assertEquals(numElements, finalResult.size()); Collections.sort(finalResult); for (int i = 0; i < numElements; i++) { assertEquals(i, finalResult.get(i).intValue()); } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }