@Test
public void testOpenClose() throws Exception {
KeySelector<Integer, Integer> keySelector =
new KeySelector<Integer, Integer>() {
@Override
public Integer getKey(Integer value) {
return value;
}
};
StreamGroupedFold<Integer, String, Integer> operator =
new StreamGroupedFold<>(new TestOpenCloseFoldFunction(), "init");
operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
OneInputStreamOperatorTestHarness<Integer, String> testHarness =
new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(keySelector, BasicTypeInfo.INT_TYPE_INFO);
long initialTime = 0L;
testHarness.open();
testHarness.processElement(new StreamRecord<>(1, initialTime));
testHarness.processElement(new StreamRecord<>(2, initialTime));
testHarness.close();
assertTrue("RichFunction methods where not called.", TestOpenCloseFoldFunction.closeCalled);
assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0);
}
@Test
public void testKeyValueStateInWindowFunction() {
try {
StatefulFunction.globalCounts.clear();
// tumbling window that triggers every 20 milliseconds
AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
new AccumulatingProcessingTimeWindowOperator<>(
new StatefulFunction(),
identitySelector,
IntSerializer.INSTANCE,
IntSerializer.INSTANCE,
50,
50);
TestTimeServiceProvider timerService = new TestTimeServiceProvider();
OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
new KeyedOneInputStreamOperatorTestHarness<>(
op,
new ExecutionConfig(),
timerService,
identitySelector,
BasicTypeInfo.INT_TYPE_INFO);
testHarness.open();
timerService.setCurrentTime(0);
testHarness.processElement(new StreamRecord<>(1));
testHarness.processElement(new StreamRecord<>(2));
op.processElement(new StreamRecord<>(1));
op.processElement(new StreamRecord<>(2));
op.processElement(new StreamRecord<>(1));
op.processElement(new StreamRecord<>(1));
op.processElement(new StreamRecord<>(2));
op.processElement(new StreamRecord<>(2));
timerService.setCurrentTime(1000);
List<Integer> result = extractFromStreamRecords(testHarness.getOutput());
assertEquals(8, result.size());
Collections.sort(result);
assertEquals(Arrays.asList(1, 1, 1, 1, 2, 2, 2, 2), result);
assertEquals(4, StatefulFunction.globalCounts.get(1).intValue());
assertEquals(4, StatefulFunction.globalCounts.get(2).intValue());
testHarness.close();
op.dispose();
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
@Test
public void checkpointRestoreWithPendingWindowSliding() {
try {
final int factor = 4;
final int windowSlide = 50;
final int windowSize = factor * windowSlide;
TestTimeServiceProvider timerService = new TestTimeServiceProvider();
// sliding window (200 msecs) every 50 msecs
AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
new AccumulatingProcessingTimeWindowOperator<>(
validatingIdentityFunction,
identitySelector,
IntSerializer.INSTANCE,
IntSerializer.INSTANCE,
windowSize,
windowSlide);
OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
new OneInputStreamOperatorTestHarness<>(op, new ExecutionConfig(), timerService);
timerService.setCurrentTime(0);
testHarness.setup();
testHarness.open();
// inject some elements
final int numElements = 1000;
final int numElementsFirst = 700;
for (int i = 0; i < numElementsFirst; i++) {
testHarness.processElement(new StreamRecord<>(i));
}
// draw a snapshot
List<Integer> resultAtSnapshot = extractFromStreamRecords(testHarness.getOutput());
int beforeSnapShot = testHarness.getOutput().size();
StreamStateHandle state = testHarness.snapshot(1L, System.currentTimeMillis());
int afterSnapShot = testHarness.getOutput().size();
assertEquals("operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
assertTrue(resultAtSnapshot.size() <= factor * numElementsFirst);
// inject the remaining elements - these should not influence the snapshot
for (int i = numElementsFirst; i < numElements; i++) {
testHarness.processElement(new StreamRecord<>(i));
}
op.dispose();
// re-create the operator and restore the state
op =
new AccumulatingProcessingTimeWindowOperator<>(
validatingIdentityFunction,
identitySelector,
IntSerializer.INSTANCE,
IntSerializer.INSTANCE,
windowSize,
windowSlide);
testHarness =
new OneInputStreamOperatorTestHarness<>(op, new ExecutionConfig(), timerService);
testHarness.setup();
testHarness.restore(state);
testHarness.open();
// inject again the remaining elements
for (int i = numElementsFirst; i < numElements; i++) {
testHarness.processElement(new StreamRecord<>(i));
}
timerService.setCurrentTime(50);
timerService.setCurrentTime(100);
timerService.setCurrentTime(150);
timerService.setCurrentTime(200);
timerService.setCurrentTime(250);
timerService.setCurrentTime(300);
timerService.setCurrentTime(350);
testHarness.close();
op.dispose();
// get and verify the result
List<Integer> finalResult = new ArrayList<>(resultAtSnapshot);
List<Integer> finalPartialResult = extractFromStreamRecords(testHarness.getOutput());
finalResult.addAll(finalPartialResult);
assertEquals(factor * numElements, finalResult.size());
Collections.sort(finalResult);
for (int i = 0; i < factor * numElements; i++) {
assertEquals(i / factor, finalResult.get(i).intValue());
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
