@Test
public void checkpointRestoreWithPendingWindowSliding() {
final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
try {
final int factor = 4;
final int windowSlide = 50;
final int windowSize = factor * windowSlide;
final CollectingOutput<Integer> out = new CollectingOutput<>(windowSlide);
final Object lock = new Object();
final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);
// sliding window (200 msecs) every 50 msecs
AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
new AccumulatingProcessingTimeWindowOperator<>(
validatingIdentityFunction,
identitySelector,
IntSerializer.INSTANCE,
IntSerializer.INSTANCE,
windowSize,
windowSlide);
op.setup(mockTask, new StreamConfig(new Configuration()), out);
op.open();
// inject some elements
final int numElements = 1000;
final int numElementsFirst = 700;
for (int i = 0; i < numElementsFirst; i++) {
synchronized (lock) {
op.processElement(new StreamRecord<Integer>(i));
}
Thread.sleep(1);
}
// draw a snapshot
StreamTaskState state;
List<Integer> resultAtSnapshot;
synchronized (lock) {
int beforeSnapShot = out.getElements().size();
state = op.snapshotOperatorState(1L, System.currentTimeMillis());
resultAtSnapshot = new ArrayList<>(out.getElements());
int afterSnapShot = out.getElements().size();
assertEquals(
"operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
}
assertTrue(resultAtSnapshot.size() <= factor * numElementsFirst);
// inject the remaining elements - these should not influence the snapshot
for (int i = numElementsFirst; i < numElements; i++) {
synchronized (lock) {
op.processElement(new StreamRecord<Integer>(i));
}
Thread.sleep(1);
}
op.dispose();
// re-create the operator and restore the state
final CollectingOutput<Integer> out2 = new CollectingOutput<>(windowSlide);
op =
new AccumulatingProcessingTimeWindowOperator<>(
validatingIdentityFunction,
identitySelector,
IntSerializer.INSTANCE,
IntSerializer.INSTANCE,
windowSize,
windowSlide);
op.setup(mockTask, new StreamConfig(new Configuration()), out2);
op.restoreState(state, 1);
op.open();
// inject again the remaining elements
for (int i = numElementsFirst; i < numElements; i++) {
synchronized (lock) {
op.processElement(new StreamRecord<Integer>(i));
}
Thread.sleep(1);
}
// for a deterministic result, we need to wait until all pending triggers
// have fired and emitted their results
long deadline = System.currentTimeMillis() + 120000;
do {
Thread.sleep(20);
} while (resultAtSnapshot.size() + out2.getElements().size() < factor * numElements
&& System.currentTimeMillis() < deadline);
synchronized (lock) {
op.close();
}
op.dispose();
// get and verify the result
List<Integer> finalResult = new ArrayList<>(resultAtSnapshot);
finalResult.addAll(out2.getElements());
assertEquals(factor * numElements, finalResult.size());
Collections.sort(finalResult);
for (int i = 0; i < factor * numElements; i++) {
assertEquals(i / factor, finalResult.get(i).intValue());
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
timerService.shutdown();
}
}
@Test
public void checkpointRestoreWithPendingWindowTumbling() {
final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
try {
final int windowSize = 200;
final CollectingOutput<Integer> out = new CollectingOutput<>(windowSize);
final Object lock = new Object();
final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);
// tumbling window that triggers every 50 milliseconds
AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
new AccumulatingProcessingTimeWindowOperator<>(
validatingIdentityFunction,
identitySelector,
IntSerializer.INSTANCE,
IntSerializer.INSTANCE,
windowSize,
windowSize);
op.setup(mockTask, new StreamConfig(new Configuration()), out);
op.open();
// inject some elements
final int numElementsFirst = 700;
final int numElements = 1000;
for (int i = 0; i < numElementsFirst; i++) {
synchronized (lock) {
op.processElement(new StreamRecord<Integer>(i));
}
Thread.sleep(1);
}
// draw a snapshot and dispose the window
StreamTaskState state;
List<Integer> resultAtSnapshot;
synchronized (lock) {
int beforeSnapShot = out.getElements().size();
state = op.snapshotOperatorState(1L, System.currentTimeMillis());
resultAtSnapshot = new ArrayList<>(out.getElements());
int afterSnapShot = out.getElements().size();
assertEquals(
"operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
assertTrue(afterSnapShot <= numElementsFirst);
}
// inject some random elements, which should not show up in the state
for (int i = 0; i < 300; i++) {
synchronized (lock) {
op.processElement(new StreamRecord<Integer>(i + numElementsFirst));
}
Thread.sleep(1);
}
op.dispose();
// re-create the operator and restore the state
final CollectingOutput<Integer> out2 = new CollectingOutput<>(windowSize);
op =
new AccumulatingProcessingTimeWindowOperator<>(
validatingIdentityFunction,
identitySelector,
IntSerializer.INSTANCE,
IntSerializer.INSTANCE,
windowSize,
windowSize);
op.setup(mockTask, new StreamConfig(new Configuration()), out2);
op.restoreState(state, 1);
op.open();
// inject some more elements
for (int i = numElementsFirst; i < numElements; i++) {
synchronized (lock) {
op.processElement(new StreamRecord<Integer>(i));
}
Thread.sleep(1);
}
synchronized (lock) {
op.close();
}
op.dispose();
// get and verify the result
List<Integer> finalResult = new ArrayList<>(resultAtSnapshot);
finalResult.addAll(out2.getElements());
assertEquals(numElements, finalResult.size());
Collections.sort(finalResult);
for (int i = 0; i < numElements; i++) {
assertEquals(i, finalResult.get(i).intValue());
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
timerService.shutdown();
}
}