/**
* Verify that registering a processing-time timer that is earlier than the existing timers
* removes the one physical timer and creates one for the earlier timestamp {@link
* ProcessingTimeService}.
*/
@Test
public void testRegisterEarlierProcessingTimerMovesPhysicalProcessingTimer() throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
createTimerService(
mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
keyContext.setCurrentKey(key);
timerService.registerProcessingTimeTimer("ciao", 20);
assertEquals(1, timerService.numProcessingTimeTimers());
assertEquals(1, processingTimeService.getNumRegisteredTimers());
assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(20L));
timerService.registerProcessingTimeTimer("ciao", 10);
assertEquals(2, timerService.numProcessingTimeTimers());
assertEquals(1, processingTimeService.getNumRegisteredTimers());
assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(10L));
}
@Test
public void testRegisteringProcessingTimeTimerInOnProcessingTimeDoesNotLeakPhysicalTimers()
throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
final HeapInternalTimerService<Integer, String> timerService =
createTimerService(
mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
keyContext.setCurrentKey(key);
timerService.registerProcessingTimeTimer("ciao", 10);
assertEquals(1, timerService.numProcessingTimeTimers());
assertEquals(1, processingTimeService.getNumRegisteredTimers());
assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(10L));
doAnswer(
new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Exception {
timerService.registerProcessingTimeTimer("ciao", 20);
return null;
}
})
.when(mockTriggerable)
.onProcessingTime(anyInternalTimer());
processingTimeService.setCurrentTime(10);
assertEquals(1, processingTimeService.getNumRegisteredTimers());
assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(20L));
doAnswer(
new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Exception {
timerService.registerProcessingTimeTimer("ciao", 30);
return null;
}
})
.when(mockTriggerable)
.onProcessingTime(anyInternalTimer());
processingTimeService.setCurrentTime(20);
assertEquals(1, timerService.numProcessingTimeTimers());
assertEquals(1, processingTimeService.getNumRegisteredTimers());
assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(30L));
}
/** This also verifies that we don't have leakage between keys/namespaces. */
@Test
public void testSetAndFireProcessingTimeTimers() throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
createTimerService(
mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
// get two different keys
int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
while (key2 == key1) {
key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
}
keyContext.setCurrentKey(key1);
timerService.registerProcessingTimeTimer("ciao", 10);
timerService.registerProcessingTimeTimer("hello", 10);
keyContext.setCurrentKey(key2);
timerService.registerProcessingTimeTimer("ciao", 10);
timerService.registerProcessingTimeTimer("hello", 10);
assertEquals(4, timerService.numProcessingTimeTimers());
assertEquals(2, timerService.numProcessingTimeTimers("hello"));
assertEquals(2, timerService.numProcessingTimeTimers("ciao"));
processingTimeService.setCurrentTime(10);
verify(mockTriggerable, times(4)).onProcessingTime(anyInternalTimer());
verify(mockTriggerable, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key1, "ciao")));
verify(mockTriggerable, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key1, "hello")));
verify(mockTriggerable, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key2, "ciao")));
verify(mockTriggerable, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key2, "hello")));
assertEquals(0, timerService.numProcessingTimeTimers());
}
/**
* This test checks whether timers are assigned to correct key groups and whether snapshot/restore
* respects key groups.
*/
@Test
public void testSnapshotAndRebalancingRestore() throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
createTimerService(
mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
int midpoint =
testKeyGroupRange.getStartKeyGroup()
+ (testKeyGroupRange.getEndKeyGroup() - testKeyGroupRange.getStartKeyGroup()) / 2;
// get two sub key-ranges so that we can restore two ranges separately
KeyGroupRange subKeyGroupRange1 =
new KeyGroupRange(testKeyGroupRange.getStartKeyGroup(), midpoint);
KeyGroupRange subKeyGroupRange2 =
new KeyGroupRange(midpoint + 1, testKeyGroupRange.getEndKeyGroup());
// get two different keys, one per sub range
int key1 = getKeyInKeyGroupRange(subKeyGroupRange1, maxParallelism);
int key2 = getKeyInKeyGroupRange(subKeyGroupRange2, maxParallelism);
keyContext.setCurrentKey(key1);
timerService.registerProcessingTimeTimer("ciao", 10);
timerService.registerEventTimeTimer("hello", 10);
keyContext.setCurrentKey(key2);
timerService.registerEventTimeTimer("ciao", 10);
timerService.registerProcessingTimeTimer("hello", 10);
assertEquals(2, timerService.numProcessingTimeTimers());
assertEquals(1, timerService.numProcessingTimeTimers("hello"));
assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
assertEquals(2, timerService.numEventTimeTimers());
assertEquals(1, timerService.numEventTimeTimers("hello"));
assertEquals(1, timerService.numEventTimeTimers("ciao"));
// one map per sub key-group range
Map<Integer, byte[]> snapshot1 = new HashMap<>();
Map<Integer, byte[]> snapshot2 = new HashMap<>();
for (Integer keyGroupIndex : testKeyGroupRange) {
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
timerService.snapshotTimersForKeyGroup(
new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
outStream.close();
if (subKeyGroupRange1.contains(keyGroupIndex)) {
snapshot1.put(keyGroupIndex, outStream.toByteArray());
} else if (subKeyGroupRange2.contains(keyGroupIndex)) {
snapshot2.put(keyGroupIndex, outStream.toByteArray());
} else {
throw new IllegalStateException("Key-Group index doesn't belong to any sub range.");
}
}
// from now on we need everything twice. once per sub key-group range
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable1 = mock(Triggerable.class);
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable2 = mock(Triggerable.class);
TestKeyContext keyContext1 = new TestKeyContext();
TestKeyContext keyContext2 = new TestKeyContext();
TestProcessingTimeService processingTimeService1 = new TestProcessingTimeService();
TestProcessingTimeService processingTimeService2 = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService1 =
restoreTimerService(
snapshot1,
mockTriggerable1,
keyContext1,
processingTimeService1,
subKeyGroupRange1,
maxParallelism);
HeapInternalTimerService<Integer, String> timerService2 =
restoreTimerService(
snapshot2,
mockTriggerable2,
keyContext2,
processingTimeService2,
subKeyGroupRange2,
maxParallelism);
processingTimeService1.setCurrentTime(10);
timerService1.advanceWatermark(10);
verify(mockTriggerable1, times(1)).onProcessingTime(anyInternalTimer());
verify(mockTriggerable1, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key1, "ciao")));
verify(mockTriggerable1, never()).onProcessingTime(eq(new InternalTimer<>(10, key2, "hello")));
verify(mockTriggerable1, times(1)).onEventTime(anyInternalTimer());
verify(mockTriggerable1, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
verify(mockTriggerable1, never()).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));
assertEquals(0, timerService1.numEventTimeTimers());
processingTimeService2.setCurrentTime(10);
timerService2.advanceWatermark(10);
verify(mockTriggerable2, times(1)).onProcessingTime(anyInternalTimer());
verify(mockTriggerable2, never()).onProcessingTime(eq(new InternalTimer<>(10, key1, "ciao")));
verify(mockTriggerable2, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key2, "hello")));
verify(mockTriggerable2, times(1)).onEventTime(anyInternalTimer());
verify(mockTriggerable2, never()).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
verify(mockTriggerable2, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));
assertEquals(0, timerService2.numEventTimeTimers());
}
@Test
public void testSnapshotAndRestore() throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
createTimerService(
mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
// get two different keys
int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
while (key2 == key1) {
key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
}
keyContext.setCurrentKey(key1);
timerService.registerProcessingTimeTimer("ciao", 10);
timerService.registerEventTimeTimer("hello", 10);
keyContext.setCurrentKey(key2);
timerService.registerEventTimeTimer("ciao", 10);
timerService.registerProcessingTimeTimer("hello", 10);
assertEquals(2, timerService.numProcessingTimeTimers());
assertEquals(1, timerService.numProcessingTimeTimers("hello"));
assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
assertEquals(2, timerService.numEventTimeTimers());
assertEquals(1, timerService.numEventTimeTimers("hello"));
assertEquals(1, timerService.numEventTimeTimers("ciao"));
Map<Integer, byte[]> snapshot = new HashMap<>();
for (Integer keyGroupIndex : testKeyGroupRange) {
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
timerService.snapshotTimersForKeyGroup(
new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
outStream.close();
snapshot.put(keyGroupIndex, outStream.toByteArray());
}
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable2 = mock(Triggerable.class);
keyContext = new TestKeyContext();
processingTimeService = new TestProcessingTimeService();
timerService =
restoreTimerService(
snapshot,
mockTriggerable2,
keyContext,
processingTimeService,
testKeyGroupRange,
maxParallelism);
processingTimeService.setCurrentTime(10);
timerService.advanceWatermark(10);
verify(mockTriggerable2, times(2)).onProcessingTime(anyInternalTimer());
verify(mockTriggerable2, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key1, "ciao")));
verify(mockTriggerable2, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key2, "hello")));
verify(mockTriggerable2, times(2)).onEventTime(anyInternalTimer());
verify(mockTriggerable2, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
verify(mockTriggerable2, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));
assertEquals(0, timerService.numEventTimeTimers());
}