Exemplo n.º 1
0
  /**
   * Verify that registering a processing-time timer that is earlier than the existing timers
   * removes the one physical timer and creates one for the earlier timestamp {@link
   * ProcessingTimeService}.
   */
  @Test
  public void testRegisterEarlierProcessingTimerMovesPhysicalProcessingTimer() throws Exception {
    @SuppressWarnings("unchecked")
    Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);

    TestKeyContext keyContext = new TestKeyContext();

    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();

    HeapInternalTimerService<Integer, String> timerService =
        createTimerService(
            mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);

    int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);

    keyContext.setCurrentKey(key);

    timerService.registerProcessingTimeTimer("ciao", 20);

    assertEquals(1, timerService.numProcessingTimeTimers());

    assertEquals(1, processingTimeService.getNumRegisteredTimers());
    assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(20L));

    timerService.registerProcessingTimeTimer("ciao", 10);

    assertEquals(2, timerService.numProcessingTimeTimers());

    assertEquals(1, processingTimeService.getNumRegisteredTimers());
    assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(10L));
  }
Exemplo n.º 2
0
  private static HeapInternalTimerService<Integer, String> restoreTimerService(
      Map<Integer, byte[]> state,
      Triggerable<Integer, String> triggerable,
      KeyContext keyContext,
      ProcessingTimeService processingTimeService,
      KeyGroupsList keyGroupsList,
      int maxParallelism)
      throws Exception {

    // create an empty service
    HeapInternalTimerService<Integer, String> service =
        new HeapInternalTimerService<>(
            maxParallelism, keyGroupsList, keyContext, processingTimeService);

    // restore the timers
    for (Integer keyGroupIndex : keyGroupsList) {
      if (state.containsKey(keyGroupIndex)) {
        service.restoreTimersForKeyGroup(
            new DataInputViewStreamWrapper(new ByteArrayInputStream(state.get(keyGroupIndex))),
            keyGroupIndex,
            HeapInternalTimerServiceTest.class.getClassLoader());
      }
    }

    // initialize the service
    service.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, triggerable);
    return service;
  }
Exemplo n.º 3
0
  @Test
  public void testRegisteringProcessingTimeTimerInOnProcessingTimeDoesNotLeakPhysicalTimers()
      throws Exception {
    @SuppressWarnings("unchecked")
    Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);

    TestKeyContext keyContext = new TestKeyContext();

    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();

    final HeapInternalTimerService<Integer, String> timerService =
        createTimerService(
            mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);

    int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);

    keyContext.setCurrentKey(key);

    timerService.registerProcessingTimeTimer("ciao", 10);

    assertEquals(1, timerService.numProcessingTimeTimers());

    assertEquals(1, processingTimeService.getNumRegisteredTimers());
    assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(10L));

    doAnswer(
            new Answer<Object>() {
              @Override
              public Object answer(InvocationOnMock invocation) throws Exception {
                timerService.registerProcessingTimeTimer("ciao", 20);
                return null;
              }
            })
        .when(mockTriggerable)
        .onProcessingTime(anyInternalTimer());

    processingTimeService.setCurrentTime(10);

    assertEquals(1, processingTimeService.getNumRegisteredTimers());
    assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(20L));

    doAnswer(
            new Answer<Object>() {
              @Override
              public Object answer(InvocationOnMock invocation) throws Exception {
                timerService.registerProcessingTimeTimer("ciao", 30);
                return null;
              }
            })
        .when(mockTriggerable)
        .onProcessingTime(anyInternalTimer());

    processingTimeService.setCurrentTime(20);

    assertEquals(1, timerService.numProcessingTimeTimers());

    assertEquals(1, processingTimeService.getNumRegisteredTimers());
    assertThat(processingTimeService.getRegisteredTimerTimestamps(), containsInAnyOrder(30L));
  }
Exemplo n.º 4
0
  private static HeapInternalTimerService<Integer, String> createTimerService(
      Triggerable<Integer, String> triggerable,
      KeyContext keyContext,
      ProcessingTimeService processingTimeService,
      KeyGroupsList keyGroupList,
      int maxParallelism) {
    HeapInternalTimerService<Integer, String> service =
        new HeapInternalTimerService<>(
            maxParallelism, keyGroupList, keyContext, processingTimeService);

    service.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, triggerable);
    return service;
  }
Exemplo n.º 5
0
  @Test
  public void testCurrentEventTime() throws Exception {

    @SuppressWarnings("unchecked")
    Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);

    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService =
        createTimerService(
            mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);

    timerService.advanceWatermark(17);
    assertEquals(17, timerService.currentWatermark());

    timerService.advanceWatermark(42);
    assertEquals(42, timerService.currentWatermark());
  }
Exemplo n.º 6
0
  /** This also verifies that we don't have leakage between keys/namespaces. */
  @Test
  public void testSetAndFireEventTimeTimers() throws Exception {
    @SuppressWarnings("unchecked")
    Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);

    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService =
        createTimerService(
            mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);

    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
      key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }

    keyContext.setCurrentKey(key1);

    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);

    keyContext.setCurrentKey(key2);

    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);

    assertEquals(4, timerService.numEventTimeTimers());
    assertEquals(2, timerService.numEventTimeTimers("hello"));
    assertEquals(2, timerService.numEventTimeTimers("ciao"));

    timerService.advanceWatermark(10);

    verify(mockTriggerable, times(4)).onEventTime(anyInternalTimer());
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "hello")));

    assertEquals(0, timerService.numEventTimeTimers());
  }
Exemplo n.º 7
0
  @Test
  public void testKeyGroupStartIndexSetting() {

    int startKeyGroupIdx = 7;
    int endKeyGroupIdx = 21;
    KeyGroupsList testKeyGroupList = new KeyGroupRange(startKeyGroupIdx, endKeyGroupIdx);

    TestKeyContext keyContext = new TestKeyContext();

    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();

    HeapInternalTimerService<Integer, String> service =
        new HeapInternalTimerService<>(
            testKeyGroupList.getNumberOfKeyGroups(),
            testKeyGroupList,
            keyContext,
            processingTimeService);

    Assert.assertEquals(startKeyGroupIdx, service.getLocalKeyGroupRangeStartIdx());
  }
Exemplo n.º 8
0
  @Test
  public void testTimerAssignmentToKeyGroups() {
    int totalNoOfTimers = 100;

    int totalNoOfKeyGroups = 100;
    int startKeyGroupIdx = 0;
    int endKeyGroupIdx = totalNoOfKeyGroups - 1; // we have 0 to 99

    @SuppressWarnings("unchecked")
    Set<InternalTimer<Integer, String>>[] expectedNonEmptyTimerSets =
        new HashSet[totalNoOfKeyGroups];

    TestKeyContext keyContext = new TestKeyContext();
    HeapInternalTimerService<Integer, String> timerService =
        new HeapInternalTimerService<>(
            totalNoOfKeyGroups,
            new KeyGroupRange(startKeyGroupIdx, endKeyGroupIdx),
            keyContext,
            new TestProcessingTimeService());

    timerService.startTimerService(
        IntSerializer.INSTANCE, StringSerializer.INSTANCE, mock(Triggerable.class));

    for (int i = 0; i < totalNoOfTimers; i++) {

      // create the timer to be registered
      InternalTimer<Integer, String> timer = new InternalTimer<>(10 + i, i, "hello_world_" + i);
      int keyGroupIdx =
          KeyGroupRangeAssignment.assignToKeyGroup(timer.getKey(), totalNoOfKeyGroups);

      // add it in the adequate expected set of timers per keygroup
      Set<InternalTimer<Integer, String>> timerSet = expectedNonEmptyTimerSets[keyGroupIdx];
      if (timerSet == null) {
        timerSet = new HashSet<>();
        expectedNonEmptyTimerSets[keyGroupIdx] = timerSet;
      }
      timerSet.add(timer);

      // register the timer as both processing and event time one
      keyContext.setCurrentKey(timer.getKey());
      timerService.registerEventTimeTimer(timer.getNamespace(), timer.getTimestamp());
      timerService.registerProcessingTimeTimer(timer.getNamespace(), timer.getTimestamp());
    }

    Set<InternalTimer<Integer, String>>[] eventTimeTimers =
        timerService.getEventTimeTimersPerKeyGroup();
    Set<InternalTimer<Integer, String>>[] processingTimeTimers =
        timerService.getProcessingTimeTimersPerKeyGroup();

    // finally verify that the actual timers per key group sets are the expected ones.
    for (int i = 0; i < expectedNonEmptyTimerSets.length; i++) {
      Set<InternalTimer<Integer, String>> expected = expectedNonEmptyTimerSets[i];
      Set<InternalTimer<Integer, String>> actualEvent = eventTimeTimers[i];
      Set<InternalTimer<Integer, String>> actualProcessing = processingTimeTimers[i];

      if (expected == null) {
        Assert.assertNull(actualEvent);
        Assert.assertNull(actualProcessing);
      } else {
        Assert.assertArrayEquals(expected.toArray(), actualEvent.toArray());
        Assert.assertArrayEquals(expected.toArray(), actualProcessing.toArray());
      }
    }
  }
Exemplo n.º 9
0
  /**
   * This test checks whether timers are assigned to correct key groups and whether snapshot/restore
   * respects key groups.
   */
  @Test
  public void testSnapshotAndRebalancingRestore() throws Exception {
    @SuppressWarnings("unchecked")
    Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);

    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService =
        createTimerService(
            mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);

    int midpoint =
        testKeyGroupRange.getStartKeyGroup()
            + (testKeyGroupRange.getEndKeyGroup() - testKeyGroupRange.getStartKeyGroup()) / 2;

    // get two sub key-ranges so that we can restore two ranges separately
    KeyGroupRange subKeyGroupRange1 =
        new KeyGroupRange(testKeyGroupRange.getStartKeyGroup(), midpoint);
    KeyGroupRange subKeyGroupRange2 =
        new KeyGroupRange(midpoint + 1, testKeyGroupRange.getEndKeyGroup());

    // get two different keys, one per sub range
    int key1 = getKeyInKeyGroupRange(subKeyGroupRange1, maxParallelism);
    int key2 = getKeyInKeyGroupRange(subKeyGroupRange2, maxParallelism);

    keyContext.setCurrentKey(key1);

    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);

    keyContext.setCurrentKey(key2);

    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);

    assertEquals(2, timerService.numProcessingTimeTimers());
    assertEquals(1, timerService.numProcessingTimeTimers("hello"));
    assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
    assertEquals(2, timerService.numEventTimeTimers());
    assertEquals(1, timerService.numEventTimeTimers("hello"));
    assertEquals(1, timerService.numEventTimeTimers("ciao"));

    // one map per sub key-group range
    Map<Integer, byte[]> snapshot1 = new HashMap<>();
    Map<Integer, byte[]> snapshot2 = new HashMap<>();
    for (Integer keyGroupIndex : testKeyGroupRange) {
      ByteArrayOutputStream outStream = new ByteArrayOutputStream();
      timerService.snapshotTimersForKeyGroup(
          new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
      outStream.close();
      if (subKeyGroupRange1.contains(keyGroupIndex)) {
        snapshot1.put(keyGroupIndex, outStream.toByteArray());
      } else if (subKeyGroupRange2.contains(keyGroupIndex)) {
        snapshot2.put(keyGroupIndex, outStream.toByteArray());
      } else {
        throw new IllegalStateException("Key-Group index doesn't belong to any sub range.");
      }
    }

    // from now on we need everything twice. once per sub key-group range
    @SuppressWarnings("unchecked")
    Triggerable<Integer, String> mockTriggerable1 = mock(Triggerable.class);

    @SuppressWarnings("unchecked")
    Triggerable<Integer, String> mockTriggerable2 = mock(Triggerable.class);

    TestKeyContext keyContext1 = new TestKeyContext();
    TestKeyContext keyContext2 = new TestKeyContext();

    TestProcessingTimeService processingTimeService1 = new TestProcessingTimeService();
    TestProcessingTimeService processingTimeService2 = new TestProcessingTimeService();

    HeapInternalTimerService<Integer, String> timerService1 =
        restoreTimerService(
            snapshot1,
            mockTriggerable1,
            keyContext1,
            processingTimeService1,
            subKeyGroupRange1,
            maxParallelism);

    HeapInternalTimerService<Integer, String> timerService2 =
        restoreTimerService(
            snapshot2,
            mockTriggerable2,
            keyContext2,
            processingTimeService2,
            subKeyGroupRange2,
            maxParallelism);

    processingTimeService1.setCurrentTime(10);
    timerService1.advanceWatermark(10);

    verify(mockTriggerable1, times(1)).onProcessingTime(anyInternalTimer());
    verify(mockTriggerable1, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable1, never()).onProcessingTime(eq(new InternalTimer<>(10, key2, "hello")));
    verify(mockTriggerable1, times(1)).onEventTime(anyInternalTimer());
    verify(mockTriggerable1, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable1, never()).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));

    assertEquals(0, timerService1.numEventTimeTimers());

    processingTimeService2.setCurrentTime(10);
    timerService2.advanceWatermark(10);

    verify(mockTriggerable2, times(1)).onProcessingTime(anyInternalTimer());
    verify(mockTriggerable2, never()).onProcessingTime(eq(new InternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable2, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key2, "hello")));
    verify(mockTriggerable2, times(1)).onEventTime(anyInternalTimer());
    verify(mockTriggerable2, never()).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable2, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));

    assertEquals(0, timerService2.numEventTimeTimers());
  }
Exemplo n.º 10
0
  @Test
  public void testSnapshotAndRestore() throws Exception {
    @SuppressWarnings("unchecked")
    Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);

    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService =
        createTimerService(
            mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);

    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
      key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }

    keyContext.setCurrentKey(key1);

    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);

    keyContext.setCurrentKey(key2);

    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);

    assertEquals(2, timerService.numProcessingTimeTimers());
    assertEquals(1, timerService.numProcessingTimeTimers("hello"));
    assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
    assertEquals(2, timerService.numEventTimeTimers());
    assertEquals(1, timerService.numEventTimeTimers("hello"));
    assertEquals(1, timerService.numEventTimeTimers("ciao"));

    Map<Integer, byte[]> snapshot = new HashMap<>();
    for (Integer keyGroupIndex : testKeyGroupRange) {
      ByteArrayOutputStream outStream = new ByteArrayOutputStream();
      timerService.snapshotTimersForKeyGroup(
          new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
      outStream.close();
      snapshot.put(keyGroupIndex, outStream.toByteArray());
    }

    @SuppressWarnings("unchecked")
    Triggerable<Integer, String> mockTriggerable2 = mock(Triggerable.class);

    keyContext = new TestKeyContext();
    processingTimeService = new TestProcessingTimeService();

    timerService =
        restoreTimerService(
            snapshot,
            mockTriggerable2,
            keyContext,
            processingTimeService,
            testKeyGroupRange,
            maxParallelism);

    processingTimeService.setCurrentTime(10);
    timerService.advanceWatermark(10);

    verify(mockTriggerable2, times(2)).onProcessingTime(anyInternalTimer());
    verify(mockTriggerable2, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable2, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key2, "hello")));
    verify(mockTriggerable2, times(2)).onEventTime(anyInternalTimer());
    verify(mockTriggerable2, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable2, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));

    assertEquals(0, timerService.numEventTimeTimers());
  }