@Test
  public void testKeyValueStateInWindowFunction() {
    final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
    try {
      final CollectingOutput<Integer> out = new CollectingOutput<>(50);
      final Object lock = new Object();
      final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);

      StatefulFunction.globalCounts.clear();

      // tumbling window that triggers every 20 milliseconds
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              new StatefulFunction(),
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              50,
              50);

      op.setup(mockTask, createTaskConfig(identitySelector, IntSerializer.INSTANCE), out);
      op.open();

      synchronized (lock) {
        op.processElement(new StreamRecord<Integer>(1));
        op.processElement(new StreamRecord<Integer>(2));
      }
      out.waitForNElements(2, 60000);

      synchronized (lock) {
        op.processElement(new StreamRecord<Integer>(1));
        op.processElement(new StreamRecord<Integer>(2));
        op.processElement(new StreamRecord<Integer>(1));
        op.processElement(new StreamRecord<Integer>(1));
        op.processElement(new StreamRecord<Integer>(2));
        op.processElement(new StreamRecord<Integer>(2));
      }
      out.waitForNElements(8, 60000);

      List<Integer> result = out.getElements();
      assertEquals(8, result.size());

      Collections.sort(result);
      assertEquals(Arrays.asList(1, 1, 1, 1, 2, 2, 2, 2), result);

      assertEquals(4, StatefulFunction.globalCounts.get(1).intValue());
      assertEquals(4, StatefulFunction.globalCounts.get(2).intValue());

      synchronized (lock) {
        op.close();
      }
      op.dispose();
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    } finally {
      timerService.shutdown();
    }
  }
  @Test
  public void testPropagateExceptionsFromClose() {
    final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
    try {
      final CollectingOutput<Integer> out = new CollectingOutput<>();
      final Object lock = new Object();
      final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);

      WindowFunction<Integer, Integer, Integer, TimeWindow> failingFunction =
          new FailingFunction(100);

      // the operator has a window time that is so long that it will not fire in this test
      final long hundredYears = 100L * 365 * 24 * 60 * 60 * 1000;
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              failingFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              hundredYears,
              hundredYears);

      op.setup(mockTask, new StreamConfig(new Configuration()), out);
      op.open();

      for (int i = 0; i < 150; i++) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i));
        }
      }

      try {
        synchronized (lock) {
          op.close();
        }
        fail("This should fail with an exception");
      } catch (Exception e) {
        assertTrue(
            e.getMessage().contains("Artificial Test Exception")
                || (e.getCause() != null
                    && e.getCause().getMessage().contains("Artificial Test Exception")));
      }

      op.dispose();
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    } finally {
      timerService.shutdown();
    }
  }
  @Test
  public void testTumblingWindow() {
    final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
    try {
      final int windowSize = 50;
      final CollectingOutput<Integer> out = new CollectingOutput<>(windowSize);
      final Object lock = new Object();
      final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);

      // tumbling window that triggers every 20 milliseconds
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              windowSize,
              windowSize);

      op.setup(mockTask, new StreamConfig(new Configuration()), out);
      op.open();

      final int numElements = 1000;

      for (int i = 0; i < numElements; i++) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i));
        }
        Thread.sleep(1);
      }

      synchronized (lock) {
        op.close();
      }
      op.dispose();

      // get and verify the result
      List<Integer> result = out.getElements();
      assertEquals(numElements, result.size());

      Collections.sort(result);
      for (int i = 0; i < numElements; i++) {
        assertEquals(i, result.get(i).intValue());
      }
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    } finally {
      timerService.shutdown();
    }
  }
  @Test
  public void testSlidingWindowSingleElements() {
    final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();

    try {
      final CollectingOutput<Integer> out = new CollectingOutput<>(50);
      final Object lock = new Object();
      final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);

      // tumbling window that triggers every 20 milliseconds
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              150,
              50);

      op.setup(mockTask, new StreamConfig(new Configuration()), out);
      op.open();

      synchronized (lock) {
        op.processElement(new StreamRecord<Integer>(1));
        op.processElement(new StreamRecord<Integer>(2));
      }

      // each element should end up in the output three times
      // wait until the elements have arrived 6 times in the output
      out.waitForNElements(6, 120000);

      List<Integer> result = out.getElements();
      assertEquals(6, result.size());

      Collections.sort(result);
      assertEquals(Arrays.asList(1, 1, 1, 2, 2, 2), result);

      synchronized (lock) {
        op.close();
      }
      op.dispose();
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    } finally {
      timerService.shutdown();
    }
  }
  @Test
  public void testEmitTrailingDataOnClose() {
    final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
    try {
      final CollectingOutput<Integer> out = new CollectingOutput<>();
      final Object lock = new Object();
      final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);

      // the operator has a window time that is so long that it will not fire in this test
      final long oneYear = 365L * 24 * 60 * 60 * 1000;
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              oneYear,
              oneYear);

      op.setup(mockTask, new StreamConfig(new Configuration()), out);
      op.open();

      List<Integer> data = Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
      for (Integer i : data) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i));
        }
      }

      synchronized (lock) {
        op.close();
      }
      op.dispose();

      // get and verify the result
      List<Integer> result = out.getElements();
      Collections.sort(result);
      assertEquals(data, result);
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    } finally {
      timerService.shutdown();
    }
  }
  @Test
  public void checkpointRestoreWithPendingWindowSliding() {
    final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
    try {
      final int factor = 4;
      final int windowSlide = 50;
      final int windowSize = factor * windowSlide;

      final CollectingOutput<Integer> out = new CollectingOutput<>(windowSlide);
      final Object lock = new Object();
      final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);

      // sliding window (200 msecs) every 50 msecs
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              windowSize,
              windowSlide);

      op.setup(mockTask, new StreamConfig(new Configuration()), out);
      op.open();

      // inject some elements
      final int numElements = 1000;
      final int numElementsFirst = 700;

      for (int i = 0; i < numElementsFirst; i++) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i));
        }
        Thread.sleep(1);
      }

      // draw a snapshot
      StreamTaskState state;
      List<Integer> resultAtSnapshot;
      synchronized (lock) {
        int beforeSnapShot = out.getElements().size();
        state = op.snapshotOperatorState(1L, System.currentTimeMillis());
        resultAtSnapshot = new ArrayList<>(out.getElements());
        int afterSnapShot = out.getElements().size();
        assertEquals(
            "operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
      }

      assertTrue(resultAtSnapshot.size() <= factor * numElementsFirst);

      // inject the remaining elements - these should not influence the snapshot
      for (int i = numElementsFirst; i < numElements; i++) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i));
        }
        Thread.sleep(1);
      }

      op.dispose();

      // re-create the operator and restore the state
      final CollectingOutput<Integer> out2 = new CollectingOutput<>(windowSlide);
      op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              windowSize,
              windowSlide);

      op.setup(mockTask, new StreamConfig(new Configuration()), out2);
      op.restoreState(state, 1);
      op.open();

      // inject again the remaining elements
      for (int i = numElementsFirst; i < numElements; i++) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i));
        }
        Thread.sleep(1);
      }

      // for a deterministic result, we need to wait until all pending triggers
      // have fired and emitted their results
      long deadline = System.currentTimeMillis() + 120000;
      do {
        Thread.sleep(20);
      } while (resultAtSnapshot.size() + out2.getElements().size() < factor * numElements
          && System.currentTimeMillis() < deadline);

      synchronized (lock) {
        op.close();
      }
      op.dispose();

      // get and verify the result
      List<Integer> finalResult = new ArrayList<>(resultAtSnapshot);
      finalResult.addAll(out2.getElements());
      assertEquals(factor * numElements, finalResult.size());

      Collections.sort(finalResult);
      for (int i = 0; i < factor * numElements; i++) {
        assertEquals(i / factor, finalResult.get(i).intValue());
      }
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    } finally {
      timerService.shutdown();
    }
  }
  @Test
  public void checkpointRestoreWithPendingWindowTumbling() {
    final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
    try {
      final int windowSize = 200;
      final CollectingOutput<Integer> out = new CollectingOutput<>(windowSize);
      final Object lock = new Object();
      final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);

      // tumbling window that triggers every 50 milliseconds
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              windowSize,
              windowSize);

      op.setup(mockTask, new StreamConfig(new Configuration()), out);
      op.open();

      // inject some elements
      final int numElementsFirst = 700;
      final int numElements = 1000;
      for (int i = 0; i < numElementsFirst; i++) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i));
        }
        Thread.sleep(1);
      }

      // draw a snapshot and dispose the window
      StreamTaskState state;
      List<Integer> resultAtSnapshot;
      synchronized (lock) {
        int beforeSnapShot = out.getElements().size();
        state = op.snapshotOperatorState(1L, System.currentTimeMillis());
        resultAtSnapshot = new ArrayList<>(out.getElements());
        int afterSnapShot = out.getElements().size();
        assertEquals(
            "operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
        assertTrue(afterSnapShot <= numElementsFirst);
      }

      // inject some random elements, which should not show up in the state
      for (int i = 0; i < 300; i++) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i + numElementsFirst));
        }
        Thread.sleep(1);
      }

      op.dispose();

      // re-create the operator and restore the state
      final CollectingOutput<Integer> out2 = new CollectingOutput<>(windowSize);
      op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              windowSize,
              windowSize);

      op.setup(mockTask, new StreamConfig(new Configuration()), out2);
      op.restoreState(state, 1);
      op.open();

      // inject some more elements
      for (int i = numElementsFirst; i < numElements; i++) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i));
        }
        Thread.sleep(1);
      }

      synchronized (lock) {
        op.close();
      }
      op.dispose();

      // get and verify the result
      List<Integer> finalResult = new ArrayList<>(resultAtSnapshot);
      finalResult.addAll(out2.getElements());
      assertEquals(numElements, finalResult.size());

      Collections.sort(finalResult);
      for (int i = 0; i < numElements; i++) {
        assertEquals(i, finalResult.get(i).intValue());
      }
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    } finally {
      timerService.shutdown();
    }
  }
  @Test
  public void testSlidingWindow() {
    final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
    try {
      final CollectingOutput<Integer> out = new CollectingOutput<>(50);
      final Object lock = new Object();
      final StreamTask<?, ?> mockTask = createMockTaskWithTimer(timerService, lock);

      // tumbling window that triggers every 20 milliseconds
      AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
          new AccumulatingProcessingTimeWindowOperator<>(
              validatingIdentityFunction,
              identitySelector,
              IntSerializer.INSTANCE,
              IntSerializer.INSTANCE,
              150,
              50);

      op.setup(mockTask, new StreamConfig(new Configuration()), out);
      op.open();

      final int numElements = 1000;

      for (int i = 0; i < numElements; i++) {
        synchronized (lock) {
          op.processElement(new StreamRecord<Integer>(i));
        }
        Thread.sleep(1);
      }

      synchronized (lock) {
        op.close();
      }
      op.dispose();

      // get and verify the result
      List<Integer> result = out.getElements();

      // if we kept this running, each element would be in the result three times (for each slide).
      // we are closing the window before the final panes are through three times, so we may have
      // less
      // elements.
      if (result.size() < numElements || result.size() > 3 * numElements) {
        fail("Wrong number of results: " + result.size());
      }

      Collections.sort(result);
      int lastNum = -1;
      int lastCount = -1;

      for (int num : result) {
        if (num == lastNum) {
          lastCount++;
          assertTrue(lastCount <= 3);
        } else {
          lastNum = num;
          lastCount = 1;
        }
      }
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    } finally {
      timerService.shutdown();
    }
  }
  @Test
  public void testWindowTriggerTimeAlignment() {
    try {
      @SuppressWarnings("unchecked")
      final Output<StreamRecord<String>> mockOut = mock(Output.class);
      final StreamTask<?, ?> mockTask = createMockTask();

      AccumulatingProcessingTimeWindowOperator<String, String, String> op;

      op =
          new AccumulatingProcessingTimeWindowOperator<>(
              mockFunction,
              mockKeySelector,
              StringSerializer.INSTANCE,
              StringSerializer.INSTANCE,
              5000,
              1000);
      op.setup(mockTask, new StreamConfig(new Configuration()), mockOut);
      op.open();
      assertTrue(op.getNextSlideTime() % 1000 == 0);
      assertTrue(op.getNextEvaluationTime() % 1000 == 0);
      op.dispose();

      op =
          new AccumulatingProcessingTimeWindowOperator<>(
              mockFunction,
              mockKeySelector,
              StringSerializer.INSTANCE,
              StringSerializer.INSTANCE,
              1000,
              1000);
      op.setup(mockTask, new StreamConfig(new Configuration()), mockOut);
      op.open();
      assertTrue(op.getNextSlideTime() % 1000 == 0);
      assertTrue(op.getNextEvaluationTime() % 1000 == 0);
      op.dispose();

      op =
          new AccumulatingProcessingTimeWindowOperator<>(
              mockFunction,
              mockKeySelector,
              StringSerializer.INSTANCE,
              StringSerializer.INSTANCE,
              1500,
              1000);
      op.setup(mockTask, new StreamConfig(new Configuration()), mockOut);
      op.open();
      assertTrue(op.getNextSlideTime() % 500 == 0);
      assertTrue(op.getNextEvaluationTime() % 1000 == 0);
      op.dispose();

      op =
          new AccumulatingProcessingTimeWindowOperator<>(
              mockFunction,
              mockKeySelector,
              StringSerializer.INSTANCE,
              StringSerializer.INSTANCE,
              1200,
              1100);
      op.setup(mockTask, new StreamConfig(new Configuration()), mockOut);
      op.open();
      assertTrue(op.getNextSlideTime() % 100 == 0);
      assertTrue(op.getNextEvaluationTime() % 1100 == 0);
      op.dispose();
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }