@Test
  @SuppressWarnings("unchecked")
  public void testSlidingEventTimeWindowsApply() throws Exception {
    closeCalled.set(0);

    final int WINDOW_SIZE = 3;
    final int WINDOW_SLIDE = 1;

    TypeInformation<Tuple2<String, Integer>> inputType =
        TypeInfoParser.parse("Tuple2<String, Integer>");

    ListStateDescriptor<Tuple2<String, Integer>> stateDesc =
        new ListStateDescriptor<>(
            "window-contents", inputType.createSerializer(new ExecutionConfig()));

    WindowOperator<
            String,
            Tuple2<String, Integer>,
            Iterable<Tuple2<String, Integer>>,
            Tuple2<String, Integer>,
            TimeWindow>
        operator =
            new WindowOperator<>(
                SlidingEventTimeWindows.of(
                    Time.of(WINDOW_SIZE, TimeUnit.SECONDS),
                    Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)),
                new TimeWindow.Serializer(),
                new TupleKeySelector(),
                BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
                stateDesc,
                new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
                EventTimeTrigger.create());

    operator.setInputType(inputType, new ExecutionConfig());

    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
        testHarness = new OneInputStreamOperatorTestHarness<>(operator);

    testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

    testHarness.open();

    testSlidingEventTimeWindows(testHarness);

    testHarness.close();

    Assert.assertEquals("Close was not called.", 1, closeCalled.get());
  }
  @Test
  @SuppressWarnings("unchecked")
  public void testTumblingEventTimeWindowsReduce() throws Exception {
    closeCalled.set(0);

    final int WINDOW_SIZE = 3;

    TypeInformation<Tuple2<String, Integer>> inputType =
        TypeInfoParser.parse("Tuple2<String, Integer>");

    ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc =
        new ReducingStateDescriptor<>(
            "window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));

    WindowOperator<
            String,
            Tuple2<String, Integer>,
            Tuple2<String, Integer>,
            Tuple2<String, Integer>,
            TimeWindow>
        operator =
            new WindowOperator<>(
                TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
                new TimeWindow.Serializer(),
                new TupleKeySelector(),
                BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
                stateDesc,
                new InternalSingleValueWindowFunction<>(
                    new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
                EventTimeTrigger.create());

    operator.setInputType(
        TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"),
        new ExecutionConfig());

    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
        testHarness = new OneInputStreamOperatorTestHarness<>(operator);

    testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

    testHarness.open();

    testTumblingEventTimeWindows(testHarness);

    testHarness.close();
  }
  @Test
  public void testTumblingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = 3000;
    final int WINDOW_SIZE = 100;
    final int NUM_KEYS = 100;
    FailingSource.reset();

    try {
      StreamExecutionEnvironment env =
          StreamExecutionEnvironment.createRemoteEnvironment(
              "localhost", cluster.getLeaderRPCPort());

      env.setParallelism(PARALLELISM);
      env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
      env.enableCheckpointing(100);
      env.setNumberOfExecutionRetries(3);
      env.getConfig().disableSysoutLogging();

      env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3))
          .rebalance()
          .keyBy(0)
          .timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS))
          .apply(
              new RichWindowFunction<
                  Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

                private boolean open = false;

                @Override
                public void open(Configuration parameters) {
                  assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                  open = true;
                }

                @Override
                public void apply(
                    Tuple tuple,
                    TimeWindow window,
                    Iterable<Tuple2<Long, IntType>> values,
                    Collector<Tuple4<Long, Long, Long, IntType>> out) {

                  // validate that the function has been opened properly
                  assertTrue(open);

                  int sum = 0;
                  long key = -1;

                  for (Tuple2<Long, IntType> value : values) {
                    sum += value.f1.value;
                    key = value.f0;
                  }
                  out.collect(
                      new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum)));
                }
              })
          .addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE))
          .setParallelism(1);

      tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }
  @Test
  public void testTumblingTimeWindowWithKVState() {
    final int NUM_ELEMENTS_PER_KEY = 3000;
    final int WINDOW_SIZE = 100;
    final int NUM_KEYS = 100;
    FailingSource.reset();

    try {
      StreamExecutionEnvironment env =
          StreamExecutionEnvironment.createRemoteEnvironment(
              "localhost", cluster.getLeaderRPCPort());

      env.setParallelism(PARALLELISM);
      env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
      env.enableCheckpointing(100);
      env.setNumberOfExecutionRetries(3);
      env.getConfig().disableSysoutLogging();

      env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3))
          .rebalance()
          .keyBy(0)
          .timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS))
          .apply(
              new RichWindowFunction<
                  Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

                private boolean open = false;

                private OperatorState<Integer> count;

                @Override
                public void open(Configuration parameters) {
                  assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                  open = true;
                  count = getRuntimeContext().getKeyValueState("count", Integer.class, 0);
                }

                @Override
                public void apply(
                    Tuple tuple,
                    TimeWindow window,
                    Iterable<Tuple2<Long, IntType>> values,
                    Collector<Tuple4<Long, Long, Long, IntType>> out)
                    throws Exception {

                  // the window count state starts with the key, so that we get
                  // different count results for each key
                  if (count.value() == 0) {
                    count.update(tuple.<Long>getField(0).intValue());
                  }

                  // validate that the function has been opened properly
                  assertTrue(open);

                  count.update(count.value() + 1);
                  out.collect(
                      new Tuple4<>(
                          tuple.<Long>getField(0),
                          window.getStart(),
                          window.getEnd(),
                          new IntType(count.value())));
                }
              })
          .addSink(new CountValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE))
          .setParallelism(1);

      tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }
  @Test
  @SuppressWarnings("unchecked")
  public void testContinuousWatermarkTrigger() throws Exception {
    closeCalled.set(0);

    final int WINDOW_SIZE = 3;

    TypeInformation<Tuple2<String, Integer>> inputType =
        TypeInfoParser.parse("Tuple2<String, Integer>");

    ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc =
        new ReducingStateDescriptor<>(
            "window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));

    WindowOperator<
            String,
            Tuple2<String, Integer>,
            Tuple2<String, Integer>,
            Tuple2<String, Integer>,
            GlobalWindow>
        operator =
            new WindowOperator<>(
                GlobalWindows.create(),
                new GlobalWindow.Serializer(),
                new TupleKeySelector(),
                BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
                stateDesc,
                new InternalSingleValueWindowFunction<>(
                    new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()),
                ContinuousEventTimeTrigger.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)));

    operator.setInputType(
        TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"),
        new ExecutionConfig());

    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
        testHarness = new OneInputStreamOperatorTestHarness<>(operator);

    testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

    testHarness.open();

    // The global window actually ignores these timestamps...

    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));

    // add elements out-of-order
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));

    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));

    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));

    testHarness.processWatermark(new Watermark(initialTime + 1000));
    expectedOutput.add(new Watermark(1000));
    TestHarnessUtil.assertOutputEqualsSorted(
        "Output was not correct.",
        expectedOutput,
        testHarness.getOutput(),
        new ResultSortComparator());

    testHarness.processWatermark(new Watermark(initialTime + 2000));
    expectedOutput.add(new Watermark(2000));
    TestHarnessUtil.assertOutputEqualsSorted(
        "Output was not correct.",
        expectedOutput,
        testHarness.getOutput(),
        new ResultSortComparator());

    testHarness.processWatermark(new Watermark(initialTime + 3000));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE));
    expectedOutput.add(new Watermark(3000));
    TestHarnessUtil.assertOutputEqualsSorted(
        "Output was not correct.",
        expectedOutput,
        testHarness.getOutput(),
        new ResultSortComparator());

    testHarness.processWatermark(new Watermark(initialTime + 4000));
    expectedOutput.add(new Watermark(4000));
    TestHarnessUtil.assertOutputEqualsSorted(
        "Output was not correct.",
        expectedOutput,
        testHarness.getOutput(),
        new ResultSortComparator());

    testHarness.processWatermark(new Watermark(initialTime + 5000));
    expectedOutput.add(new Watermark(5000));
    TestHarnessUtil.assertOutputEqualsSorted(
        "Output was not correct.",
        expectedOutput,
        testHarness.getOutput(),
        new ResultSortComparator());

    testHarness.processWatermark(new Watermark(initialTime + 6000));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 5), Long.MAX_VALUE));
    expectedOutput.add(new Watermark(6000));
    TestHarnessUtil.assertOutputEqualsSorted(
        "Output was not correct.",
        expectedOutput,
        testHarness.getOutput(),
        new ResultSortComparator());

    // those don't have any effect...
    testHarness.processWatermark(new Watermark(initialTime + 7000));
    testHarness.processWatermark(new Watermark(initialTime + 8000));
    expectedOutput.add(new Watermark(7000));
    expectedOutput.add(new Watermark(8000));

    TestHarnessUtil.assertOutputEqualsSorted(
        "Output was not correct.",
        expectedOutput,
        testHarness.getOutput(),
        new ResultSortComparator());

    testHarness.close();
  }
Exemplo n.º 6
0
 /**
  * Creates a new {@code TumblingTimeWindows} {@link WindowAssigner} that assigns elements to time
  * windows based on the element timestamp.
  *
  * @param size The size of the generated windows.
  * @return The time policy.
  */
 public static TumblingTimeWindows of(Time size) {
   return new TumblingTimeWindows(size.toMilliseconds());
 }