@Test
@SuppressWarnings("unchecked")
public void testSlidingEventTimeWindowsApply() throws Exception {
closeCalled.set(0);
final int WINDOW_SIZE = 3;
final int WINDOW_SLIDE = 1;
TypeInformation<Tuple2<String, Integer>> inputType =
TypeInfoParser.parse("Tuple2<String, Integer>");
ListStateDescriptor<Tuple2<String, Integer>> stateDesc =
new ListStateDescriptor<>(
"window-contents", inputType.createSerializer(new ExecutionConfig()));
WindowOperator<
String,
Tuple2<String, Integer>,
Iterable<Tuple2<String, Integer>>,
Tuple2<String, Integer>,
TimeWindow>
operator =
new WindowOperator<>(
SlidingEventTimeWindows.of(
Time.of(WINDOW_SIZE, TimeUnit.SECONDS),
Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
EventTimeTrigger.create());
operator.setInputType(inputType, new ExecutionConfig());
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.open();
testSlidingEventTimeWindows(testHarness);
testHarness.close();
Assert.assertEquals("Close was not called.", 1, closeCalled.get());
}
@Test
@SuppressWarnings("unchecked")
public void testTumblingEventTimeWindowsReduce() throws Exception {
closeCalled.set(0);
final int WINDOW_SIZE = 3;
TypeInformation<Tuple2<String, Integer>> inputType =
TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc =
new ReducingStateDescriptor<>(
"window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));
WindowOperator<
String,
Tuple2<String, Integer>,
Tuple2<String, Integer>,
Tuple2<String, Integer>,
TimeWindow>
operator =
new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(
new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
EventTimeTrigger.create());
operator.setInputType(
TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"),
new ExecutionConfig());
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.open();
testTumblingEventTimeWindows(testHarness);
testHarness.close();
}
@Test
public void testTumblingTimeWindow() {
final int NUM_ELEMENTS_PER_KEY = 3000;
final int WINDOW_SIZE = 100;
final int NUM_KEYS = 100;
FailingSource.reset();
try {
StreamExecutionEnvironment env =
StreamExecutionEnvironment.createRemoteEnvironment(
"localhost", cluster.getLeaderRPCPort());
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setNumberOfExecutionRetries(3);
env.getConfig().disableSysoutLogging();
env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS))
.apply(
new RichWindowFunction<
Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out) {
// validate that the function has been opened properly
assertTrue(open);
int sum = 0;
long key = -1;
for (Tuple2<Long, IntType> value : values) {
sum += value.f1.value;
key = value.f0;
}
out.collect(
new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum)));
}
})
.addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE))
.setParallelism(1);
tryExecute(env, "Tumbling Window Test");
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
@Test
public void testTumblingTimeWindowWithKVState() {
final int NUM_ELEMENTS_PER_KEY = 3000;
final int WINDOW_SIZE = 100;
final int NUM_KEYS = 100;
FailingSource.reset();
try {
StreamExecutionEnvironment env =
StreamExecutionEnvironment.createRemoteEnvironment(
"localhost", cluster.getLeaderRPCPort());
env.setParallelism(PARALLELISM);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(100);
env.setNumberOfExecutionRetries(3);
env.getConfig().disableSysoutLogging();
env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3))
.rebalance()
.keyBy(0)
.timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS))
.apply(
new RichWindowFunction<
Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {
private boolean open = false;
private OperatorState<Integer> count;
@Override
public void open(Configuration parameters) {
assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
open = true;
count = getRuntimeContext().getKeyValueState("count", Integer.class, 0);
}
@Override
public void apply(
Tuple tuple,
TimeWindow window,
Iterable<Tuple2<Long, IntType>> values,
Collector<Tuple4<Long, Long, Long, IntType>> out)
throws Exception {
// the window count state starts with the key, so that we get
// different count results for each key
if (count.value() == 0) {
count.update(tuple.<Long>getField(0).intValue());
}
// validate that the function has been opened properly
assertTrue(open);
count.update(count.value() + 1);
out.collect(
new Tuple4<>(
tuple.<Long>getField(0),
window.getStart(),
window.getEnd(),
new IntType(count.value())));
}
})
.addSink(new CountValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE))
.setParallelism(1);
tryExecute(env, "Tumbling Window Test");
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
@Test
@SuppressWarnings("unchecked")
public void testContinuousWatermarkTrigger() throws Exception {
closeCalled.set(0);
final int WINDOW_SIZE = 3;
TypeInformation<Tuple2<String, Integer>> inputType =
TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc =
new ReducingStateDescriptor<>(
"window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));
WindowOperator<
String,
Tuple2<String, Integer>,
Tuple2<String, Integer>,
Tuple2<String, Integer>,
GlobalWindow>
operator =
new WindowOperator<>(
GlobalWindows.create(),
new GlobalWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(
new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()),
ContinuousEventTimeTrigger.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)));
operator.setInputType(
TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"),
new ExecutionConfig());
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
long initialTime = 0L;
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
// The global window actually ignores these timestamps...
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
// add elements out-of-order
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
testHarness.processWatermark(new Watermark(initialTime + 1000));
expectedOutput.add(new Watermark(1000));
TestHarnessUtil.assertOutputEqualsSorted(
"Output was not correct.",
expectedOutput,
testHarness.getOutput(),
new ResultSortComparator());
testHarness.processWatermark(new Watermark(initialTime + 2000));
expectedOutput.add(new Watermark(2000));
TestHarnessUtil.assertOutputEqualsSorted(
"Output was not correct.",
expectedOutput,
testHarness.getOutput(),
new ResultSortComparator());
testHarness.processWatermark(new Watermark(initialTime + 3000));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE));
expectedOutput.add(new Watermark(3000));
TestHarnessUtil.assertOutputEqualsSorted(
"Output was not correct.",
expectedOutput,
testHarness.getOutput(),
new ResultSortComparator());
testHarness.processWatermark(new Watermark(initialTime + 4000));
expectedOutput.add(new Watermark(4000));
TestHarnessUtil.assertOutputEqualsSorted(
"Output was not correct.",
expectedOutput,
testHarness.getOutput(),
new ResultSortComparator());
testHarness.processWatermark(new Watermark(initialTime + 5000));
expectedOutput.add(new Watermark(5000));
TestHarnessUtil.assertOutputEqualsSorted(
"Output was not correct.",
expectedOutput,
testHarness.getOutput(),
new ResultSortComparator());
testHarness.processWatermark(new Watermark(initialTime + 6000));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 5), Long.MAX_VALUE));
expectedOutput.add(new Watermark(6000));
TestHarnessUtil.assertOutputEqualsSorted(
"Output was not correct.",
expectedOutput,
testHarness.getOutput(),
new ResultSortComparator());
// those don't have any effect...
testHarness.processWatermark(new Watermark(initialTime + 7000));
testHarness.processWatermark(new Watermark(initialTime + 8000));
expectedOutput.add(new Watermark(7000));
expectedOutput.add(new Watermark(8000));
TestHarnessUtil.assertOutputEqualsSorted(
"Output was not correct.",
expectedOutput,
testHarness.getOutput(),
new ResultSortComparator());
testHarness.close();
}
/**
* Creates a new {@code TumblingTimeWindows} {@link WindowAssigner} that assigns elements to time
* windows based on the element timestamp.
*
* @param size The size of the generated windows.
* @return The time policy.
*/
public static TumblingTimeWindows of(Time size) {
return new TumblingTimeWindows(size.toMilliseconds());
}