@Test
public void testMultiDimensionalArray() {
// tuple
TypeInformation<?> ti = TypeInfoParser.parse("Tuple2<Integer, Double>[][]");
Assert.assertEquals(
"ObjectArrayTypeInfo<ObjectArrayTypeInfo<Java Tuple2<Integer, Double>>>", ti.toString());
// pojos
ti =
TypeInfoParser.parse(
"org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyPojo<basic=String>[][][]");
Assert.assertEquals(
"ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<"
+ "PojoType<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyPojo, fields = [basic: String]>"
+ ">>>",
ti.toString());
// basic types
ti = TypeInfoParser.parse("Float[][][]");
Assert.assertEquals(
"ObjectArrayTypeInfo<ObjectArrayTypeInfo<BasicArrayTypeInfo<Float>>>", ti.toString());
ti = TypeInfoParser.parse("String[][][]");
Assert.assertEquals(
"ObjectArrayTypeInfo<ObjectArrayTypeInfo<BasicArrayTypeInfo<String>>>", ti.toString());
ti = TypeInfoParser.parse("Date[][][]");
Assert.assertEquals(
"ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<Date>>>", ti.toString());
// primitive types
ti = TypeInfoParser.parse("int[][][]");
Assert.assertEquals("ObjectArrayTypeInfo<ObjectArrayTypeInfo<int[]>>", ti.toString());
ti = TypeInfoParser.parse("boolean[][][]");
Assert.assertEquals("ObjectArrayTypeInfo<ObjectArrayTypeInfo<boolean[]>>", ti.toString());
// value types
ti = TypeInfoParser.parse("IntValue[][][]");
Assert.assertEquals(
"ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<ValueType<IntValue>>>>",
ti.toString());
// writable types
ti =
TypeInfoParser.parse(
"Writable<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyWritable>[][][]");
Assert.assertEquals(
"ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<"
+ "WritableType<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyWritable>"
+ ">>>",
ti.toString());
// enum types
ti =
TypeInfoParser.parse(
"Enum<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>[][][]");
Assert.assertEquals(
"ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<"
+ "EnumTypeInfo<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>"
+ ">>>",
ti.toString());
}
@Test
public void testException() {
try {
TypeInfoParser.parse("THIS_CLASS_DOES_NOT_EXIST");
Assert.fail("exception expected");
} catch (IllegalArgumentException e) {
// right
}
try {
TypeInfoParser.parse("Tuple2<Integer>");
Assert.fail("exception expected");
} catch (IllegalArgumentException e) {
// right
}
try {
TypeInfoParser.parse("Tuple3<Integer,,>");
Assert.fail("exception expected");
} catch (IllegalArgumentException e) {
// right
}
try {
TypeInfoParser.parse("Tuple1<Integer,Double>");
Assert.fail("exception expected");
} catch (IllegalArgumentException e) {
// right
}
}
@Test
public void testTuples() {
TypeInformation<?> ti = TypeInfoParser.parse("Tuple2<Integer, Long>");
Assert.assertEquals(2, ti.getArity());
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) ti).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) ti).getTypeAt(1));
ti = TypeInfoParser.parse("Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>");
Assert.assertEquals(
"Java Tuple3<Java Tuple1<String>, Java Tuple1<Integer>, Java Tuple2<Long, Long>>",
ti.toString());
}
@Test
public void testEnumType() {
TypeInformation<?> ti =
TypeInfoParser.parse("Enum<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>");
Assert.assertEquals(
"EnumTypeInfo<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>",
ti.toString());
TypeInformation<?> ti2 =
TypeInfoParser.parse(
"java.lang.Enum<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>");
Assert.assertEquals(
"EnumTypeInfo<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>",
ti2.toString());
}
@Test
@SuppressWarnings("unchecked")
public void testTumblingEventTimeWindowsApply() throws Exception {
closeCalled.set(0);
final int WINDOW_SIZE = 3;
TypeInformation<Tuple2<String, Integer>> inputType =
TypeInfoParser.parse("Tuple2<String, Integer>");
ListStateDescriptor<Tuple2<String, Integer>> stateDesc =
new ListStateDescriptor<>(
"window-contents", inputType.createSerializer(new ExecutionConfig()));
WindowOperator<
String,
Tuple2<String, Integer>,
Iterable<Tuple2<String, Integer>>,
Tuple2<String, Integer>,
TimeWindow>
operator =
new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
EventTimeTrigger.create());
operator.setInputType(
TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"),
new ExecutionConfig());
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.open();
testTumblingEventTimeWindows(testHarness);
testHarness.close();
Assert.assertEquals("Close was not called.", 1, closeCalled.get());
}
@Test
public void testWritableType() {
TypeInformation<?> ti =
TypeInfoParser.parse(
"Writable<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyWritable>");
Assert.assertTrue(ti instanceof WritableTypeInfo<?>);
Assert.assertEquals(MyWritable.class, ((WritableTypeInfo<?>) ti).getTypeClass());
}
@Test
public void testObjectArrays() {
TypeInformation<?> ti = TypeInfoParser.parse("java.lang.Class[]");
Assert.assertTrue(ti instanceof ObjectArrayTypeInfo<?, ?>);
Assert.assertEquals(Class.class, ((ObjectArrayTypeInfo<?, ?>) ti).getComponentType());
TypeInformation<?> ti2 = TypeInfoParser.parse("Tuple2<Integer,Double>[]");
Assert.assertTrue(ti2 instanceof ObjectArrayTypeInfo<?, ?>);
Assert.assertTrue(
((ObjectArrayTypeInfo<?, ?>) ti2).getComponentInfo() instanceof TupleTypeInfo);
TypeInformation<?> ti3 = TypeInfoParser.parse("Tuple2<Integer[],Double>[]");
Assert.assertEquals(
"ObjectArrayTypeInfo<Java Tuple2<BasicArrayTypeInfo<Integer>, Double>>", ti3.toString());
}
@Test
public void testLargeMixedTuple() {
TypeInformation<?> ti =
TypeInfoParser.parse(
"org.apache.flink.api.java.tuple.Tuple4<Double,java.lang.Class[],StringValue,Tuple1<int>>[]");
Assert.assertEquals(
"ObjectArrayTypeInfo<Java Tuple4<Double, ObjectArrayTypeInfo<GenericType<java.lang.Class>>, ValueType<StringValue>, Java Tuple1<Integer>>>",
ti.toString());
}
public static void generateLongStringTupleSequence(
StreamExecutionEnvironment env,
String brokerConnection,
String topic,
int numPartitions,
final int from,
final int to)
throws Exception {
TypeInformation<Tuple2<Integer, Integer>> resultType =
TypeInfoParser.parse("Tuple2<Integer, Integer>");
env.setParallelism(numPartitions);
env.getConfig().disableSysoutLogging();
env.setNumberOfExecutionRetries(0);
DataStream<Tuple2<Integer, Integer>> stream =
env.addSource(
new RichParallelSourceFunction<Tuple2<Integer, Integer>>() {
private volatile boolean running = true;
@Override
public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
int cnt = from;
int partition = getRuntimeContext().getIndexOfThisSubtask();
while (running && cnt <= to) {
ctx.collect(new Tuple2<Integer, Integer>(partition, cnt));
cnt++;
}
}
@Override
public void cancel() {
running = false;
}
});
stream.addSink(
new FlinkKafkaProducer<>(
topic,
new TypeInformationSerializationSchema<>(resultType, env.getConfig()),
FlinkKafkaProducer.getPropertiesFromBrokerList(brokerConnection),
new Tuple2Partitioner(numPartitions)));
env.execute("Data generator (Int, Int) stream to topic " + topic);
}
@Test
public void testPojoType2() {
TypeInformation<?> ti =
TypeInfoParser.parse(
"Tuple2<String,Tuple2<Integer,org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyPojo<basic=String>>>");
Assert.assertTrue(ti instanceof TupleTypeInfo);
TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
Assert.assertTrue(tti.getTypeAt(0) instanceof BasicTypeInfo);
Assert.assertTrue(tti.getTypeAt(1) instanceof TupleTypeInfo);
TupleTypeInfo<?> tti2 = (TupleTypeInfo<?>) tti.getTypeAt(1);
Assert.assertTrue(tti2.getTypeAt(0) instanceof BasicTypeInfo);
Assert.assertTrue(tti2.getTypeAt(1) instanceof PojoTypeInfo);
PojoTypeInfo<?> pti = (PojoTypeInfo<?>) tti2.getTypeAt(1);
Assert.assertEquals("basic", pti.getPojoFieldAt(0).field.getName());
Assert.assertTrue(pti.getPojoFieldAt(0).type instanceof BasicTypeInfo);
}
@Test
@SuppressWarnings("unchecked")
public void testSlidingEventTimeWindowsReduce() throws Exception {
closeCalled.set(0);
final int WINDOW_SIZE = 3;
final int WINDOW_SLIDE = 1;
TypeInformation<Tuple2<String, Integer>> inputType =
TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc =
new ReducingStateDescriptor<>(
"window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));
WindowOperator<
String,
Tuple2<String, Integer>,
Tuple2<String, Integer>,
Tuple2<String, Integer>,
TimeWindow>
operator =
new WindowOperator<>(
SlidingEventTimeWindows.of(
Time.of(WINDOW_SIZE, TimeUnit.SECONDS),
Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(
new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
EventTimeTrigger.create());
operator.setInputType(inputType, new ExecutionConfig());
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.open();
testSlidingEventTimeWindows(testHarness);
testHarness.close();
}
@Test
public void testPojoType() {
TypeInformation<?> ti =
TypeInfoParser.parse(
"org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyPojo<"
+ "basic=Integer,"
+ "tuple=Tuple2<String, Integer>,"
+ "hadoopCitizen=Writable<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyWritable>,"
+ "array=String[]"
+ ">");
Assert.assertTrue(ti instanceof PojoTypeInfo);
PojoTypeInfo<?> pti = (PojoTypeInfo<?>) ti;
Assert.assertEquals("array", pti.getPojoFieldAt(0).field.getName());
Assert.assertTrue(pti.getPojoFieldAt(0).type instanceof BasicArrayTypeInfo);
Assert.assertEquals("basic", pti.getPojoFieldAt(1).field.getName());
Assert.assertTrue(pti.getPojoFieldAt(1).type instanceof BasicTypeInfo);
Assert.assertEquals("hadoopCitizen", pti.getPojoFieldAt(2).field.getName());
Assert.assertTrue(pti.getPojoFieldAt(2).type instanceof WritableTypeInfo);
Assert.assertEquals("tuple", pti.getPojoFieldAt(3).field.getName());
Assert.assertTrue(pti.getPojoFieldAt(3).type instanceof TupleTypeInfo);
}
@Test
public void testBasicTypes() {
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, TypeInfoParser.parse("Integer"));
Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, TypeInfoParser.parse("Double"));
Assert.assertEquals(BasicTypeInfo.BYTE_TYPE_INFO, TypeInfoParser.parse("Byte"));
Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, TypeInfoParser.parse("Float"));
Assert.assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, TypeInfoParser.parse("Short"));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, TypeInfoParser.parse("Long"));
Assert.assertEquals(BasicTypeInfo.CHAR_TYPE_INFO, TypeInfoParser.parse("Character"));
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, TypeInfoParser.parse("String"));
Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, TypeInfoParser.parse("Boolean"));
Assert.assertEquals(BasicTypeInfo.VOID_TYPE_INFO, TypeInfoParser.parse("Void"));
Assert.assertEquals(BasicTypeInfo.DATE_TYPE_INFO, TypeInfoParser.parse("Date"));
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, TypeInfoParser.parse("java.lang.Integer"));
Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, TypeInfoParser.parse("java.lang.Double"));
Assert.assertEquals(BasicTypeInfo.BYTE_TYPE_INFO, TypeInfoParser.parse("java.lang.Byte"));
Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, TypeInfoParser.parse("java.lang.Float"));
Assert.assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, TypeInfoParser.parse("java.lang.Short"));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, TypeInfoParser.parse("java.lang.Long"));
Assert.assertEquals(BasicTypeInfo.CHAR_TYPE_INFO, TypeInfoParser.parse("java.lang.Character"));
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, TypeInfoParser.parse("java.lang.String"));
Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, TypeInfoParser.parse("java.lang.Boolean"));
Assert.assertEquals(BasicTypeInfo.VOID_TYPE_INFO, TypeInfoParser.parse("java.lang.Void"));
Assert.assertEquals(BasicTypeInfo.DATE_TYPE_INFO, TypeInfoParser.parse("java.util.Date"));
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, TypeInfoParser.parse("int"));
Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, TypeInfoParser.parse("double"));
Assert.assertEquals(BasicTypeInfo.BYTE_TYPE_INFO, TypeInfoParser.parse("byte"));
Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, TypeInfoParser.parse("float"));
Assert.assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, TypeInfoParser.parse("short"));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, TypeInfoParser.parse("long"));
Assert.assertEquals(BasicTypeInfo.CHAR_TYPE_INFO, TypeInfoParser.parse("char"));
Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, TypeInfoParser.parse("boolean"));
Assert.assertEquals(BasicTypeInfo.VOID_TYPE_INFO, TypeInfoParser.parse("void"));
}
@Test
public void testGenericType() {
TypeInformation<?> ti = TypeInfoParser.parse("java.lang.Class");
Assert.assertTrue(ti instanceof GenericTypeInfo);
Assert.assertEquals(Class.class, ((GenericTypeInfo<?>) ti).getTypeClass());
}
@Test
public void testBasicArrays() {
Assert.assertEquals(BasicArrayTypeInfo.INT_ARRAY_TYPE_INFO, TypeInfoParser.parse("Integer[]"));
Assert.assertEquals(
BasicArrayTypeInfo.DOUBLE_ARRAY_TYPE_INFO, TypeInfoParser.parse("Double[]"));
Assert.assertEquals(BasicArrayTypeInfo.BYTE_ARRAY_TYPE_INFO, TypeInfoParser.parse("Byte[]"));
Assert.assertEquals(BasicArrayTypeInfo.FLOAT_ARRAY_TYPE_INFO, TypeInfoParser.parse("Float[]"));
Assert.assertEquals(BasicArrayTypeInfo.SHORT_ARRAY_TYPE_INFO, TypeInfoParser.parse("Short[]"));
Assert.assertEquals(BasicArrayTypeInfo.LONG_ARRAY_TYPE_INFO, TypeInfoParser.parse("Long[]"));
Assert.assertEquals(
BasicArrayTypeInfo.CHAR_ARRAY_TYPE_INFO, TypeInfoParser.parse("Character[]"));
Assert.assertEquals(
BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, TypeInfoParser.parse("String[]"));
Assert.assertEquals(
BasicArrayTypeInfo.BOOLEAN_ARRAY_TYPE_INFO, TypeInfoParser.parse("Boolean[]"));
Assert.assertEquals(
BasicArrayTypeInfo.INT_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Integer[]"));
Assert.assertEquals(
BasicArrayTypeInfo.DOUBLE_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Double[]"));
Assert.assertEquals(
BasicArrayTypeInfo.BYTE_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Byte[]"));
Assert.assertEquals(
BasicArrayTypeInfo.FLOAT_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Float[]"));
Assert.assertEquals(
BasicArrayTypeInfo.SHORT_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Short[]"));
Assert.assertEquals(
BasicArrayTypeInfo.LONG_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Long[]"));
Assert.assertEquals(
BasicArrayTypeInfo.CHAR_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Character[]"));
Assert.assertEquals(
BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.String[]"));
Assert.assertEquals(
BasicArrayTypeInfo.BOOLEAN_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Boolean[]"));
Assert.assertEquals(
PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("int[]"));
Assert.assertEquals(
PrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("double[]"));
Assert.assertEquals(
PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("byte[]"));
Assert.assertEquals(
PrimitiveArrayTypeInfo.FLOAT_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("float[]"));
Assert.assertEquals(
PrimitiveArrayTypeInfo.SHORT_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("short[]"));
Assert.assertEquals(
PrimitiveArrayTypeInfo.LONG_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("long[]"));
Assert.assertEquals(
PrimitiveArrayTypeInfo.CHAR_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("char[]"));
Assert.assertEquals(
PrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO,
TypeInfoParser.parse("boolean[]"));
}
private static void helperValueType(String str, Class<?> clazz) {
TypeInformation<?> ti = TypeInfoParser.parse(str);
Assert.assertTrue(ti instanceof ValueTypeInfo);
ValueTypeInfo<?> vti = (ValueTypeInfo<?>) ti;
Assert.assertEquals(clazz, vti.getTypeClass());
}
@Test
@SuppressWarnings("unchecked")
public void testCountTrigger() throws Exception {
closeCalled.set(0);
final int WINDOW_SIZE = 4;
TypeInformation<Tuple2<String, Integer>> inputType =
TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc =
new ReducingStateDescriptor<>(
"window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));
WindowOperator<
String,
Tuple2<String, Integer>,
Tuple2<String, Integer>,
Tuple2<String, Integer>,
GlobalWindow>
operator =
new WindowOperator<>(
GlobalWindows.create(),
new GlobalWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(
new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()),
PurgingTrigger.of(CountTrigger.of(WINDOW_SIZE)));
operator.setInputType(
TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"),
new ExecutionConfig());
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
long initialTime = 0L;
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
// The global window actually ignores these timestamps...
// add elements out-of-order
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE));
TestHarnessUtil.assertOutputEqualsSorted(
"Output was not correct.",
expectedOutput,
testHarness.getOutput(),
new ResultSortComparator());
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10999));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), Long.MAX_VALUE));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE));
TestHarnessUtil.assertOutputEqualsSorted(
"Output was not correct.",
expectedOutput,
testHarness.getOutput(),
new ResultSortComparator());
testHarness.close();
}
public class SlidingTimePreReducerTest {
TypeSerializer<Integer> serializer = TypeExtractor.getForObject(1).createSerializer(null);
TypeInformation<Tuple2<Integer, Integer>> tupleType =
TypeInfoParser.parse("Tuple2<Integer,Integer>");
ReduceFunction<Integer> reducer = new SumReducer();
ReduceFunction<Tuple2<Integer, Integer>> tupleReducer = new TupleSumReducer();
@Test
@SuppressWarnings("unchecked")
public void testPreReduce1() throws Exception {
// This ensures that the buffer is properly cleared after a burst of elements by
// replaying the same sequence of elements with a later timestamp and expecting the same
// result.
TestOutput<StreamWindow<Tuple2<Integer, Integer>>> collector =
new TestOutput<StreamWindow<Tuple2<Integer, Integer>>>();
SlidingTimePreReducer<Tuple2<Integer, Integer>> preReducer =
new SlidingTimePreReducer<Tuple2<Integer, Integer>>(
tupleReducer,
tupleType.createSerializer(new ExecutionConfig()),
3,
2,
new TimestampWrapper<Tuple2<Integer, Integer>>(
new Timestamp<Tuple2<Integer, Integer>>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Tuple2<Integer, Integer> value) {
return value.f0;
}
},
1));
int timeOffset = 0;
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 1, 1));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 2, 2));
preReducer.emitWindow(collector);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 3, 3));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 4, 4));
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 5, 5));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 6, 6));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 7, 7));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 8, 8));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 9, 9));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 10, 10));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 11, 11));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 12, 12));
preReducer.emitWindow(collector);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 13, 13));
// ensure that everything is cleared out
preReducer.evict(100);
timeOffset = 25; // a little while later...
// Repeat the same sequence, this should produce the same result
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 1, 1));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 2, 2));
preReducer.emitWindow(collector);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 3, 3));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 4, 4));
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 5, 5));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 6, 6));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 7, 7));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 8, 8));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 9, 9));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 10, 10));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 11, 11));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 12, 12));
preReducer.emitWindow(collector);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 13, 13));
List<StreamWindow<Tuple2<Integer, Integer>>> expected =
new ArrayList<StreamWindow<Tuple2<Integer, Integer>>>();
timeOffset = 0; // rewind ...
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 1, 3)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 2, 9)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 4, 15)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 6, 21)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 8, 27)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 10, 33)));
timeOffset = 25; // and back to the future ...
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 1, 3)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 2, 9)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 4, 15)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 6, 21)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 8, 27)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 10, 33)));
assertEquals(expected, collector.getCollected());
}
@Test
public void testPreReduce2() throws Exception {
TestOutput<StreamWindow<Integer>> collector = new TestOutput<StreamWindow<Integer>>();
SlidingTimePreReducer<Integer> preReducer =
new SlidingTimePreReducer<Integer>(
reducer,
serializer,
5,
2,
new TimestampWrapper<Integer>(
new Timestamp<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Integer value) {
return value;
}
},
1));
preReducer.store(1);
preReducer.store(2);
preReducer.emitWindow(collector);
preReducer.store(3);
preReducer.store(4);
preReducer.emitWindow(collector);
preReducer.store(5);
preReducer.store(6);
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(7);
preReducer.store(8);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(9);
preReducer.store(10);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(11);
preReducer.store(12);
preReducer.emitWindow(collector);
preReducer.store(13);
List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
expected.add(StreamWindow.fromElements(3));
expected.add(StreamWindow.fromElements(10));
expected.add(StreamWindow.fromElements(20));
expected.add(StreamWindow.fromElements(30));
expected.add(StreamWindow.fromElements(40));
expected.add(StreamWindow.fromElements(50));
assertEquals(expected, collector.getCollected());
}
@Test
public void testPreReduce3() throws Exception {
TestOutput<StreamWindow<Integer>> collector = new TestOutput<StreamWindow<Integer>>();
SlidingTimePreReducer<Integer> preReducer =
new SlidingTimePreReducer<Integer>(
reducer,
serializer,
6,
3,
new TimestampWrapper<Integer>(
new Timestamp<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Integer value) {
return value;
}
},
1));
preReducer.store(1);
preReducer.store(2);
preReducer.store(3);
preReducer.emitWindow(collector);
preReducer.store(4);
preReducer.store(5);
preReducer.store(6);
preReducer.emitWindow(collector);
preReducer.evict(3);
preReducer.store(7);
preReducer.store(8);
preReducer.store(9);
preReducer.emitWindow(collector);
preReducer.evict(3);
preReducer.store(10);
preReducer.store(11);
preReducer.store(12);
preReducer.emitWindow(collector);
preReducer.evict(3);
preReducer.store(13);
List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
expected.add(StreamWindow.fromElements(6));
expected.add(StreamWindow.fromElements(21));
expected.add(StreamWindow.fromElements(39));
expected.add(StreamWindow.fromElements(57));
assertEquals(expected, collector.getCollected());
}
@Test
public void testPreReduce4() throws Exception {
TestOutput<StreamWindow<Integer>> collector = new TestOutput<StreamWindow<Integer>>();
SlidingTimePreReducer<Integer> preReducer =
new SlidingTimePreReducer<Integer>(
reducer,
serializer,
3,
2,
new TimestampWrapper<Integer>(
new Timestamp<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Integer value) {
return value;
}
},
1));
preReducer.store(1);
preReducer.store(2);
preReducer.emitWindow(collector);
preReducer.store(3);
preReducer.store(4);
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(5);
preReducer.store(6);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(7);
preReducer.store(8);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.emitWindow(collector);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(14);
preReducer.emitWindow(collector);
preReducer.emitWindow(collector);
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.emitWindow(collector);
preReducer.store(21);
preReducer.emitWindow(collector);
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.store(9);
List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
expected.add(StreamWindow.fromElements(3));
expected.add(StreamWindow.fromElements(9));
expected.add(StreamWindow.fromElements(15));
expected.add(StreamWindow.fromElements(21));
expected.add(StreamWindow.fromElements(8));
expected.add(StreamWindow.fromElements(8));
expected.add(StreamWindow.fromElements(14));
expected.add(StreamWindow.fromElements(14));
expected.add(StreamWindow.fromElements(21));
assertEquals(expected, collector.getCollected());
}
private static class SumReducer implements ReduceFunction<Integer> {
private static final long serialVersionUID = 1L;
@Override
public Integer reduce(Integer value1, Integer value2) throws Exception {
return value1 + value2;
}
}
private static class TupleSumReducer implements ReduceFunction<Tuple2<Integer, Integer>> {
private static final long serialVersionUID = 1L;
@Override
public Tuple2<Integer, Integer> reduce(
Tuple2<Integer, Integer> value1, Tuple2<Integer, Integer> value2) throws Exception {
return new Tuple2<Integer, Integer>(value1.f0, value1.f1 + value2.f1);
}
}
}
