public static void main(final String[] args) throws Exception {
if (!parseParameters(args)) {
return;
}
// set up the execution environment
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// get input data
final DataStream<String> text = getTextDataStream(env);
final DataStream<Tuple2<String, Integer>> counts =
text
// split up the lines in pairs (2-tuples) containing: (word,1)
// this is done by a bolt that is wrapped accordingly
.transform(
"BoltTokenizer",
TypeExtractor.getForObject(new Tuple2<String, Integer>("", 0)),
new BoltWrapper<String, Tuple2<String, Integer>>(new BoltTokenizer()))
// group by the tuple field "0" and sum up tuple field "1"
.keyBy(0)
.sum(1);
// emit result
if (fileOutput) {
counts.writeAsText(outputPath);
} else {
counts.print();
}
// execute program
env.execute("Streaming WordCount with bolt tokenizer");
}
@Test
public void testAllReduceDriverImmutableEmpty() {
try {
TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
context.setInput1(input, typeInfo.createSerializer());
context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
AllReduceDriver<Tuple2<String, Integer>> driver =
new AllReduceDriver<Tuple2<String, Integer>>();
driver.setup(context);
driver.prepare();
driver.run();
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
private TypeSerializer<Object> createSerializer(Object key, int pos) {
if (key == null) {
throw new NullKeyFieldException(pos);
}
try {
TypeInformation<Object> info = TypeExtractor.getForObject(key);
return info.createSerializer(executionConfig);
} catch (Throwable t) {
throw new RuntimeException("Could not create key serializer for type " + key);
}
}
@Test
public void testTypeInfo() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<Long> src1 = env.generateSequence(0, 0);
assertEquals(TypeExtractor.getForClass(Long.class), src1.getType());
DataStream<Tuple2<Integer, String>> map =
src1.map(
new MapFunction<Long, Tuple2<Integer, String>>() {
@Override
public Tuple2<Integer, String> map(Long value) throws Exception {
return null;
}
});
assertEquals(TypeExtractor.getForObject(new Tuple2<>(0, "")), map.getType());
DataStream<String> window =
map.windowAll(GlobalWindows.create())
.trigger(PurgingTrigger.of(CountTrigger.of(5)))
.apply(
new AllWindowFunction<Tuple2<Integer, String>, String, GlobalWindow>() {
@Override
public void apply(
GlobalWindow window,
Iterable<Tuple2<Integer, String>> values,
Collector<String> out)
throws Exception {}
});
assertEquals(TypeExtractor.getForClass(String.class), window.getType());
DataStream<CustomPOJO> flatten =
window
.windowAll(GlobalWindows.create())
.trigger(PurgingTrigger.of(CountTrigger.of(5)))
.fold(
new CustomPOJO(),
new FoldFunction<String, CustomPOJO>() {
private static final long serialVersionUID = 1L;
@Override
public CustomPOJO fold(CustomPOJO accumulator, String value) throws Exception {
return null;
}
});
assertEquals(TypeExtractor.getForClass(CustomPOJO.class), flatten.getType());
}
private static DataStream<Tuple1<String>> getTextDataStream(
final StreamExecutionEnvironment env) {
if (fileOutput) {
// read the text file from given input path
TupleTypeInfo<Tuple1<String>> sourceType =
(TupleTypeInfo<Tuple1<String>>) TypeExtractor.getForObject(new Tuple1<String>(""));
return env.createInput(
new CsvInputFormat<Tuple1<String>>(
new Path(textPath),
CsvInputFormat.DEFAULT_LINE_DELIMITER,
CsvInputFormat.DEFAULT_LINE_DELIMITER,
sourceType),
sourceType);
}
return env.fromElements(WordCountDataTuple.TUPLES);
}
@Test
public void testTypeInfo() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<Long> src1 = env.generateSequence(0, 0);
assertEquals(TypeExtractor.getForClass(Long.class), src1.getType());
DataStream<Tuple2<Integer, String>> map =
src1.map(
new MapFunction<Long, Tuple2<Integer, String>>() {
@Override
public Tuple2<Integer, String> map(Long value) throws Exception {
return null;
}
});
assertEquals(TypeExtractor.getForObject(new Tuple2<Integer, String>(0, "")), map.getType());
WindowedDataStream<String> window =
map.window(Count.of(5))
.mapWindow(
new WindowMapFunction<Tuple2<Integer, String>, String>() {
@Override
public void mapWindow(
Iterable<Tuple2<Integer, String>> values, Collector<String> out)
throws Exception {}
});
assertEquals(TypeExtractor.getForClass(String.class), window.getType());
DataStream<CustomPOJO> flatten =
window
.foldWindow(
new CustomPOJO(),
new FoldFunction<String, CustomPOJO>() {
@Override
public CustomPOJO fold(CustomPOJO accumulator, String value) throws Exception {
return null;
}
})
.flatten();
assertEquals(TypeExtractor.getForClass(CustomPOJO.class), flatten.getType());
}
@Test
public void testImmutableEmpty() {
try {
TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>(
1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
Collections.shuffle(data);
TupleTypeInfo<Tuple2<String, Integer>> typeInfo =
(TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
TypeComparator<Tuple2<String, Integer>> comparator =
typeInfo.createComparator(new int[] {0}, new boolean[] {true}, 0, new ExecutionConfig());
GatheringCollector<Tuple2<String, Integer>> result =
new GatheringCollector<Tuple2<String, Integer>>(
typeInfo.createSerializer(new ExecutionConfig()));
context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
context.setComparator1(comparator);
context.setCollector(result);
ReduceCombineDriver<Tuple2<String, Integer>> driver =
new ReduceCombineDriver<Tuple2<String, Integer>>();
driver.setup(context);
driver.prepare();
driver.run();
Assert.assertEquals(0, result.getList().size());
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testReduceDriverMutable() {
try {
{
TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>
context =
new TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(
1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo =
(TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input =
new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(
data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
TypeComparator<Tuple2<StringValue, IntValue>> comparator =
typeInfo.createComparator(
new int[] {0}, new boolean[] {true}, 0, new ExecutionConfig());
GatheringCollector<Tuple2<StringValue, IntValue>> result =
new GatheringCollector<Tuple2<StringValue, IntValue>>(
typeInfo.createSerializer(new ExecutionConfig()));
context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
context.setComparator1(comparator);
context.setCollector(result);
context.setUdf(new ConcatSumFirstMutableReducer());
ReduceCombineDriver<Tuple2<StringValue, IntValue>> driver =
new ReduceCombineDriver<Tuple2<StringValue, IntValue>>();
driver.setup(context);
driver.prepare();
driver.run();
Object[] res = result.getList().toArray();
Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
DriverTestData.compareTupleArrays(expected, res);
}
{
TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>
context =
new TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(
1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo =
(TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input =
new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(
data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
TypeComparator<Tuple2<StringValue, IntValue>> comparator =
typeInfo.createComparator(
new int[] {0}, new boolean[] {true}, 0, new ExecutionConfig());
GatheringCollector<Tuple2<StringValue, IntValue>> result =
new GatheringCollector<Tuple2<StringValue, IntValue>>(
typeInfo.createSerializer(new ExecutionConfig()));
context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
context.setComparator1(comparator);
context.setCollector(result);
context.setUdf(new ConcatSumSecondMutableReducer());
ReduceCombineDriver<Tuple2<StringValue, IntValue>> driver =
new ReduceCombineDriver<Tuple2<StringValue, IntValue>>();
driver.setup(context);
driver.prepare();
driver.run();
Object[] res = result.getList().toArray();
Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
DriverTestData.compareTupleArrays(expected, res);
}
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
public class TumblingGroupedPreReducerTest {
TypeInformation<Tuple2<Integer, Integer>> type =
TypeExtractor.getForObject(new Tuple2<Integer, Integer>(1, 1));
TypeSerializer<Tuple2<Integer, Integer>> serializer = type.createSerializer(null);
KeySelector<Tuple2<Integer, Integer>, ?> key =
KeySelectorUtil.getSelectorForKeys(
new Keys.ExpressionKeys<Tuple2<Integer, Integer>>(new int[] {0}, type), type, null);
Reducer reducer = new Reducer();
@SuppressWarnings("unchecked")
@Test
public void testEmitWindow() throws Exception {
List<Tuple2<Integer, Integer>> inputs = new ArrayList<Tuple2<Integer, Integer>>();
inputs.add(new Tuple2<Integer, Integer>(1, 1));
inputs.add(new Tuple2<Integer, Integer>(0, 0));
inputs.add(new Tuple2<Integer, Integer>(1, -1));
inputs.add(new Tuple2<Integer, Integer>(1, -2));
TestCollector<StreamWindow<Tuple2<Integer, Integer>>> collector =
new TestCollector<StreamWindow<Tuple2<Integer, Integer>>>();
List<StreamWindow<Tuple2<Integer, Integer>>> collected = collector.getCollected();
WindowBuffer<Tuple2<Integer, Integer>> wb =
new TumblingGroupedPreReducer<Tuple2<Integer, Integer>>(reducer, key, serializer);
wb.store(serializer.copy(inputs.get(0)));
wb.store(serializer.copy(inputs.get(1)));
wb.emitWindow(collector);
wb.evict(2);
assertEquals(1, collected.size());
assertSetEquals(
StreamWindow.fromElements(
new Tuple2<Integer, Integer>(1, 1), new Tuple2<Integer, Integer>(0, 0)),
collected.get(0));
wb.store(serializer.copy(inputs.get(0)));
wb.store(serializer.copy(inputs.get(1)));
wb.store(serializer.copy(inputs.get(2)));
wb.store(serializer.copy(inputs.get(3)));
wb.emitWindow(collector);
wb.evict(4);
assertEquals(2, collected.size());
assertSetEquals(
StreamWindow.fromElements(
new Tuple2<Integer, Integer>(3, -2), new Tuple2<Integer, Integer>(0, 0)),
collected.get(1));
// Test whether function is mutating inputs or not
assertEquals(2, reducer.allInputs.size());
assertEquals(reducer.allInputs.get(0), inputs.get(2));
assertEquals(reducer.allInputs.get(1), inputs.get(3));
}
@SuppressWarnings("unchecked")
@Test
public void testEmitWindow2() throws Exception {
List<Tuple2<Integer, Integer>> inputs = new ArrayList<Tuple2<Integer, Integer>>();
inputs.add(new Tuple2<Integer, Integer>(1, 1));
inputs.add(new Tuple2<Integer, Integer>(0, 0));
inputs.add(new Tuple2<Integer, Integer>(1, -1));
inputs.add(new Tuple2<Integer, Integer>(1, -2));
TestCollector<StreamWindow<Tuple2<Integer, Integer>>> collector =
new TestCollector<StreamWindow<Tuple2<Integer, Integer>>>();
List<StreamWindow<Tuple2<Integer, Integer>>> collected = collector.getCollected();
WindowBuffer<Tuple2<Integer, Integer>> wb =
new TumblingGroupedPreReducer<Tuple2<Integer, Integer>>(reducer, key, serializer)
.sequentialID();
wb.store(serializer.copy(inputs.get(0)));
wb.store(serializer.copy(inputs.get(1)));
wb.emitWindow(collector);
wb.evict(2);
assertSetEquals(StreamWindow.fromElements(inputs.get(0), inputs.get(1)), collected.get(0));
wb.store(serializer.copy(inputs.get(0)));
wb.store(serializer.copy(inputs.get(1)));
wb.store(serializer.copy(inputs.get(2)));
wb.emitWindow(collector);
wb.evict(3);
assertSetEquals(
StreamWindow.fromElements(new Tuple2<Integer, Integer>(2, 0), inputs.get(1)),
collected.get(1));
}
private static <T> void assertSetEquals(Collection<T> first, Collection<T> second) {
assertEquals(new HashSet<T>(first), new HashSet<T>(second));
}
@SuppressWarnings("serial")
private class Reducer implements ReduceFunction<Tuple2<Integer, Integer>> {
public List<Tuple2<Integer, Integer>> allInputs = new ArrayList<Tuple2<Integer, Integer>>();
@Override
public Tuple2<Integer, Integer> reduce(
Tuple2<Integer, Integer> value1, Tuple2<Integer, Integer> value2) throws Exception {
allInputs.add(value2);
value1.f0 = value1.f0 + value2.f0;
value1.f1 = value1.f1 + value2.f1;
return value1;
}
}
}
public class SlidingTimePreReducerTest {
TypeSerializer<Integer> serializer = TypeExtractor.getForObject(1).createSerializer(null);
TypeInformation<Tuple2<Integer, Integer>> tupleType =
TypeInfoParser.parse("Tuple2<Integer,Integer>");
ReduceFunction<Integer> reducer = new SumReducer();
ReduceFunction<Tuple2<Integer, Integer>> tupleReducer = new TupleSumReducer();
@Test
@SuppressWarnings("unchecked")
public void testPreReduce1() throws Exception {
// This ensures that the buffer is properly cleared after a burst of elements by
// replaying the same sequence of elements with a later timestamp and expecting the same
// result.
TestOutput<StreamWindow<Tuple2<Integer, Integer>>> collector =
new TestOutput<StreamWindow<Tuple2<Integer, Integer>>>();
SlidingTimePreReducer<Tuple2<Integer, Integer>> preReducer =
new SlidingTimePreReducer<Tuple2<Integer, Integer>>(
tupleReducer,
tupleType.createSerializer(new ExecutionConfig()),
3,
2,
new TimestampWrapper<Tuple2<Integer, Integer>>(
new Timestamp<Tuple2<Integer, Integer>>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Tuple2<Integer, Integer> value) {
return value.f0;
}
},
1));
int timeOffset = 0;
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 1, 1));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 2, 2));
preReducer.emitWindow(collector);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 3, 3));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 4, 4));
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 5, 5));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 6, 6));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 7, 7));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 8, 8));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 9, 9));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 10, 10));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 11, 11));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 12, 12));
preReducer.emitWindow(collector);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 13, 13));
// ensure that everything is cleared out
preReducer.evict(100);
timeOffset = 25; // a little while later...
// Repeat the same sequence, this should produce the same result
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 1, 1));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 2, 2));
preReducer.emitWindow(collector);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 3, 3));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 4, 4));
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 5, 5));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 6, 6));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 7, 7));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 8, 8));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 9, 9));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 10, 10));
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 11, 11));
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 12, 12));
preReducer.emitWindow(collector);
preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 13, 13));
List<StreamWindow<Tuple2<Integer, Integer>>> expected =
new ArrayList<StreamWindow<Tuple2<Integer, Integer>>>();
timeOffset = 0; // rewind ...
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 1, 3)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 2, 9)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 4, 15)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 6, 21)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 8, 27)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 10, 33)));
timeOffset = 25; // and back to the future ...
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 1, 3)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 2, 9)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 4, 15)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 6, 21)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 8, 27)));
expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 10, 33)));
assertEquals(expected, collector.getCollected());
}
@Test
public void testPreReduce2() throws Exception {
TestOutput<StreamWindow<Integer>> collector = new TestOutput<StreamWindow<Integer>>();
SlidingTimePreReducer<Integer> preReducer =
new SlidingTimePreReducer<Integer>(
reducer,
serializer,
5,
2,
new TimestampWrapper<Integer>(
new Timestamp<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Integer value) {
return value;
}
},
1));
preReducer.store(1);
preReducer.store(2);
preReducer.emitWindow(collector);
preReducer.store(3);
preReducer.store(4);
preReducer.emitWindow(collector);
preReducer.store(5);
preReducer.store(6);
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(7);
preReducer.store(8);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(9);
preReducer.store(10);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(11);
preReducer.store(12);
preReducer.emitWindow(collector);
preReducer.store(13);
List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
expected.add(StreamWindow.fromElements(3));
expected.add(StreamWindow.fromElements(10));
expected.add(StreamWindow.fromElements(20));
expected.add(StreamWindow.fromElements(30));
expected.add(StreamWindow.fromElements(40));
expected.add(StreamWindow.fromElements(50));
assertEquals(expected, collector.getCollected());
}
@Test
public void testPreReduce3() throws Exception {
TestOutput<StreamWindow<Integer>> collector = new TestOutput<StreamWindow<Integer>>();
SlidingTimePreReducer<Integer> preReducer =
new SlidingTimePreReducer<Integer>(
reducer,
serializer,
6,
3,
new TimestampWrapper<Integer>(
new Timestamp<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Integer value) {
return value;
}
},
1));
preReducer.store(1);
preReducer.store(2);
preReducer.store(3);
preReducer.emitWindow(collector);
preReducer.store(4);
preReducer.store(5);
preReducer.store(6);
preReducer.emitWindow(collector);
preReducer.evict(3);
preReducer.store(7);
preReducer.store(8);
preReducer.store(9);
preReducer.emitWindow(collector);
preReducer.evict(3);
preReducer.store(10);
preReducer.store(11);
preReducer.store(12);
preReducer.emitWindow(collector);
preReducer.evict(3);
preReducer.store(13);
List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
expected.add(StreamWindow.fromElements(6));
expected.add(StreamWindow.fromElements(21));
expected.add(StreamWindow.fromElements(39));
expected.add(StreamWindow.fromElements(57));
assertEquals(expected, collector.getCollected());
}
@Test
public void testPreReduce4() throws Exception {
TestOutput<StreamWindow<Integer>> collector = new TestOutput<StreamWindow<Integer>>();
SlidingTimePreReducer<Integer> preReducer =
new SlidingTimePreReducer<Integer>(
reducer,
serializer,
3,
2,
new TimestampWrapper<Integer>(
new Timestamp<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Integer value) {
return value;
}
},
1));
preReducer.store(1);
preReducer.store(2);
preReducer.emitWindow(collector);
preReducer.store(3);
preReducer.store(4);
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(5);
preReducer.store(6);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(7);
preReducer.store(8);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.emitWindow(collector);
preReducer.emitWindow(collector);
preReducer.evict(2);
preReducer.store(14);
preReducer.emitWindow(collector);
preReducer.emitWindow(collector);
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.emitWindow(collector);
preReducer.store(21);
preReducer.emitWindow(collector);
preReducer.evict(1);
preReducer.emitWindow(collector);
preReducer.store(9);
List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
expected.add(StreamWindow.fromElements(3));
expected.add(StreamWindow.fromElements(9));
expected.add(StreamWindow.fromElements(15));
expected.add(StreamWindow.fromElements(21));
expected.add(StreamWindow.fromElements(8));
expected.add(StreamWindow.fromElements(8));
expected.add(StreamWindow.fromElements(14));
expected.add(StreamWindow.fromElements(14));
expected.add(StreamWindow.fromElements(21));
assertEquals(expected, collector.getCollected());
}
private static class SumReducer implements ReduceFunction<Integer> {
private static final long serialVersionUID = 1L;
@Override
public Integer reduce(Integer value1, Integer value2) throws Exception {
return value1 + value2;
}
}
private static class TupleSumReducer implements ReduceFunction<Tuple2<Integer, Integer>> {
private static final long serialVersionUID = 1L;
@Override
public Tuple2<Integer, Integer> reduce(
Tuple2<Integer, Integer> value1, Tuple2<Integer, Integer> value2) throws Exception {
return new Tuple2<Integer, Integer>(value1.f0, value1.f1 + value2.f1);
}
}
}
@Test
public void testAllReduceDriverImmutable() {
try {
{
TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input =
new RegularToMutableObjectIterator<Tuple2<String, Integer>>(
data.iterator(), typeInfo.createSerializer());
GatheringCollector<Tuple2<String, Integer>> result =
new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer());
context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
context.setInput1(input, typeInfo.createSerializer());
context.setCollector(result);
context.setUdf(new ConcatSumFirstReducer());
AllReduceDriver<Tuple2<String, Integer>> driver =
new AllReduceDriver<Tuple2<String, Integer>>();
driver.setup(context);
driver.prepare();
driver.run();
Tuple2<String, Integer> res = result.getList().get(0);
char[] foundString = res.f0.toCharArray();
Arrays.sort(foundString);
char[] expectedString = "abcddeeeffff".toCharArray();
Arrays.sort(expectedString);
Assert.assertArrayEquals(expectedString, foundString);
Assert.assertEquals(78, res.f1.intValue());
}
{
TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input =
new RegularToMutableObjectIterator<Tuple2<String, Integer>>(
data.iterator(), typeInfo.createSerializer());
GatheringCollector<Tuple2<String, Integer>> result =
new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer());
context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
context.setInput1(input, typeInfo.createSerializer());
context.setCollector(result);
context.setUdf(new ConcatSumSecondReducer());
AllReduceDriver<Tuple2<String, Integer>> driver =
new AllReduceDriver<Tuple2<String, Integer>>();
driver.setup(context);
driver.prepare();
driver.run();
Tuple2<String, Integer> res = result.getList().get(0);
char[] foundString = res.f0.toCharArray();
Arrays.sort(foundString);
char[] expectedString = "abcddeeeffff".toCharArray();
Arrays.sort(expectedString);
Assert.assertArrayEquals(expectedString, foundString);
Assert.assertEquals(78, res.f1.intValue());
}
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
public class JumpingTimePreReducerTest {
TypeSerializer<Integer> serializer = TypeExtractor.getForObject(1).createSerializer(null);
ReduceFunction<Integer> reducer = new SumReducer();
@Test
public void testEmitWindow() throws Exception {
TestCollector<StreamWindow<Integer>> collector = new TestCollector<StreamWindow<Integer>>();
List<StreamWindow<Integer>> collected = collector.getCollected();
WindowBuffer<Integer> wb =
new JumpingTimePreReducer<Integer>(
reducer,
serializer,
3,
2,
new TimestampWrapper<Integer>(
new Timestamp<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Integer value) {
return value;
}
},
1));
wb.store(1);
wb.store(2);
wb.store(3);
wb.evict(1);
wb.emitWindow(collector);
assertEquals(1, collected.size());
assertEquals(StreamWindow.fromElements(5), collected.get(0));
wb.store(4);
wb.store(5);
// Nothing should happen here
wb.evict(2);
wb.store(6);
wb.emitWindow(collector);
wb.evict(2);
wb.emitWindow(collector);
wb.store(12);
wb.emitWindow(collector);
assertEquals(3, collected.size());
assertEquals(StreamWindow.fromElements(11), collected.get(1));
assertEquals(StreamWindow.fromElements(12), collected.get(2));
}
private static class SumReducer implements ReduceFunction<Integer> {
private static final long serialVersionUID = 1L;
@Override
public Integer reduce(Integer value1, Integer value2) throws Exception {
return value1 + value2;
}
}
}
