@Test
public void testChannelSelectors() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Long> src = env.generateSequence(0, 0);
DataStream<Long> broadcast = src.broadcast();
DataStreamSink<Long> broadcastSink = broadcast.print();
StreamPartitioner<?> broadcastPartitioner =
env.getStreamGraph()
.getStreamEdges(src.getId(), broadcastSink.getTransformation().getId())
.get(0)
.getPartitioner();
assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);
DataStream<Long> shuffle = src.shuffle();
DataStreamSink<Long> shuffleSink = shuffle.print();
StreamPartitioner<?> shufflePartitioner =
env.getStreamGraph()
.getStreamEdges(src.getId(), shuffleSink.getTransformation().getId())
.get(0)
.getPartitioner();
assertTrue(shufflePartitioner instanceof ShufflePartitioner);
DataStream<Long> forward = src.forward();
DataStreamSink<Long> forwardSink = forward.print();
StreamPartitioner<?> forwardPartitioner =
env.getStreamGraph()
.getStreamEdges(src.getId(), forwardSink.getTransformation().getId())
.get(0)
.getPartitioner();
assertTrue(forwardPartitioner instanceof ForwardPartitioner);
DataStream<Long> rebalance = src.rebalance();
DataStreamSink<Long> rebalanceSink = rebalance.print();
StreamPartitioner<?> rebalancePartitioner =
env.getStreamGraph()
.getStreamEdges(src.getId(), rebalanceSink.getTransformation().getId())
.get(0)
.getPartitioner();
assertTrue(rebalancePartitioner instanceof RebalancePartitioner);
DataStream<Long> global = src.global();
DataStreamSink<Long> globalSink = global.print();
StreamPartitioner<?> globalPartitioner =
env.getStreamGraph()
.getStreamEdges(src.getId(), globalSink.getTransformation().getId())
.get(0)
.getPartitioner();
assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
/** Tests whether parallelism gets set. */
@Test
public void testParallelism() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Tuple2<Long, Long>> src = env.fromElements(new Tuple2<>(0L, 0L));
env.setParallelism(10);
SingleOutputStreamOperator<Long, ?> map =
src.map(
new MapFunction<Tuple2<Long, Long>, Long>() {
@Override
public Long map(Tuple2<Long, Long> value) throws Exception {
return null;
}
})
.name("MyMap");
DataStream<Long> windowed =
map.windowAll(GlobalWindows.create())
.trigger(PurgingTrigger.of(CountTrigger.of(10)))
.fold(
0L,
new FoldFunction<Long, Long>() {
@Override
public Long fold(Long accumulator, Long value) throws Exception {
return null;
}
});
windowed.addSink(new NoOpSink<Long>());
DataStreamSink<Long> sink =
map.addSink(
new SinkFunction<Long>() {
private static final long serialVersionUID = 1L;
@Override
public void invoke(Long value) throws Exception {}
});
assertEquals(1, env.getStreamGraph().getStreamNode(src.getId()).getParallelism());
assertEquals(10, env.getStreamGraph().getStreamNode(map.getId()).getParallelism());
assertEquals(1, env.getStreamGraph().getStreamNode(windowed.getId()).getParallelism());
assertEquals(
10, env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getParallelism());
env.setParallelism(7);
// Some parts, such as windowing rely on the fact that previous operators have a parallelism
// set when instantiating the Discretizer. This would break if we dynamically changed
// the parallelism of operations when changing the setting on the Execution Environment.
assertEquals(1, env.getStreamGraph().getStreamNode(src.getId()).getParallelism());
assertEquals(10, env.getStreamGraph().getStreamNode(map.getId()).getParallelism());
assertEquals(1, env.getStreamGraph().getStreamNode(windowed.getId()).getParallelism());
assertEquals(
10, env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getParallelism());
try {
src.setParallelism(3);
fail();
} catch (IllegalArgumentException success) {
// do nothing
}
DataStreamSource<Long> parallelSource = env.generateSequence(0, 0);
parallelSource.addSink(new NoOpSink<Long>());
assertEquals(7, env.getStreamGraph().getStreamNode(parallelSource.getId()).getParallelism());
parallelSource.setParallelism(3);
assertEquals(3, env.getStreamGraph().getStreamNode(parallelSource.getId()).getParallelism());
map.setParallelism(2);
assertEquals(2, env.getStreamGraph().getStreamNode(map.getId()).getParallelism());
sink.setParallelism(4);
assertEquals(
4, env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getParallelism());
}