private static boolean isCustomPartitioned(List<StreamEdge> edges) {
boolean result = true;
for (StreamEdge edge : edges) {
if (!(edge.getPartitioner() instanceof CustomPartitionerWrapper)) {
result = false;
}
}
return result;
}
private static boolean isCustomPartitioned(StreamEdge edge) {
return edge.getPartitioner() instanceof CustomPartitionerWrapper;
}
private static boolean isPartitioned(StreamEdge edge) {
return edge.getPartitioner() instanceof HashPartitioner;
}
@Test
public void operatorTest() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Long> src = env.generateSequence(0, 0);
MapFunction<Long, Integer> mapFunction =
new MapFunction<Long, Integer>() {
@Override
public Integer map(Long value) throws Exception {
return null;
}
};
DataStream<Integer> map = src.map(mapFunction);
map.addSink(new NoOpSink<Integer>());
assertEquals(mapFunction, getFunctionForDataStream(map));
FlatMapFunction<Long, Integer> flatMapFunction =
new FlatMapFunction<Long, Integer>() {
private static final long serialVersionUID = 1L;
@Override
public void flatMap(Long value, Collector<Integer> out) throws Exception {}
};
DataStream<Integer> flatMap = src.flatMap(flatMapFunction);
flatMap.addSink(new NoOpSink<Integer>());
assertEquals(flatMapFunction, getFunctionForDataStream(flatMap));
FilterFunction<Integer> filterFunction =
new FilterFunction<Integer>() {
@Override
public boolean filter(Integer value) throws Exception {
return false;
}
};
DataStream<Integer> unionFilter = map.union(flatMap).filter(filterFunction);
unionFilter.addSink(new NoOpSink<Integer>());
assertEquals(filterFunction, getFunctionForDataStream(unionFilter));
try {
env.getStreamGraph().getStreamEdge(map.getId(), unionFilter.getId());
} catch (RuntimeException e) {
fail(e.getMessage());
}
try {
env.getStreamGraph().getStreamEdge(flatMap.getId(), unionFilter.getId());
} catch (RuntimeException e) {
fail(e.getMessage());
}
OutputSelector<Integer> outputSelector =
new OutputSelector<Integer>() {
@Override
public Iterable<String> select(Integer value) {
return null;
}
};
SplitStream<Integer> split = unionFilter.split(outputSelector);
split.select("dummy").addSink(new NoOpSink<Integer>());
List<OutputSelector<?>> outputSelectors =
env.getStreamGraph().getStreamNode(unionFilter.getId()).getOutputSelectors();
assertEquals(1, outputSelectors.size());
assertEquals(outputSelector, outputSelectors.get(0));
DataStream<Integer> select = split.select("a");
DataStreamSink<Integer> sink = select.print();
StreamEdge splitEdge =
env.getStreamGraph().getStreamEdge(unionFilter.getId(), sink.getTransformation().getId());
assertEquals("a", splitEdge.getSelectedNames().get(0));
ConnectedStreams<Integer, Integer> connect = map.connect(flatMap);
CoMapFunction<Integer, Integer, String> coMapper =
new CoMapFunction<Integer, Integer, String>() {
private static final long serialVersionUID = 1L;
@Override
public String map1(Integer value) {
return null;
}
@Override
public String map2(Integer value) {
return null;
}
};
DataStream<String> coMap = connect.map(coMapper);
coMap.addSink(new NoOpSink<String>());
assertEquals(coMapper, getFunctionForDataStream(coMap));
try {
env.getStreamGraph().getStreamEdge(map.getId(), coMap.getId());
} catch (RuntimeException e) {
fail(e.getMessage());
}
try {
env.getStreamGraph().getStreamEdge(flatMap.getId(), coMap.getId());
} catch (RuntimeException e) {
fail(e.getMessage());
}
}
/**
* Tests union functionality. This ensures that self-unions and unions of streams with differing
* parallelism work.
*
* @throws Exception
*/
@Test
public void testUnion() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(4);
DataStream<Long> input1 =
env.generateSequence(0, 0)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
});
DataStream<Long> selfUnion =
input1
.union(input1)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
});
DataStream<Long> input6 =
env.generateSequence(0, 0)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
});
DataStream<Long> selfUnionDifferentPartition =
input6
.broadcast()
.union(input6)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
});
DataStream<Long> input2 =
env.generateSequence(0, 0)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
})
.setParallelism(4);
DataStream<Long> input3 =
env.generateSequence(0, 0)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
})
.setParallelism(2);
DataStream<Long> unionDifferingParallelism =
input2
.union(input3)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
})
.setParallelism(4);
DataStream<Long> input4 =
env.generateSequence(0, 0)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
})
.setParallelism(2);
DataStream<Long> input5 =
env.generateSequence(0, 0)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
})
.setParallelism(4);
DataStream<Long> unionDifferingPartitioning =
input4
.broadcast()
.union(input5)
.map(
new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
return null;
}
})
.setParallelism(4);
StreamGraph streamGraph = env.getStreamGraph();
// verify self union
assertTrue(streamGraph.getStreamNode(selfUnion.getId()).getInEdges().size() == 2);
for (StreamEdge edge : streamGraph.getStreamNode(selfUnion.getId()).getInEdges()) {
assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
}
// verify self union with differnt partitioners
assertTrue(
streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges().size() == 2);
boolean hasForward = false;
boolean hasBroadcast = false;
for (StreamEdge edge :
streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges()) {
if (edge.getPartitioner() instanceof ForwardPartitioner) {
hasForward = true;
}
if (edge.getPartitioner() instanceof BroadcastPartitioner) {
hasBroadcast = true;
}
}
assertTrue(hasForward && hasBroadcast);
// verify union of streams with differing parallelism
assertTrue(
streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges().size() == 2);
for (StreamEdge edge :
streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges()) {
if (edge.getSourceId() == input2.getId()) {
assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
} else if (edge.getSourceId() == input3.getId()) {
assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
} else {
fail("Wrong input edge.");
}
}
// verify union of streams with differing partitionings
assertTrue(
streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges().size() == 2);
for (StreamEdge edge :
streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges()) {
if (edge.getSourceId() == input4.getId()) {
assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
} else if (edge.getSourceId() == input5.getId()) {
assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
} else {
fail("Wrong input edge.");
}
}
}