@Test public void testChannelSelectors() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSource<Long> src = env.generateSequence(0, 0); DataStream<Long> broadcast = src.broadcast(); DataStreamSink<Long> broadcastSink = broadcast.print(); StreamPartitioner<?> broadcastPartitioner = env.getStreamGraph() .getStreamEdges(src.getId(), broadcastSink.getTransformation().getId()) .get(0) .getPartitioner(); assertTrue(broadcastPartitioner instanceof BroadcastPartitioner); DataStream<Long> shuffle = src.shuffle(); DataStreamSink<Long> shuffleSink = shuffle.print(); StreamPartitioner<?> shufflePartitioner = env.getStreamGraph() .getStreamEdges(src.getId(), shuffleSink.getTransformation().getId()) .get(0) .getPartitioner(); assertTrue(shufflePartitioner instanceof ShufflePartitioner); DataStream<Long> forward = src.forward(); DataStreamSink<Long> forwardSink = forward.print(); StreamPartitioner<?> forwardPartitioner = env.getStreamGraph() .getStreamEdges(src.getId(), forwardSink.getTransformation().getId()) .get(0) .getPartitioner(); assertTrue(forwardPartitioner instanceof ForwardPartitioner); DataStream<Long> rebalance = src.rebalance(); DataStreamSink<Long> rebalanceSink = rebalance.print(); StreamPartitioner<?> rebalancePartitioner = env.getStreamGraph() .getStreamEdges(src.getId(), rebalanceSink.getTransformation().getId()) .get(0) .getPartitioner(); assertTrue(rebalancePartitioner instanceof RebalancePartitioner); DataStream<Long> global = src.global(); DataStreamSink<Long> globalSink = global.print(); StreamPartitioner<?> globalPartitioner = env.getStreamGraph() .getStreamEdges(src.getId(), globalSink.getTransformation().getId()) .get(0) .getPartitioner(); assertTrue(globalPartitioner instanceof GlobalPartitioner); }
public static void main(String[] args) throws Exception { // create execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // parse user parameters // ParameterTool parameterTool = ParameterTool.fromArgs(args); // DataStream<String> messageStream = env.addSource(new // FlinkKafkaConsumer(parameterTool.getRequired("topic"), new SimpleStringSchema(), // parameterTool.getProperties())); Properties properties = new Properties(); properties.setProperty("bootstrap.servers", "node2:9092"); properties.setProperty("zookeeper.connect", "node2:2181"); properties.setProperty("group.id", "1"); DataStream<String> messageStream = env.addSource( new FlinkKafkaConsumer082<>("demo", new SimpleStringSchema(), properties)); // print(); messageStream.print(); System.out.print(messageStream + " Hello\n"); // print() will write the contents of the stream to the TaskManager's standard out stream // the rebelance call is causing a repartitioning of the data so that all machines // see the messages (for example in cases when "num kafka partitions" < "num flink operators" // messageStream.rebalance().map(new MapFunction<String, String>() { // private static final long serialVersionUID = -6867736771747690202L; // @Override // public String map(String value) throws Exception { // return "Kafka and Flink says: " + value; // } // }).print(); env.execute("kafka consumer"); }
private static Integer createDownStreamId(DataStream<?> dataStream) { return dataStream.print().getTransformation().getId(); }
@Test public void operatorTest() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSource<Long> src = env.generateSequence(0, 0); MapFunction<Long, Integer> mapFunction = new MapFunction<Long, Integer>() { @Override public Integer map(Long value) throws Exception { return null; } }; DataStream<Integer> map = src.map(mapFunction); map.addSink(new NoOpSink<Integer>()); assertEquals(mapFunction, getFunctionForDataStream(map)); FlatMapFunction<Long, Integer> flatMapFunction = new FlatMapFunction<Long, Integer>() { private static final long serialVersionUID = 1L; @Override public void flatMap(Long value, Collector<Integer> out) throws Exception {} }; DataStream<Integer> flatMap = src.flatMap(flatMapFunction); flatMap.addSink(new NoOpSink<Integer>()); assertEquals(flatMapFunction, getFunctionForDataStream(flatMap)); FilterFunction<Integer> filterFunction = new FilterFunction<Integer>() { @Override public boolean filter(Integer value) throws Exception { return false; } }; DataStream<Integer> unionFilter = map.union(flatMap).filter(filterFunction); unionFilter.addSink(new NoOpSink<Integer>()); assertEquals(filterFunction, getFunctionForDataStream(unionFilter)); try { env.getStreamGraph().getStreamEdge(map.getId(), unionFilter.getId()); } catch (RuntimeException e) { fail(e.getMessage()); } try { env.getStreamGraph().getStreamEdge(flatMap.getId(), unionFilter.getId()); } catch (RuntimeException e) { fail(e.getMessage()); } OutputSelector<Integer> outputSelector = new OutputSelector<Integer>() { @Override public Iterable<String> select(Integer value) { return null; } }; SplitStream<Integer> split = unionFilter.split(outputSelector); split.select("dummy").addSink(new NoOpSink<Integer>()); List<OutputSelector<?>> outputSelectors = env.getStreamGraph().getStreamNode(unionFilter.getId()).getOutputSelectors(); assertEquals(1, outputSelectors.size()); assertEquals(outputSelector, outputSelectors.get(0)); DataStream<Integer> select = split.select("a"); DataStreamSink<Integer> sink = select.print(); StreamEdge splitEdge = env.getStreamGraph().getStreamEdge(unionFilter.getId(), sink.getTransformation().getId()); assertEquals("a", splitEdge.getSelectedNames().get(0)); ConnectedStreams<Integer, Integer> connect = map.connect(flatMap); CoMapFunction<Integer, Integer, String> coMapper = new CoMapFunction<Integer, Integer, String>() { private static final long serialVersionUID = 1L; @Override public String map1(Integer value) { return null; } @Override public String map2(Integer value) { return null; } }; DataStream<String> coMap = connect.map(coMapper); coMap.addSink(new NoOpSink<String>()); assertEquals(coMapper, getFunctionForDataStream(coMap)); try { env.getStreamGraph().getStreamEdge(map.getId(), coMap.getId()); } catch (RuntimeException e) { fail(e.getMessage()); } try { env.getStreamGraph().getStreamEdge(flatMap.getId(), coMap.getId()); } catch (RuntimeException e) { fail(e.getMessage()); } }