private static DataStream<String> getTextDataStream(final StreamExecutionEnvironment env) { if (fileOutput) { // read the text file from given input path return env.readTextFile(textPath); } return env.fromElements(WordCountData.WORDS); }
public static void main(String[] args) throws Exception { if (!parseParameters(args)) { return; } // set up input for the stream of integer pairs // obtain execution environment and set setBufferTimeout to 1 to enable // continuous flushing of the output buffers (lowest latency) StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment().setBufferTimeout(1); // create input stream of integer pairs DataStream<Tuple2<Integer, Integer>> inputStream; if (fileInput) { inputStream = env.readTextFile(inputPath).map(new FibonacciInputMap()); } else { inputStream = env.addSource(new RandomFibonacciSource()); } // create an iterative data stream from the input with 5 second timeout IterativeStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> it = inputStream.map(new InputMap()).iterate(5000); // apply the step function to get the next Fibonacci number // increment the counter and split the output with the output selector SplitStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> step = it.map(new Step()).split(new MySelector()); // close the iteration by selecting the tuples that were directed to the // 'iterate' channel in the output selector it.closeWith(step.select("iterate")); // to produce the final output select the tuples directed to the // 'output' channel then get the input pairs that have the greatest iteration counter // on a 1 second sliding window DataStream<Tuple2<Tuple2<Integer, Integer>, Integer>> numbers = step.select("output").map(new OutputMap()); // emit results if (fileOutput) { numbers.writeAsText(outputPath, 1); } else { numbers.print(); } // execute the program env.execute("Streaming Iteration Example"); }
public static void main(String[] args) throws Exception { if (!parseParameters(args)) { return; } final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings({"rawtypes", "serial"}) DataStream<Tuple4<Integer, Integer, Double, Long>> carData; if (fileInput) { carData = env.readTextFile(inputPath).map(new ParseCarData()); } else { carData = env.addSource(CarSource.create(numOfCars)); } DataStream<Tuple4<Integer, Integer, Double, Long>> topSpeeds = carData .groupBy(0) .window(Time.of(evictionSec * 1000, new CarTimestamp())) .every( Delta.of( triggerMeters, new DeltaFunction<Tuple4<Integer, Integer, Double, Long>>() { private static final long serialVersionUID = 1L; @Override public double getDelta( Tuple4<Integer, Integer, Double, Long> oldDataPoint, Tuple4<Integer, Integer, Double, Long> newDataPoint) { return newDataPoint.f2 - oldDataPoint.f2; } }, new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l))) .local() .maxBy(1) .flatten(); if (fileOutput) { topSpeeds.writeAsText(outputPath); } else { topSpeeds.print(); } env.execute("CarTopSpeedWindowingExample"); }