private static DataStream<String> getTextDataStream(final StreamExecutionEnvironment env) {
if (fileOutput) {
// read the text file from given input path
return env.readTextFile(textPath);
}
return env.fromElements(WordCountData.WORDS);
}
public static void main(String[] args) throws Exception {
if (!parseParameters(args)) {
return;
}
// set up input for the stream of integer pairs
// obtain execution environment and set setBufferTimeout to 1 to enable
// continuous flushing of the output buffers (lowest latency)
StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironment().setBufferTimeout(1);
// create input stream of integer pairs
DataStream<Tuple2<Integer, Integer>> inputStream;
if (fileInput) {
inputStream = env.readTextFile(inputPath).map(new FibonacciInputMap());
} else {
inputStream = env.addSource(new RandomFibonacciSource());
}
// create an iterative data stream from the input with 5 second timeout
IterativeStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> it =
inputStream.map(new InputMap()).iterate(5000);
// apply the step function to get the next Fibonacci number
// increment the counter and split the output with the output selector
SplitStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> step =
it.map(new Step()).split(new MySelector());
// close the iteration by selecting the tuples that were directed to the
// 'iterate' channel in the output selector
it.closeWith(step.select("iterate"));
// to produce the final output select the tuples directed to the
// 'output' channel then get the input pairs that have the greatest iteration counter
// on a 1 second sliding window
DataStream<Tuple2<Tuple2<Integer, Integer>, Integer>> numbers =
step.select("output").map(new OutputMap());
// emit results
if (fileOutput) {
numbers.writeAsText(outputPath, 1);
} else {
numbers.print();
}
// execute the program
env.execute("Streaming Iteration Example");
}
public static void main(String[] args) throws Exception {
if (!parseParameters(args)) {
return;
}
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings({"rawtypes", "serial"})
DataStream<Tuple4<Integer, Integer, Double, Long>> carData;
if (fileInput) {
carData = env.readTextFile(inputPath).map(new ParseCarData());
} else {
carData = env.addSource(CarSource.create(numOfCars));
}
DataStream<Tuple4<Integer, Integer, Double, Long>> topSpeeds =
carData
.groupBy(0)
.window(Time.of(evictionSec * 1000, new CarTimestamp()))
.every(
Delta.of(
triggerMeters,
new DeltaFunction<Tuple4<Integer, Integer, Double, Long>>() {
private static final long serialVersionUID = 1L;
@Override
public double getDelta(
Tuple4<Integer, Integer, Double, Long> oldDataPoint,
Tuple4<Integer, Integer, Double, Long> newDataPoint) {
return newDataPoint.f2 - oldDataPoint.f2;
}
},
new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l)))
.local()
.maxBy(1)
.flatten();
if (fileOutput) {
topSpeeds.writeAsText(outputPath);
} else {
topSpeeds.print();
}
env.execute("CarTopSpeedWindowingExample");
}