/**
* Note: this test fails if we don't have the synchronized block in {@link
* org.apache.flink.streaming.runtime.tasks.SourceStreamTask.SourceOutput}
*/
@Test
public void testOneInputOperatorWithoutChaining() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
DataStream<String> source = env.addSource(new InfiniteTestSource());
source.transform(
"Custom Operator",
BasicTypeInfo.STRING_TYPE_INFO,
new TimerOperator(StreamOperator.ChainingStrategy.NEVER));
boolean testSuccess = false;
try {
env.execute("Timer test");
} catch (JobExecutionException e) {
if (e.getCause() instanceof TimerException) {
TimerException te = (TimerException) e.getCause();
if (te.getCause() instanceof RuntimeException) {
RuntimeException re = (RuntimeException) te.getCause();
if (re.getMessage().equals("TEST SUCCESS")) {
testSuccess = true;
} else {
throw e;
}
} else {
throw e;
}
} else {
throw e;
}
}
Assert.assertTrue(testSuccess);
}
@Test
public void testSimplePatternWithSingleState() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<Tuple2<Integer, Integer>> input =
env.fromElements(new Tuple2<>(0, 1), new Tuple2<>(0, 2));
Pattern<Tuple2<Integer, Integer>, ?> pattern =
Pattern.<Tuple2<Integer, Integer>>begin("start")
.where(
new FilterFunction<Tuple2<Integer, Integer>>() {
@Override
public boolean filter(Tuple2<Integer, Integer> rec) throws Exception {
return rec.f1 == 1;
}
});
PatternStream<Tuple2<Integer, Integer>> pStream = CEP.pattern(input, pattern);
DataStream<Tuple2<Integer, Integer>> result =
pStream.select(
new PatternSelectFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>() {
@Override
public Tuple2<Integer, Integer> select(Map<String, Tuple2<Integer, Integer>> pattern)
throws Exception {
return pattern.get("start");
}
});
result.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
expected = "(0,1)";
env.execute();
}
public static void main(final String[] args) throws Exception {
if (!parseParameters(args)) {
return;
}
// set up the execution environment
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// get input data
final DataStream<String> text = getTextDataStream(env);
final DataStream<Tuple2<String, Integer>> counts =
text
// split up the lines in pairs (2-tuples) containing: (word,1)
// this is done by a bolt that is wrapped accordingly
.transform(
"BoltTokenizer",
TypeExtractor.getForObject(new Tuple2<String, Integer>("", 0)),
new BoltWrapper<String, Tuple2<String, Integer>>(new BoltTokenizer()))
// group by the tuple field "0" and sum up tuple field "1"
.keyBy(0)
.sum(1);
// emit result
if (fileOutput) {
counts.writeAsText(outputPath);
} else {
counts.print();
}
// execute program
env.execute("Streaming WordCount with bolt tokenizer");
}
/** This tests {@link StringWriter} with non-rolling output. */
@Test
public void testNonRollingStringWriter() throws Exception {
final int NUM_ELEMENTS = 20;
final int PARALLELISM = 2;
final String outPath = hdfsURI + "/string-non-rolling-out";
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
DataStream<Tuple2<Integer, String>> source =
env.addSource(new TestSourceFunction(NUM_ELEMENTS)).broadcast().filter(new OddEvenFilter());
RollingSink<String> sink =
new RollingSink<String>(outPath)
.setBucketer(new NonRollingBucketer())
.setPartPrefix("part")
.setPendingPrefix("")
.setPendingSuffix("");
source
.map(
new MapFunction<Tuple2<Integer, String>, String>() {
private static final long serialVersionUID = 1L;
@Override
public String map(Tuple2<Integer, String> value) throws Exception {
return value.f1;
}
})
.addSink(sink);
env.execute("RollingSink String Write Test");
FSDataInputStream inStream = dfs.open(new Path(outPath + "/part-0-0"));
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 0; i < NUM_ELEMENTS; i += 2) {
String line = br.readLine();
Assert.assertEquals("message #" + i, line);
}
inStream.close();
inStream = dfs.open(new Path(outPath + "/part-1-0"));
br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 1; i < NUM_ELEMENTS; i += 2) {
String line = br.readLine();
Assert.assertEquals("message #" + i, line);
}
inStream.close();
}
@Override
protected void testProgram() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<String> text = env.fromElements(WordCountData.TEXT);
DataStream<Tuple2<String, Integer>> counts =
text.flatMap(new CsvOutputFormatITCase.Tokenizer()).keyBy(0).sum(1);
counts.writeAsText(resultPath);
env.execute("WriteAsTextTest");
}
public static void main(String[] args) throws Exception {
if (!parseParameters(args)) {
return;
}
// set up input for the stream of integer pairs
// obtain execution environment and set setBufferTimeout to 1 to enable
// continuous flushing of the output buffers (lowest latency)
StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironment().setBufferTimeout(1);
// create input stream of integer pairs
DataStream<Tuple2<Integer, Integer>> inputStream;
if (fileInput) {
inputStream = env.readTextFile(inputPath).map(new FibonacciInputMap());
} else {
inputStream = env.addSource(new RandomFibonacciSource());
}
// create an iterative data stream from the input with 5 second timeout
IterativeStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> it =
inputStream.map(new InputMap()).iterate(5000);
// apply the step function to get the next Fibonacci number
// increment the counter and split the output with the output selector
SplitStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> step =
it.map(new Step()).split(new MySelector());
// close the iteration by selecting the tuples that were directed to the
// 'iterate' channel in the output selector
it.closeWith(step.select("iterate"));
// to produce the final output select the tuples directed to the
// 'output' channel then get the input pairs that have the greatest iteration counter
// on a 1 second sliding window
DataStream<Tuple2<Tuple2<Integer, Integer>, Integer>> numbers =
step.select("output").map(new OutputMap());
// emit results
if (fileOutput) {
numbers.writeAsText(outputPath, 1);
} else {
numbers.print();
}
// execute the program
env.execute("Streaming Iteration Example");
}
public static void generateLongStringTupleSequence(
StreamExecutionEnvironment env,
String brokerConnection,
String topic,
int numPartitions,
final int from,
final int to)
throws Exception {
TypeInformation<Tuple2<Integer, Integer>> resultType =
TypeInfoParser.parse("Tuple2<Integer, Integer>");
env.setParallelism(numPartitions);
env.getConfig().disableSysoutLogging();
env.setNumberOfExecutionRetries(0);
DataStream<Tuple2<Integer, Integer>> stream =
env.addSource(
new RichParallelSourceFunction<Tuple2<Integer, Integer>>() {
private volatile boolean running = true;
@Override
public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
int cnt = from;
int partition = getRuntimeContext().getIndexOfThisSubtask();
while (running && cnt <= to) {
ctx.collect(new Tuple2<Integer, Integer>(partition, cnt));
cnt++;
}
}
@Override
public void cancel() {
running = false;
}
});
stream.addSink(
new FlinkKafkaProducer<>(
topic,
new TypeInformationSerializationSchema<>(resultType, env.getConfig()),
FlinkKafkaProducer.getPropertiesFromBrokerList(brokerConnection),
new Tuple2Partitioner(numPartitions)));
env.execute("Data generator (Int, Int) stream to topic " + topic);
}
public static void tryExecute(StreamExecutionEnvironment env, String jobName) throws Exception {
try {
env.execute(jobName);
} catch (ProgramInvocationException | JobExecutionException root) {
Throwable cause = root.getCause();
// search for nested SuccessExceptions
int depth = 0;
while (!(cause instanceof SuccessException)) {
if (cause == null || depth++ == 20) {
root.printStackTrace();
fail("Test failed: " + root.getMessage());
} else {
cause = cause.getCause();
}
}
}
}
public static void main(String[] args) throws Exception {
if (!parseParameters(args)) {
return;
}
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings({"rawtypes", "serial"})
DataStream<Tuple4<Integer, Integer, Double, Long>> carData;
if (fileInput) {
carData = env.readTextFile(inputPath).map(new ParseCarData());
} else {
carData = env.addSource(CarSource.create(numOfCars));
}
DataStream<Tuple4<Integer, Integer, Double, Long>> topSpeeds =
carData
.groupBy(0)
.window(Time.of(evictionSec * 1000, new CarTimestamp()))
.every(
Delta.of(
triggerMeters,
new DeltaFunction<Tuple4<Integer, Integer, Double, Long>>() {
private static final long serialVersionUID = 1L;
@Override
public double getDelta(
Tuple4<Integer, Integer, Double, Long> oldDataPoint,
Tuple4<Integer, Integer, Double, Long> newDataPoint) {
return newDataPoint.f2 - oldDataPoint.f2;
}
},
new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l)))
.local()
.maxBy(1)
.flatten();
if (fileOutput) {
topSpeeds.writeAsText(outputPath);
} else {
topSpeeds.print();
}
env.execute("CarTopSpeedWindowingExample");
}
/**
* Verify that the user-specified state backend is used even if checkpointing is disabled.
*
* @throws Exception
*/
@Test
public void testStateBackendWithoutCheckpointing() throws Exception {
StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
see.setParallelism(1);
see.getConfig().setRestartStrategy(RestartStrategies.noRestart());
see.setStateBackend(new FailingStateBackend());
see.fromElements(new Tuple2<>("Hello", 1))
.keyBy(0)
.map(
new RichMapFunction<Tuple2<String, Integer>, String>() {
private static final long serialVersionUID = 1L;
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
getRuntimeContext().getKeyValueState("test", String.class, "");
}
@Override
public String map(Tuple2<String, Integer> value) throws Exception {
return value.f0;
}
})
.print();
try {
see.execute();
fail();
} catch (JobExecutionException e) {
Throwable t = e.getCause();
if (!(t != null && t.getCause() instanceof SuccessException)) {
throw e;
}
}
}
public static void main(String[] args) throws Exception {
// create execution environment
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// parse user parameters
// ParameterTool parameterTool = ParameterTool.fromArgs(args);
// DataStream<String> messageStream = env.addSource(new
// FlinkKafkaConsumer(parameterTool.getRequired("topic"), new SimpleStringSchema(),
// parameterTool.getProperties()));
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "node2:9092");
properties.setProperty("zookeeper.connect", "node2:2181");
properties.setProperty("group.id", "1");
DataStream<String> messageStream =
env.addSource(
new FlinkKafkaConsumer082<>("demo", new SimpleStringSchema(), properties)); // print();
messageStream.print();
System.out.print(messageStream + " Hello\n");
// print() will write the contents of the stream to the TaskManager's standard out stream
// the rebelance call is causing a repartitioning of the data so that all machines
// see the messages (for example in cases when "num kafka partitions" < "num flink operators"
// messageStream.rebalance().map(new MapFunction<String, String>() {
// private static final long serialVersionUID = -6867736771747690202L;
// @Override
// public String map(String value) throws Exception {
// return "Kafka and Flink says: " + value;
// }
// }).print();
env.execute("kafka consumer");
}
/**
* Runs the following program:
*
* <pre>
* [ (source)->(filter)->(map) ] -> [ (map) ] -> [ (groupBy/reduce)->(sink) ]
* </pre>
*/
@Test
public void runCheckpointedProgram() {
final long NUM_STRINGS = 10000000L;
assertTrue("Broken test setup", NUM_STRINGS % 40 == 0);
try {
StreamExecutionEnvironment env =
StreamExecutionEnvironment.createRemoteEnvironment(
"localhost", cluster.getJobManagerRPCPort());
env.setParallelism(PARALLELISM);
env.enableCheckpointing(500);
env.getConfig().disableSysoutLogging();
DataStream<String> stream = env.addSource(new StringGeneratingSourceFunction(NUM_STRINGS));
stream
// -------------- first vertex, chained to the source ----------------
.filter(new StringRichFilterFunction())
// -------------- seconds vertex - the stateful one that also fails ----------------
.map(new StringPrefixCountRichMapFunction())
.startNewChain()
.map(new StatefulCounterFunction())
// -------------- third vertex - reducer and the sink ----------------
.groupBy("prefix")
.reduce(new OnceFailingReducer(NUM_STRINGS))
.addSink(
new RichSinkFunction<PrefixCount>() {
private Map<Character, Long> counts = new HashMap<Character, Long>();
@Override
public void invoke(PrefixCount value) {
Character first = value.prefix.charAt(0);
Long previous = counts.get(first);
if (previous == null) {
counts.put(first, value.count);
} else {
counts.put(first, Math.max(previous, value.count));
}
}
// @Override
// public void close() {
// for (Long count : counts.values()) {
// assertEquals(NUM_STRINGS / 40, count.longValue());
// }
// }
});
env.execute();
long filterSum = 0;
for (long l : StringRichFilterFunction.counts) {
filterSum += l;
}
long mapSum = 0;
for (long l : StringPrefixCountRichMapFunction.counts) {
mapSum += l;
}
long countSum = 0;
for (long l : StatefulCounterFunction.counts) {
countSum += l;
}
// verify that we counted exactly right
// this line should be uncommented once the "exactly one off by one" is fixed
// if this fails we see at which point the count is off
assertEquals(NUM_STRINGS, filterSum);
assertEquals(NUM_STRINGS, mapSum);
assertEquals(NUM_STRINGS, countSum);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
/**
* Checks that a certain event sequence is recognized
*
* @throws Exception
*/
@Test
public void testSimplePatternCEP() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<Event> input =
env.fromElements(
new Event(1, "barfoo", 1.0),
new Event(2, "start", 2.0),
new Event(3, "foobar", 3.0),
new SubEvent(4, "foo", 4.0, 1.0),
new Event(5, "middle", 5.0),
new SubEvent(6, "middle", 6.0, 2.0),
new SubEvent(7, "bar", 3.0, 3.0),
new Event(42, "42", 42.0),
new Event(8, "end", 1.0));
Pattern<Event, ?> pattern =
Pattern.<Event>begin("start")
.where(
new FilterFunction<Event>() {
@Override
public boolean filter(Event value) throws Exception {
return value.getName().equals("start");
}
})
.followedBy("middle")
.subtype(SubEvent.class)
.where(
new FilterFunction<SubEvent>() {
@Override
public boolean filter(SubEvent value) throws Exception {
return value.getName().equals("middle");
}
})
.followedBy("end")
.where(
new FilterFunction<Event>() {
@Override
public boolean filter(Event value) throws Exception {
return value.getName().equals("end");
}
});
DataStream<String> result =
CEP.pattern(input, pattern)
.select(
new PatternSelectFunction<Event, String>() {
@Override
public String select(Map<String, Event> pattern) {
StringBuilder builder = new StringBuilder();
builder
.append(pattern.get("start").getId())
.append(",")
.append(pattern.get("middle").getId())
.append(",")
.append(pattern.get("end").getId());
return builder.toString();
}
});
result.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
// expected sequence of matching event ids
expected = "2,6,8";
env.execute();
}
@Test
public void testSimpleKeyedPatternEventTime() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.setParallelism(2);
// (Event, timestamp)
DataStream<Event> input =
env.fromElements(
Tuple2.of(new Event(1, "start", 1.0), 5L),
Tuple2.of(new Event(1, "middle", 2.0), 1L),
Tuple2.of(new Event(2, "middle", 2.0), 4L),
Tuple2.of(new Event(2, "start", 2.0), 3L),
Tuple2.of(new Event(1, "end", 3.0), 3L),
Tuple2.of(new Event(3, "start", 4.1), 5L),
Tuple2.of(new Event(1, "end", 4.0), 10L),
Tuple2.of(new Event(2, "end", 2.0), 8L),
Tuple2.of(new Event(1, "middle", 5.0), 7L),
Tuple2.of(new Event(3, "middle", 6.0), 9L),
Tuple2.of(new Event(3, "end", 7.0), 7L),
// last element for high final watermark
Tuple2.of(new Event(3, "end", 7.0), 100L))
.assignTimestampsAndWatermarks(
new AssignerWithPunctuatedWatermarks<Tuple2<Event, Long>>() {
@Override
public long extractTimestamp(Tuple2<Event, Long> element, long currentTimestamp) {
return element.f1;
}
@Override
public Watermark checkAndGetNextWatermark(
Tuple2<Event, Long> lastElement, long extractedTimestamp) {
return new Watermark(lastElement.f1 - 5);
}
})
.map(
new MapFunction<Tuple2<Event, Long>, Event>() {
@Override
public Event map(Tuple2<Event, Long> value) throws Exception {
return value.f0;
}
})
.keyBy(
new KeySelector<Event, Integer>() {
@Override
public Integer getKey(Event value) throws Exception {
return value.getId();
}
});
Pattern<Event, ?> pattern =
Pattern.<Event>begin("start")
.where(
new FilterFunction<Event>() {
@Override
public boolean filter(Event value) throws Exception {
return value.getName().equals("start");
}
})
.followedBy("middle")
.where(
new FilterFunction<Event>() {
@Override
public boolean filter(Event value) throws Exception {
return value.getName().equals("middle");
}
})
.followedBy("end")
.where(
new FilterFunction<Event>() {
@Override
public boolean filter(Event value) throws Exception {
return value.getName().equals("end");
}
});
DataStream<String> result =
CEP.pattern(input, pattern)
.select(
new PatternSelectFunction<Event, String>() {
@Override
public String select(Map<String, Event> pattern) {
StringBuilder builder = new StringBuilder();
builder
.append(pattern.get("start").getId())
.append(",")
.append(pattern.get("middle").getId())
.append(",")
.append(pattern.get("end").getId());
return builder.toString();
}
});
result.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
// the expected sequences of matching event ids
expected = "1,1,1\n2,2,2";
env.execute();
}
public static void generateRandomizedIntegerSequence(
StreamExecutionEnvironment env,
String brokerConnection,
String topic,
final int numPartitions,
final int numElements,
final boolean randomizeOrder)
throws Exception {
env.setParallelism(numPartitions);
env.getConfig().disableSysoutLogging();
env.setNumberOfExecutionRetries(0);
DataStream<Integer> stream =
env.addSource(
new RichParallelSourceFunction<Integer>() {
private volatile boolean running = true;
@Override
public void run(SourceContext<Integer> ctx) {
// create a sequence
int[] elements = new int[numElements];
for (int i = 0, val = getRuntimeContext().getIndexOfThisSubtask();
i < numElements;
i++, val += getRuntimeContext().getNumberOfParallelSubtasks()) {
elements[i] = val;
}
// scramble the sequence
if (randomizeOrder) {
Random rnd = new Random();
for (int i = 0; i < elements.length; i++) {
int otherPos = rnd.nextInt(elements.length);
int tmp = elements[i];
elements[i] = elements[otherPos];
elements[otherPos] = tmp;
}
}
// emit the sequence
int pos = 0;
while (running && pos < elements.length) {
ctx.collect(elements[pos++]);
}
}
@Override
public void cancel() {
running = false;
}
});
stream
.rebalance()
.addSink(
new FlinkKafkaProducer<>(
topic,
new TypeInformationSerializationSchema<>(
BasicTypeInfo.INT_TYPE_INFO, env.getConfig()),
FlinkKafkaProducer.getPropertiesFromBrokerList(brokerConnection),
new KafkaPartitioner() {
@Override
public int partition(Object key, int numPartitions) {
return ((Integer) key) % numPartitions;
}
}));
env.execute("Scrambles int sequence generator");
}
/**
* Directly executes the Storm topology based on the current context (local when in IDE and remote
* when executed through ./bin/flink).
*
* @return The Flink {@link JobExecutionResult} after the execution of the Storm topology.
* @throws Exception which occurs during execution of the translated Storm topology.
*/
public JobExecutionResult execute() throws Exception {
return env.execute();
}
/** This tests {@link SequenceFileWriter} with non-rolling output but with compression. */
@Test
public void testNonRollingSequenceFileWithCompressionWriter() throws Exception {
final int NUM_ELEMENTS = 20;
final int PARALLELISM = 2;
final String outPath = hdfsURI + "/seq-non-rolling-out";
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
DataStream<Tuple2<Integer, String>> source =
env.addSource(new TestSourceFunction(NUM_ELEMENTS)).broadcast().filter(new OddEvenFilter());
DataStream<Tuple2<IntWritable, Text>> mapped =
source.map(
new MapFunction<Tuple2<Integer, String>, Tuple2<IntWritable, Text>>() {
private static final long serialVersionUID = 1L;
@Override
public Tuple2<IntWritable, Text> map(Tuple2<Integer, String> value) throws Exception {
return Tuple2.of(new IntWritable(value.f0), new Text(value.f1));
}
});
RollingSink<Tuple2<IntWritable, Text>> sink =
new RollingSink<Tuple2<IntWritable, Text>>(outPath)
.setWriter(
new SequenceFileWriter<IntWritable, Text>(
"Default", SequenceFile.CompressionType.BLOCK))
.setBucketer(new NonRollingBucketer())
.setPartPrefix("part")
.setPendingPrefix("")
.setPendingSuffix("");
mapped.addSink(sink);
env.execute("RollingSink String Write Test");
FSDataInputStream inStream = dfs.open(new Path(outPath + "/part-0-0"));
SequenceFile.Reader reader =
new SequenceFile.Reader(inStream, 1000, 0, 100000, new Configuration());
IntWritable intWritable = new IntWritable();
Text txt = new Text();
for (int i = 0; i < NUM_ELEMENTS; i += 2) {
reader.next(intWritable, txt);
Assert.assertEquals(i, intWritable.get());
Assert.assertEquals("message #" + i, txt.toString());
}
reader.close();
inStream.close();
inStream = dfs.open(new Path(outPath + "/part-1-0"));
reader = new SequenceFile.Reader(inStream, 1000, 0, 100000, new Configuration());
for (int i = 1; i < NUM_ELEMENTS; i += 2) {
reader.next(intWritable, txt);
Assert.assertEquals(i, intWritable.get());
Assert.assertEquals("message #" + i, txt.toString());
}
reader.close();
inStream.close();
}
/**
* This uses {@link org.apache.flink.streaming.connectors.fs.DateTimeBucketer} to produce rolling
* files. The clock of DateTimeBucketer is set to {@link ModifyableClock} to keep the time in
* lockstep with the processing of elements using latches.
*/
@Test
public void testDateTimeRollingStringWriter() throws Exception {
final int NUM_ELEMENTS = 20;
final int PARALLELISM = 2;
final String outPath = hdfsURI + "/rolling-out";
DateTimeBucketer.setClock(new ModifyableClock());
ModifyableClock.setCurrentTime(0);
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
DataStream<Tuple2<Integer, String>> source =
env.addSource(new WaitingTestSourceFunction(NUM_ELEMENTS)).broadcast();
// the parallel flatMap is chained to the sink, so when it has seen 5 elements it can
// fire the latch
DataStream<String> mapped =
source.flatMap(
new RichFlatMapFunction<Tuple2<Integer, String>, String>() {
private static final long serialVersionUID = 1L;
int count = 0;
@Override
public void flatMap(Tuple2<Integer, String> value, Collector<String> out)
throws Exception {
out.collect(value.f1);
count++;
if (count >= 5) {
if (getRuntimeContext().getIndexOfThisSubtask() == 0) {
latch1.trigger();
} else {
latch2.trigger();
}
count = 0;
}
}
});
RollingSink<String> sink =
new RollingSink<String>(outPath)
.setBucketer(new DateTimeBucketer("ss"))
.setPartPrefix("part")
.setPendingPrefix("")
.setPendingSuffix("");
mapped.addSink(sink);
env.execute("RollingSink String Write Test");
RemoteIterator<LocatedFileStatus> files = dfs.listFiles(new Path(outPath), true);
// we should have 8 rolling files, 4 time intervals and parallelism of 2
int numFiles = 0;
while (files.hasNext()) {
LocatedFileStatus file = files.next();
numFiles++;
if (file.getPath().toString().contains("rolling-out/00")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 0; i < 5; i++) {
String line = br.readLine();
Assert.assertEquals("message #" + i, line);
}
inStream.close();
} else if (file.getPath().toString().contains("rolling-out/05")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 5; i < 10; i++) {
String line = br.readLine();
Assert.assertEquals("message #" + i, line);
}
inStream.close();
} else if (file.getPath().toString().contains("rolling-out/10")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 10; i < 15; i++) {
String line = br.readLine();
Assert.assertEquals("message #" + i, line);
}
inStream.close();
} else if (file.getPath().toString().contains("rolling-out/15")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 15; i < 20; i++) {
String line = br.readLine();
Assert.assertEquals("message #" + i, line);
}
inStream.close();
} else {
Assert.fail("File " + file + " does not match any expected roll pattern.");
}
}
Assert.assertEquals(8, numFiles);
}
