@Test
public void testAfterPaneWithFixedWindow() throws Exception {
Duration windowDuration = Duration.millis(10);
ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
ReduceFnTester.nonCombining(
FixedWindows.of(windowDuration),
AfterPane.<IntervalWindow>elementCountAtLeast(2),
AccumulationMode.DISCARDING_FIRED_PANES,
Duration.millis(100));
tester.injectElements(
TimestampedValue.of(1, new Instant(1)), // first in window [0, 10)
TimestampedValue.of(2, new Instant(9)));
assertThat(
tester.extractOutput(),
Matchers.contains(
WindowMatchers.isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10)));
// This element belongs in the window that has already fired. It should not be re-output because
// that trigger (which was one-time) has already gone off.
tester.injectElements(TimestampedValue.of(6, new Instant(2)));
assertThat(tester.extractOutput(), Matchers.emptyIterable());
assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10))));
assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20))));
tester.assertHasOnlyGlobalAndFinishedSetsFor(
new IntervalWindow(new Instant(0), new Instant(10)));
}
@Test
public void testAtWatermarkAndLate() throws Exception {
tester =
TriggerTester.forTrigger(
AfterWatermark.<IntervalWindow>pastEndOfWindow().withLateFirings(mockLate),
FixedWindows.of(Duration.millis(100)));
injectElements(1);
IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));
// No early firing, just double checking
when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(true);
assertFalse(tester.shouldFire(window));
tester.fireIfShouldFire(window);
assertFalse(tester.isMarkedFinished(window));
// Fire due to watermark
when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false);
tester.advanceInputWatermark(new Instant(100));
assertTrue(tester.shouldFire(window));
tester.fireIfShouldFire(window);
assertFalse(tester.isMarkedFinished(window));
testRunningAsTrigger(mockLate, window);
}
@Test
public void testAfterPaneWithGlobalWindowsAndCombining() throws Exception {
Duration windowDuration = Duration.millis(10);
ReduceFnTester<Integer, Integer, IntervalWindow> tester =
ReduceFnTester.combining(
FixedWindows.of(windowDuration),
AfterPane.<IntervalWindow>elementCountAtLeast(2),
AccumulationMode.DISCARDING_FIRED_PANES,
new SumIntegerFn().<String>asKeyedFn(),
VarIntCoder.of(),
Duration.millis(100));
tester.injectElements(
TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(9)));
assertThat(
tester.extractOutput(),
Matchers.contains(WindowMatchers.isSingleWindowedValue(Matchers.equalTo(3), 1, 0, 10)));
// This element should not be output because that trigger (which was one-time) has already
// gone off.
tester.injectElements(TimestampedValue.of(6, new Instant(2)));
assertThat(tester.extractOutput(), Matchers.emptyIterable());
assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10))));
assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20))));
tester.assertHasOnlyGlobalAndFinishedSetsFor(
new IntervalWindow(new Instant(0), new Instant(10)));
}
@Test
public void testShouldFire() throws Exception {
TriggerTester<Integer, IntervalWindow> tester =
TriggerTester.forTrigger(
new ReshuffleTrigger<IntervalWindow>(), FixedWindows.of(Duration.millis(100)));
IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(300), new Instant(400));
assertTrue(tester.shouldFire(arbitraryWindow));
}
@Test
public void testOnTimer() throws Exception {
TriggerTester<Integer, IntervalWindow> tester =
TriggerTester.forTrigger(
new ReshuffleTrigger<IntervalWindow>(), FixedWindows.of(Duration.millis(100)));
IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(100), new Instant(200));
tester.fireIfShouldFire(arbitraryWindow);
assertFalse(tester.isMarkedFinished(arbitraryWindow));
}
public static void main(String[] args) {
PipelineOptionsFactory.register(KafkaStreamingWordCountOptions.class);
KafkaStreamingWordCountOptions options =
PipelineOptionsFactory.fromArgs(args).as(KafkaStreamingWordCountOptions.class);
options.setJobName("KafkaExample - WindowSize: " + options.getWindowSize() + " seconds");
options.setStreaming(true);
options.setCheckpointingInterval(1000L);
options.setNumberOfExecutionRetries(5);
options.setExecutionRetryDelay(3000L);
options.setRunner(FlinkPipelineRunner.class);
System.out.println(
options.getKafkaTopic()
+ " "
+ options.getZookeeper()
+ " "
+ options.getBroker()
+ " "
+ options.getGroup());
Pipeline pipeline = Pipeline.create(options);
Properties p = new Properties();
p.setProperty("zookeeper.connect", options.getZookeeper());
p.setProperty("bootstrap.servers", options.getBroker());
p.setProperty("group.id", options.getGroup());
// this is the Flink consumer that reads the input to
// the program from a kafka topic.
FlinkKafkaConsumer08<String> kafkaConsumer =
new FlinkKafkaConsumer08<>(options.getKafkaTopic(), new SimpleStringSchema(), p);
PCollection<String> words =
pipeline
.apply(Read.from(new UnboundedFlinkSource<>(kafkaConsumer)).named("StreamingWordCount"))
.apply(ParDo.of(new ExtractWordsFn()))
.apply(
Window.<String>into(
FixedWindows.of(Duration.standardSeconds(options.getWindowSize())))
.triggering(AfterWatermark.pastEndOfWindow())
.withAllowedLateness(Duration.ZERO)
.discardingFiredPanes());
PCollection<KV<String, Long>> wordCounts = words.apply(Count.<String>perElement());
wordCounts.apply(ParDo.of(new FormatAsStringFn())).apply(TextIO.Write.to("./outputKafka.txt"));
pipeline.run();
}
@Test
public void testTopEmptyWithIncompatibleWindows() {
Pipeline p = TestPipeline.create();
Bound<String> windowingFn = Window.<String>into(FixedWindows.of(Duration.standardDays(10L)));
PCollection<String> input =
p.apply(Create.timestamped(Collections.<String>emptyList(), Collections.<Long>emptyList()))
.apply(windowingFn);
expectedEx.expect(IllegalStateException.class);
expectedEx.expectMessage("Top");
expectedEx.expectMessage("GlobalWindows");
expectedEx.expectMessage("withoutDefaults");
expectedEx.expectMessage("asSingletonView");
input.apply(Top.of(1, new OrderByLength()));
}
@Test
public void testDefaultTriggerWithFixedWindow() throws Exception {
ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
ReduceFnTester.nonCombining(
FixedWindows.of(Duration.millis(10)),
DefaultTrigger.<IntervalWindow>of(),
AccumulationMode.DISCARDING_FIRED_PANES,
Duration.millis(100));
tester.injectElements(
TimestampedValue.of(1, new Instant(1)),
TimestampedValue.of(2, new Instant(9)),
TimestampedValue.of(3, new Instant(15)),
TimestampedValue.of(4, new Instant(19)),
TimestampedValue.of(5, new Instant(30)));
// Advance the watermark almost to the end of the first window.
tester.advanceProcessingTime(new Instant(500));
tester.advanceInputWatermark(new Instant(8));
assertThat(tester.extractOutput(), Matchers.emptyIterable());
// Advance watermark to 10 (past end of the window), which causes the first fixed window to
// be emitted
tester.advanceInputWatermark(new Instant(10));
assertThat(
tester.extractOutput(),
Matchers.contains(isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10)));
// Advance watermark to 100, which causes the remaining two windows to be emitted.
// Since their timers were at different timestamps, they should fire in order.
tester.advanceInputWatermark(new Instant(100));
assertThat(
tester.extractOutput(),
Matchers.contains(
isSingleWindowedValue(Matchers.containsInAnyOrder(3, 4), 15, 10, 20),
isSingleWindowedValue(Matchers.contains(5), 30, 30, 40)));
assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(30), new Instant(40))));
tester.assertHasOnlyGlobalAndPaneInfoFor(
new IntervalWindow(new Instant(0), new Instant(10)),
new IntervalWindow(new Instant(10), new Instant(20)),
new IntervalWindow(new Instant(30), new Instant(40)));
}