@Test public void testAtWatermarkAndLate() throws Exception { tester = TriggerTester.forTrigger( AfterWatermark.<IntervalWindow>pastEndOfWindow().withLateFirings(mockLate), FixedWindows.of(Duration.millis(100))); injectElements(1); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); // No early firing, just double checking when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(true); assertFalse(tester.shouldFire(window)); tester.fireIfShouldFire(window); assertFalse(tester.isMarkedFinished(window)); // Fire due to watermark when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false); tester.advanceInputWatermark(new Instant(100)); assertTrue(tester.shouldFire(window)); tester.fireIfShouldFire(window); assertFalse(tester.isMarkedFinished(window)); testRunningAsTrigger(mockLate, window); }
public static void main(String[] args) { PipelineOptionsFactory.register(KafkaStreamingWordCountOptions.class); KafkaStreamingWordCountOptions options = PipelineOptionsFactory.fromArgs(args).as(KafkaStreamingWordCountOptions.class); options.setJobName("KafkaExample - WindowSize: " + options.getWindowSize() + " seconds"); options.setStreaming(true); options.setCheckpointingInterval(1000L); options.setNumberOfExecutionRetries(5); options.setExecutionRetryDelay(3000L); options.setRunner(FlinkPipelineRunner.class); System.out.println( options.getKafkaTopic() + " " + options.getZookeeper() + " " + options.getBroker() + " " + options.getGroup()); Pipeline pipeline = Pipeline.create(options); Properties p = new Properties(); p.setProperty("zookeeper.connect", options.getZookeeper()); p.setProperty("bootstrap.servers", options.getBroker()); p.setProperty("group.id", options.getGroup()); // this is the Flink consumer that reads the input to // the program from a kafka topic. FlinkKafkaConsumer08<String> kafkaConsumer = new FlinkKafkaConsumer08<>(options.getKafkaTopic(), new SimpleStringSchema(), p); PCollection<String> words = pipeline .apply(Read.from(new UnboundedFlinkSource<>(kafkaConsumer)).named("StreamingWordCount")) .apply(ParDo.of(new ExtractWordsFn())) .apply( Window.<String>into( FixedWindows.of(Duration.standardSeconds(options.getWindowSize()))) .triggering(AfterWatermark.pastEndOfWindow()) .withAllowedLateness(Duration.ZERO) .discardingFiredPanes()); PCollection<KV<String, Long>> wordCounts = words.apply(Count.<String>perElement()); wordCounts.apply(ParDo.of(new FormatAsStringFn())).apply(TextIO.Write.to("./outputKafka.txt")); pipeline.run(); }
/** * Tests that if the EOW is finished in both as well as the merged window, then it is finished in * the merged result. * * <p>Because windows are discarded when a trigger finishes, we need to embed this in a sequence * in order to check that it is re-activated. So this test is potentially sensitive to other * triggers' correctness. */ @Test public void testEarlyAndLateOnMergeAlreadyFinished() throws Exception { tester = TriggerTester.forTrigger( AfterWatermark.<IntervalWindow>pastEndOfWindow() .withEarlyFirings(AfterPane.<IntervalWindow>elementCountAtLeast(100)) .withLateFirings(AfterPane.<IntervalWindow>elementCountAtLeast(1)), Sessions.withGapDuration(Duration.millis(10))); tester.injectElements(1); tester.injectElements(5); IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); // Finish the AfterWatermark.pastEndOfWindow() bit of the trigger in both windows tester.advanceInputWatermark(new Instant(15)); assertTrue(tester.shouldFire(firstWindow)); assertTrue(tester.shouldFire(secondWindow)); tester.fireIfShouldFire(firstWindow); tester.fireIfShouldFire(secondWindow); // Confirm that we are on the late trigger by probing assertFalse(tester.shouldFire(firstWindow)); assertFalse(tester.shouldFire(secondWindow)); tester.injectElements(1); tester.injectElements(5); assertTrue(tester.shouldFire(firstWindow)); assertTrue(tester.shouldFire(secondWindow)); tester.fireIfShouldFire(firstWindow); tester.fireIfShouldFire(secondWindow); // Merging should leave it on the late trigger tester.mergeWindows(); // Confirm that we are on the late trigger by probing assertFalse(tester.shouldFire(mergedWindow)); tester.injectElements(1); assertTrue(tester.shouldFire(mergedWindow)); }
/** * Tests that the trigger rewinds to be non-finished in the merged window. * * <p>Because windows are discarded when a trigger finishes, we need to embed this in a sequence * in order to check that it is re-activated. So this test is potentially sensitive to other * triggers' correctness. */ @Test public void testOnMergeRewinds() throws Exception { tester = TriggerTester.forTrigger( AfterEach.inOrder( AfterWatermark.<IntervalWindow>pastEndOfWindow(), Repeatedly.forever(AfterPane.<IntervalWindow>elementCountAtLeast(1))), Sessions.withGapDuration(Duration.millis(10))); tester.injectElements(1); tester.injectElements(5); IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); // Finish the AfterWatermark.pastEndOfWindow() trigger in only the first window tester.advanceInputWatermark(new Instant(11)); assertTrue(tester.shouldFire(firstWindow)); assertFalse(tester.shouldFire(secondWindow)); tester.fireIfShouldFire(firstWindow); // Confirm that we are on the second trigger by probing assertFalse(tester.shouldFire(firstWindow)); tester.injectElements(1); assertTrue(tester.shouldFire(firstWindow)); tester.fireIfShouldFire(firstWindow); // Merging should re-activate the watermark trigger in the merged window tester.mergeWindows(); // Confirm that we are not on the second trigger by probing assertFalse(tester.shouldFire(mergedWindow)); tester.injectElements(1); assertFalse(tester.shouldFire(mergedWindow)); // And confirm that advancing the watermark fires again tester.advanceInputWatermark(new Instant(15)); assertTrue(tester.shouldFire(mergedWindow)); }