@Test public void testAfterPaneWithFixedWindow() throws Exception { Duration windowDuration = Duration.millis(10); ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(windowDuration), AfterPane.<IntervalWindow>elementCountAtLeast(2), AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), // first in window [0, 10) TimestampedValue.of(2, new Instant(9))); assertThat( tester.extractOutput(), Matchers.contains( WindowMatchers.isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10))); // This element belongs in the window that has already fired. It should not be re-output because // that trigger (which was one-time) has already gone off. tester.injectElements(TimestampedValue.of(6, new Instant(2))); assertThat(tester.extractOutput(), Matchers.emptyIterable()); assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10)))); assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20)))); tester.assertHasOnlyGlobalAndFinishedSetsFor( new IntervalWindow(new Instant(0), new Instant(10))); }
@Test public void testAtWatermarkAndLate() throws Exception { tester = TriggerTester.forTrigger( AfterWatermark.<IntervalWindow>pastEndOfWindow().withLateFirings(mockLate), FixedWindows.of(Duration.millis(100))); injectElements(1); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); // No early firing, just double checking when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(true); assertFalse(tester.shouldFire(window)); tester.fireIfShouldFire(window); assertFalse(tester.isMarkedFinished(window)); // Fire due to watermark when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false); tester.advanceInputWatermark(new Instant(100)); assertTrue(tester.shouldFire(window)); tester.fireIfShouldFire(window); assertFalse(tester.isMarkedFinished(window)); testRunningAsTrigger(mockLate, window); }
@Test public void testAfterPaneWithGlobalWindowsAndCombining() throws Exception { Duration windowDuration = Duration.millis(10); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining( FixedWindows.of(windowDuration), AfterPane.<IntervalWindow>elementCountAtLeast(2), AccumulationMode.DISCARDING_FIRED_PANES, new SumIntegerFn().<String>asKeyedFn(), VarIntCoder.of(), Duration.millis(100)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(9))); assertThat( tester.extractOutput(), Matchers.contains(WindowMatchers.isSingleWindowedValue(Matchers.equalTo(3), 1, 0, 10))); // This element should not be output because that trigger (which was one-time) has already // gone off. tester.injectElements(TimestampedValue.of(6, new Instant(2))); assertThat(tester.extractOutput(), Matchers.emptyIterable()); assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10)))); assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20)))); tester.assertHasOnlyGlobalAndFinishedSetsFor( new IntervalWindow(new Instant(0), new Instant(10))); }
@Test public void testShouldFire() throws Exception { TriggerTester<Integer, IntervalWindow> tester = TriggerTester.forTrigger( new ReshuffleTrigger<IntervalWindow>(), FixedWindows.of(Duration.millis(100))); IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(300), new Instant(400)); assertTrue(tester.shouldFire(arbitraryWindow)); }
@Test public void testOnTimer() throws Exception { TriggerTester<Integer, IntervalWindow> tester = TriggerTester.forTrigger( new ReshuffleTrigger<IntervalWindow>(), FixedWindows.of(Duration.millis(100))); IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(100), new Instant(200)); tester.fireIfShouldFire(arbitraryWindow); assertFalse(tester.isMarkedFinished(arbitraryWindow)); }
public static void main(String[] args) { PipelineOptionsFactory.register(KafkaStreamingWordCountOptions.class); KafkaStreamingWordCountOptions options = PipelineOptionsFactory.fromArgs(args).as(KafkaStreamingWordCountOptions.class); options.setJobName("KafkaExample - WindowSize: " + options.getWindowSize() + " seconds"); options.setStreaming(true); options.setCheckpointingInterval(1000L); options.setNumberOfExecutionRetries(5); options.setExecutionRetryDelay(3000L); options.setRunner(FlinkPipelineRunner.class); System.out.println( options.getKafkaTopic() + " " + options.getZookeeper() + " " + options.getBroker() + " " + options.getGroup()); Pipeline pipeline = Pipeline.create(options); Properties p = new Properties(); p.setProperty("zookeeper.connect", options.getZookeeper()); p.setProperty("bootstrap.servers", options.getBroker()); p.setProperty("group.id", options.getGroup()); // this is the Flink consumer that reads the input to // the program from a kafka topic. FlinkKafkaConsumer08<String> kafkaConsumer = new FlinkKafkaConsumer08<>(options.getKafkaTopic(), new SimpleStringSchema(), p); PCollection<String> words = pipeline .apply(Read.from(new UnboundedFlinkSource<>(kafkaConsumer)).named("StreamingWordCount")) .apply(ParDo.of(new ExtractWordsFn())) .apply( Window.<String>into( FixedWindows.of(Duration.standardSeconds(options.getWindowSize()))) .triggering(AfterWatermark.pastEndOfWindow()) .withAllowedLateness(Duration.ZERO) .discardingFiredPanes()); PCollection<KV<String, Long>> wordCounts = words.apply(Count.<String>perElement()); wordCounts.apply(ParDo.of(new FormatAsStringFn())).apply(TextIO.Write.to("./outputKafka.txt")); pipeline.run(); }
@Test public void testTopEmptyWithIncompatibleWindows() { Pipeline p = TestPipeline.create(); Bound<String> windowingFn = Window.<String>into(FixedWindows.of(Duration.standardDays(10L))); PCollection<String> input = p.apply(Create.timestamped(Collections.<String>emptyList(), Collections.<Long>emptyList())) .apply(windowingFn); expectedEx.expect(IllegalStateException.class); expectedEx.expectMessage("Top"); expectedEx.expectMessage("GlobalWindows"); expectedEx.expectMessage("withoutDefaults"); expectedEx.expectMessage("asSingletonView"); input.apply(Top.of(1, new OrderByLength())); }
@Test public void testDefaultTriggerWithFixedWindow() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(Duration.millis(10)), DefaultTrigger.<IntervalWindow>of(), AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(9)), TimestampedValue.of(3, new Instant(15)), TimestampedValue.of(4, new Instant(19)), TimestampedValue.of(5, new Instant(30))); // Advance the watermark almost to the end of the first window. tester.advanceProcessingTime(new Instant(500)); tester.advanceInputWatermark(new Instant(8)); assertThat(tester.extractOutput(), Matchers.emptyIterable()); // Advance watermark to 10 (past end of the window), which causes the first fixed window to // be emitted tester.advanceInputWatermark(new Instant(10)); assertThat( tester.extractOutput(), Matchers.contains(isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10))); // Advance watermark to 100, which causes the remaining two windows to be emitted. // Since their timers were at different timestamps, they should fire in order. tester.advanceInputWatermark(new Instant(100)); assertThat( tester.extractOutput(), Matchers.contains( isSingleWindowedValue(Matchers.containsInAnyOrder(3, 4), 15, 10, 20), isSingleWindowedValue(Matchers.contains(5), 30, 30, 40))); assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(30), new Instant(40)))); tester.assertHasOnlyGlobalAndPaneInfoFor( new IntervalWindow(new Instant(0), new Instant(10)), new IntervalWindow(new Instant(10), new Instant(20)), new IntervalWindow(new Instant(30), new Instant(40))); }