Exemplo n.º 1
0
  @Test
  public void testAfterPaneWithFixedWindow() throws Exception {
    Duration windowDuration = Duration.millis(10);
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
        ReduceFnTester.nonCombining(
            FixedWindows.of(windowDuration),
            AfterPane.<IntervalWindow>elementCountAtLeast(2),
            AccumulationMode.DISCARDING_FIRED_PANES,
            Duration.millis(100));

    tester.injectElements(
        TimestampedValue.of(1, new Instant(1)), // first in window [0, 10)
        TimestampedValue.of(2, new Instant(9)));

    assertThat(
        tester.extractOutput(),
        Matchers.contains(
            WindowMatchers.isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10)));

    // This element belongs in the window that has already fired. It should not be re-output because
    // that trigger (which was one-time) has already gone off.
    tester.injectElements(TimestampedValue.of(6, new Instant(2)));
    assertThat(tester.extractOutput(), Matchers.emptyIterable());

    assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10))));
    assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20))));

    tester.assertHasOnlyGlobalAndFinishedSetsFor(
        new IntervalWindow(new Instant(0), new Instant(10)));
  }
  @Test
  public void testAtWatermarkAndLate() throws Exception {
    tester =
        TriggerTester.forTrigger(
            AfterWatermark.<IntervalWindow>pastEndOfWindow().withLateFirings(mockLate),
            FixedWindows.of(Duration.millis(100)));

    injectElements(1);
    IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));

    // No early firing, just double checking
    when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(true);
    assertFalse(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.isMarkedFinished(window));

    // Fire due to watermark
    when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false);
    tester.advanceInputWatermark(new Instant(100));
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.isMarkedFinished(window));

    testRunningAsTrigger(mockLate, window);
  }
Exemplo n.º 3
0
  @Test
  public void testAfterPaneWithGlobalWindowsAndCombining() throws Exception {
    Duration windowDuration = Duration.millis(10);
    ReduceFnTester<Integer, Integer, IntervalWindow> tester =
        ReduceFnTester.combining(
            FixedWindows.of(windowDuration),
            AfterPane.<IntervalWindow>elementCountAtLeast(2),
            AccumulationMode.DISCARDING_FIRED_PANES,
            new SumIntegerFn().<String>asKeyedFn(),
            VarIntCoder.of(),
            Duration.millis(100));

    tester.injectElements(
        TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(9)));

    assertThat(
        tester.extractOutput(),
        Matchers.contains(WindowMatchers.isSingleWindowedValue(Matchers.equalTo(3), 1, 0, 10)));

    // This element should not be output because that trigger (which was one-time) has already
    // gone off.
    tester.injectElements(TimestampedValue.of(6, new Instant(2)));
    assertThat(tester.extractOutput(), Matchers.emptyIterable());

    assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10))));
    assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20))));

    tester.assertHasOnlyGlobalAndFinishedSetsFor(
        new IntervalWindow(new Instant(0), new Instant(10)));
  }
 @Test
 public void testShouldFire() throws Exception {
   TriggerTester<Integer, IntervalWindow> tester =
       TriggerTester.forTrigger(
           new ReshuffleTrigger<IntervalWindow>(), FixedWindows.of(Duration.millis(100)));
   IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(300), new Instant(400));
   assertTrue(tester.shouldFire(arbitraryWindow));
 }
 @Test
 public void testOnTimer() throws Exception {
   TriggerTester<Integer, IntervalWindow> tester =
       TriggerTester.forTrigger(
           new ReshuffleTrigger<IntervalWindow>(), FixedWindows.of(Duration.millis(100)));
   IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(100), new Instant(200));
   tester.fireIfShouldFire(arbitraryWindow);
   assertFalse(tester.isMarkedFinished(arbitraryWindow));
 }
  public static void main(String[] args) {
    PipelineOptionsFactory.register(KafkaStreamingWordCountOptions.class);
    KafkaStreamingWordCountOptions options =
        PipelineOptionsFactory.fromArgs(args).as(KafkaStreamingWordCountOptions.class);
    options.setJobName("KafkaExample - WindowSize: " + options.getWindowSize() + " seconds");
    options.setStreaming(true);
    options.setCheckpointingInterval(1000L);
    options.setNumberOfExecutionRetries(5);
    options.setExecutionRetryDelay(3000L);
    options.setRunner(FlinkPipelineRunner.class);

    System.out.println(
        options.getKafkaTopic()
            + " "
            + options.getZookeeper()
            + " "
            + options.getBroker()
            + " "
            + options.getGroup());
    Pipeline pipeline = Pipeline.create(options);

    Properties p = new Properties();
    p.setProperty("zookeeper.connect", options.getZookeeper());
    p.setProperty("bootstrap.servers", options.getBroker());
    p.setProperty("group.id", options.getGroup());

    // this is the Flink consumer that reads the input to
    // the program from a kafka topic.
    FlinkKafkaConsumer08<String> kafkaConsumer =
        new FlinkKafkaConsumer08<>(options.getKafkaTopic(), new SimpleStringSchema(), p);

    PCollection<String> words =
        pipeline
            .apply(Read.from(new UnboundedFlinkSource<>(kafkaConsumer)).named("StreamingWordCount"))
            .apply(ParDo.of(new ExtractWordsFn()))
            .apply(
                Window.<String>into(
                        FixedWindows.of(Duration.standardSeconds(options.getWindowSize())))
                    .triggering(AfterWatermark.pastEndOfWindow())
                    .withAllowedLateness(Duration.ZERO)
                    .discardingFiredPanes());

    PCollection<KV<String, Long>> wordCounts = words.apply(Count.<String>perElement());

    wordCounts.apply(ParDo.of(new FormatAsStringFn())).apply(TextIO.Write.to("./outputKafka.txt"));

    pipeline.run();
  }
Exemplo n.º 7
0
  @Test
  public void testTopEmptyWithIncompatibleWindows() {
    Pipeline p = TestPipeline.create();
    Bound<String> windowingFn = Window.<String>into(FixedWindows.of(Duration.standardDays(10L)));
    PCollection<String> input =
        p.apply(Create.timestamped(Collections.<String>emptyList(), Collections.<Long>emptyList()))
            .apply(windowingFn);

    expectedEx.expect(IllegalStateException.class);
    expectedEx.expectMessage("Top");
    expectedEx.expectMessage("GlobalWindows");
    expectedEx.expectMessage("withoutDefaults");
    expectedEx.expectMessage("asSingletonView");

    input.apply(Top.of(1, new OrderByLength()));
  }
Exemplo n.º 8
0
  @Test
  public void testDefaultTriggerWithFixedWindow() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
        ReduceFnTester.nonCombining(
            FixedWindows.of(Duration.millis(10)),
            DefaultTrigger.<IntervalWindow>of(),
            AccumulationMode.DISCARDING_FIRED_PANES,
            Duration.millis(100));

    tester.injectElements(
        TimestampedValue.of(1, new Instant(1)),
        TimestampedValue.of(2, new Instant(9)),
        TimestampedValue.of(3, new Instant(15)),
        TimestampedValue.of(4, new Instant(19)),
        TimestampedValue.of(5, new Instant(30)));

    // Advance the watermark almost to the end of the first window.
    tester.advanceProcessingTime(new Instant(500));
    tester.advanceInputWatermark(new Instant(8));
    assertThat(tester.extractOutput(), Matchers.emptyIterable());

    // Advance watermark to 10 (past end of the window), which causes the first fixed window to
    // be emitted
    tester.advanceInputWatermark(new Instant(10));
    assertThat(
        tester.extractOutput(),
        Matchers.contains(isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10)));

    // Advance watermark to 100, which causes the remaining two windows to be emitted.
    // Since their timers were at different timestamps, they should fire in order.
    tester.advanceInputWatermark(new Instant(100));
    assertThat(
        tester.extractOutput(),
        Matchers.contains(
            isSingleWindowedValue(Matchers.containsInAnyOrder(3, 4), 15, 10, 20),
            isSingleWindowedValue(Matchers.contains(5), 30, 30, 40)));
    assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(30), new Instant(40))));
    tester.assertHasOnlyGlobalAndPaneInfoFor(
        new IntervalWindow(new Instant(0), new Instant(10)),
        new IntervalWindow(new Instant(10), new Instant(20)),
        new IntervalWindow(new Instant(30), new Instant(40)));
  }