示例#1
0
  @Test
  public void testAfterPaneWithGlobalWindowsAndCombining() throws Exception {
    Duration windowDuration = Duration.millis(10);
    ReduceFnTester<Integer, Integer, IntervalWindow> tester =
        ReduceFnTester.combining(
            FixedWindows.of(windowDuration),
            AfterPane.<IntervalWindow>elementCountAtLeast(2),
            AccumulationMode.DISCARDING_FIRED_PANES,
            new SumIntegerFn().<String>asKeyedFn(),
            VarIntCoder.of(),
            Duration.millis(100));

    tester.injectElements(
        TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(9)));

    assertThat(
        tester.extractOutput(),
        Matchers.contains(WindowMatchers.isSingleWindowedValue(Matchers.equalTo(3), 1, 0, 10)));

    // This element should not be output because that trigger (which was one-time) has already
    // gone off.
    tester.injectElements(TimestampedValue.of(6, new Instant(2)));
    assertThat(tester.extractOutput(), Matchers.emptyIterable());

    assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10))));
    assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20))));

    tester.assertHasOnlyGlobalAndFinishedSetsFor(
        new IntervalWindow(new Instant(0), new Instant(10)));
  }
示例#2
0
  @Test
  public void testAfterPaneWithFixedWindow() throws Exception {
    Duration windowDuration = Duration.millis(10);
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
        ReduceFnTester.nonCombining(
            FixedWindows.of(windowDuration),
            AfterPane.<IntervalWindow>elementCountAtLeast(2),
            AccumulationMode.DISCARDING_FIRED_PANES,
            Duration.millis(100));

    tester.injectElements(
        TimestampedValue.of(1, new Instant(1)), // first in window [0, 10)
        TimestampedValue.of(2, new Instant(9)));

    assertThat(
        tester.extractOutput(),
        Matchers.contains(
            WindowMatchers.isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10)));

    // This element belongs in the window that has already fired. It should not be re-output because
    // that trigger (which was one-time) has already gone off.
    tester.injectElements(TimestampedValue.of(6, new Instant(2)));
    assertThat(tester.extractOutput(), Matchers.emptyIterable());

    assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10))));
    assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20))));

    tester.assertHasOnlyGlobalAndFinishedSetsFor(
        new IntervalWindow(new Instant(0), new Instant(10)));
  }
  @Test
  public void testDefaultTriggerWithSessionWindow() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
        ReduceFnTester.nonCombining(
            Sessions.withGapDuration(Duration.millis(10)),
            DefaultTrigger.<IntervalWindow>of(),
            AccumulationMode.DISCARDING_FIRED_PANES,
            Duration.millis(100));

    tester.injectElements(
        TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(9)));

    // no output, because we merged into the [9-19) session
    tester.advanceInputWatermark(new Instant(10));
    assertThat(tester.extractOutput(), Matchers.emptyIterable());

    tester.injectElements(
        TimestampedValue.of(3, new Instant(15)), TimestampedValue.of(4, new Instant(30)));

    tester.advanceInputWatermark(new Instant(100));
    assertThat(
        tester.extractOutput(),
        Matchers.contains(
            isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2, 3), 1, 1, 25),
            isSingleWindowedValue(Matchers.contains(4), 30, 30, 40)));
    assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(1), new Instant(25))));
    assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(30), new Instant(40))));
    tester.assertHasOnlyGlobalAndPaneInfoFor(
        new IntervalWindow(new Instant(1), new Instant(25)),
        new IntervalWindow(new Instant(30), new Instant(40)));
  }
  @Test
  public void testDefaultTriggerWithFixedWindow() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
        ReduceFnTester.nonCombining(
            FixedWindows.of(Duration.millis(10)),
            DefaultTrigger.<IntervalWindow>of(),
            AccumulationMode.DISCARDING_FIRED_PANES,
            Duration.millis(100));

    tester.injectElements(
        TimestampedValue.of(1, new Instant(1)),
        TimestampedValue.of(2, new Instant(9)),
        TimestampedValue.of(3, new Instant(15)),
        TimestampedValue.of(4, new Instant(19)),
        TimestampedValue.of(5, new Instant(30)));

    // Advance the watermark almost to the end of the first window.
    tester.advanceProcessingTime(new Instant(500));
    tester.advanceInputWatermark(new Instant(8));
    assertThat(tester.extractOutput(), Matchers.emptyIterable());

    // Advance watermark to 10 (past end of the window), which causes the first fixed window to
    // be emitted
    tester.advanceInputWatermark(new Instant(10));
    assertThat(
        tester.extractOutput(),
        Matchers.contains(isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10)));

    // Advance watermark to 100, which causes the remaining two windows to be emitted.
    // Since their timers were at different timestamps, they should fire in order.
    tester.advanceInputWatermark(new Instant(100));
    assertThat(
        tester.extractOutput(),
        Matchers.contains(
            isSingleWindowedValue(Matchers.containsInAnyOrder(3, 4), 15, 10, 20),
            isSingleWindowedValue(Matchers.contains(5), 30, 30, 40)));
    assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(30), new Instant(40))));
    tester.assertHasOnlyGlobalAndPaneInfoFor(
        new IntervalWindow(new Instant(0), new Instant(10)),
        new IntervalWindow(new Instant(10), new Instant(20)),
        new IntervalWindow(new Instant(30), new Instant(40)));
  }
示例#5
0
  @Test
  public void testAfterPaneWithMerging() throws Exception {
    Duration windowDuration = Duration.millis(10);
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
        ReduceFnTester.nonCombining(
            Sessions.withGapDuration(windowDuration),
            AfterPane.<IntervalWindow>elementCountAtLeast(2),
            AccumulationMode.DISCARDING_FIRED_PANES,
            Duration.millis(100));

    assertThat(tester.extractOutput(), Matchers.emptyIterable());

    tester.injectElements(
        TimestampedValue.of(1, new Instant(1)), // in [1, 11)
        TimestampedValue.of(2, new Instant(2))); // in [2, 12)

    assertThat(
        tester.extractOutput(),
        Matchers.contains(
            WindowMatchers.isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 1, 12)));

    // Because we closed the previous window, we don't have it around to merge with.
    tester.injectElements(
        TimestampedValue.of(3, new Instant(7)), // in [7, 17)
        TimestampedValue.of(4, new Instant(8))); // in [8, 18)

    assertThat(
        tester.extractOutput(),
        Matchers.contains(
            WindowMatchers.isSingleWindowedValue(Matchers.containsInAnyOrder(3, 4), 7, 7, 18)));

    assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(1), new Instant(12))));

    tester.assertHasOnlyGlobalAndFinishedSetsFor(
        new IntervalWindow(new Instant(1), new Instant(12)),
        new IntervalWindow(new Instant(7), new Instant(18)));
  }
示例#6
0
 /**
  * GithubProfile can accept asset from repo name that contains a dot.
  *
  * @throws Exception In case of error.
  */
 @Test
 public void acceptsAssetsFromDotRepo() throws Exception {
   final Github github = new MkGithub("jeff");
   final String name = "te.st";
   final Repo repo = github.repos().create(new Repos.RepoCreate(name, false));
   repo.contents()
       .create(
           Json.createObjectBuilder()
               .add("path", ".rultor.yml")
               .add("message", "just test")
               .add(
                   "content",
                   Base64.encodeBase64String(
                       Joiner.on('\n')
                           .join(
                               "assets: ",
                               String.format("  something.xml: jeff/%s#.rultor.yml", name),
                               "friends:",
                               String.format("  - jeff/%s", name))
                           .getBytes()))
               .build());
   MatcherAssert.assertThat(
       new GithubProfile(repo).assets().entrySet(), Matchers.not(Matchers.emptyIterable()));
 }
 /**
  * MkGitignores can iterate over templates.
  *
  * @throws Exception if something goes wrong.
  */
 @Test
 public void canIterateOverTemplates() throws Exception {
   final Gitignores gitignores = new MkGithub().gitignores();
   MatcherAssert.assertThat(gitignores.iterate(), Matchers.not(Matchers.emptyIterable()));
 }
 @Test
 public void testSearchAndRelocateConcurrently() throws Exception {
   final int numShards = between(10, 20);
   String mapping =
       XContentFactory.jsonBuilder()
           .startObject()
           .startObject("type")
           .startObject("properties")
           .startObject("loc")
           .field("type", "geo_point")
           .endObject()
           .startObject("test")
           .field("type", "string")
           .endObject()
           .endObject()
           .endObject()
           .endObject()
           .string();
   client()
       .admin()
       .indices()
       .prepareCreate("test")
       .setSettings(
           settingsBuilder()
               .put("index.number_of_shards", numShards)
               .put("index.number_of_replicas", 0))
       .addMapping("type1", mapping)
       .execute()
       .actionGet();
   ensureYellow();
   List<IndexRequestBuilder> indexBuilders = new ArrayList<IndexRequestBuilder>();
   final int numDocs = between(10, 20);
   for (int i = 0; i < numDocs; i++) {
     indexBuilders.add(
         new IndexRequestBuilder(client())
             .setType("type")
             .setId(Integer.toString(i))
             .setIndex("test")
             .setSource(
                 jsonBuilder()
                     .startObject()
                     .field("test", "value")
                     .startObject("loc")
                     .field("lat", 11)
                     .field("lon", 21)
                     .endObject()
                     .endObject()));
   }
   indexRandom("test", true, indexBuilders.toArray(new IndexRequestBuilder[indexBuilders.size()]));
   final int numIters = atLeast(3);
   for (int i = 0; i < numIters; i++) {
     allowNodes("test", between(1, 3));
     client().admin().cluster().prepareReroute().get();
     final AtomicBoolean stop = new AtomicBoolean(false);
     final List<Throwable> thrownExceptions = new CopyOnWriteArrayList<Throwable>();
     final Thread t =
         new Thread() {
           public void run() {
             final List<Float> lonlat = new ArrayList<Float>();
             lonlat.add(new Float(20));
             lonlat.add(new Float(11));
             try {
               while (!stop.get()) {
                 SearchResponse sr =
                     client()
                         .search(
                             searchRequest()
                                 .searchType(SearchType.QUERY_THEN_FETCH)
                                 .source(
                                     searchSource()
                                         .size(numDocs)
                                         .query(
                                             functionScoreQuery(
                                                     termQuery("test", "value"),
                                                     gaussDecayFunction("loc", lonlat, "1000km"))
                                                 .boostMode(CombineFunction.MULT.getName()))))
                         .get();
                 final SearchHits sh = sr.getHits();
                 assertThat(
                     "Expect num docs in getTotalHits() ",
                     sh.getTotalHits(),
                     equalTo((long) (numDocs)));
                 assertThat(
                     "Expected hits to be the same size the actual hits array",
                     sh.getTotalHits(),
                     equalTo((long) (sh.getHits().length)));
               }
             } catch (Throwable t) {
               thrownExceptions.add(t);
             }
           }
         };
     t.start();
     ClusterHealthResponse resp =
         client()
             .admin()
             .cluster()
             .prepareHealth()
             .setWaitForRelocatingShards(0)
             .execute()
             .actionGet();
     stop.set(true);
     t.join();
     assertThat(resp.isTimedOut(), equalTo(false));
     assertThat("failed in iteration " + i, thrownExceptions, Matchers.emptyIterable());
   }
 }