@Test
public void testAfterPaneWithGlobalWindowsAndCombining() throws Exception {
Duration windowDuration = Duration.millis(10);
ReduceFnTester<Integer, Integer, IntervalWindow> tester =
ReduceFnTester.combining(
FixedWindows.of(windowDuration),
AfterPane.<IntervalWindow>elementCountAtLeast(2),
AccumulationMode.DISCARDING_FIRED_PANES,
new SumIntegerFn().<String>asKeyedFn(),
VarIntCoder.of(),
Duration.millis(100));
tester.injectElements(
TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(9)));
assertThat(
tester.extractOutput(),
Matchers.contains(WindowMatchers.isSingleWindowedValue(Matchers.equalTo(3), 1, 0, 10)));
// This element should not be output because that trigger (which was one-time) has already
// gone off.
tester.injectElements(TimestampedValue.of(6, new Instant(2)));
assertThat(tester.extractOutput(), Matchers.emptyIterable());
assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10))));
assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20))));
tester.assertHasOnlyGlobalAndFinishedSetsFor(
new IntervalWindow(new Instant(0), new Instant(10)));
}
@Test
public void testAfterPaneWithFixedWindow() throws Exception {
Duration windowDuration = Duration.millis(10);
ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
ReduceFnTester.nonCombining(
FixedWindows.of(windowDuration),
AfterPane.<IntervalWindow>elementCountAtLeast(2),
AccumulationMode.DISCARDING_FIRED_PANES,
Duration.millis(100));
tester.injectElements(
TimestampedValue.of(1, new Instant(1)), // first in window [0, 10)
TimestampedValue.of(2, new Instant(9)));
assertThat(
tester.extractOutput(),
Matchers.contains(
WindowMatchers.isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10)));
// This element belongs in the window that has already fired. It should not be re-output because
// that trigger (which was one-time) has already gone off.
tester.injectElements(TimestampedValue.of(6, new Instant(2)));
assertThat(tester.extractOutput(), Matchers.emptyIterable());
assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(0), new Instant(10))));
assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20))));
tester.assertHasOnlyGlobalAndFinishedSetsFor(
new IntervalWindow(new Instant(0), new Instant(10)));
}
@Test
public void testDefaultTriggerWithSessionWindow() throws Exception {
ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
ReduceFnTester.nonCombining(
Sessions.withGapDuration(Duration.millis(10)),
DefaultTrigger.<IntervalWindow>of(),
AccumulationMode.DISCARDING_FIRED_PANES,
Duration.millis(100));
tester.injectElements(
TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(9)));
// no output, because we merged into the [9-19) session
tester.advanceInputWatermark(new Instant(10));
assertThat(tester.extractOutput(), Matchers.emptyIterable());
tester.injectElements(
TimestampedValue.of(3, new Instant(15)), TimestampedValue.of(4, new Instant(30)));
tester.advanceInputWatermark(new Instant(100));
assertThat(
tester.extractOutput(),
Matchers.contains(
isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2, 3), 1, 1, 25),
isSingleWindowedValue(Matchers.contains(4), 30, 30, 40)));
assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(1), new Instant(25))));
assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(30), new Instant(40))));
tester.assertHasOnlyGlobalAndPaneInfoFor(
new IntervalWindow(new Instant(1), new Instant(25)),
new IntervalWindow(new Instant(30), new Instant(40)));
}
@Test
public void testDefaultTriggerWithFixedWindow() throws Exception {
ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
ReduceFnTester.nonCombining(
FixedWindows.of(Duration.millis(10)),
DefaultTrigger.<IntervalWindow>of(),
AccumulationMode.DISCARDING_FIRED_PANES,
Duration.millis(100));
tester.injectElements(
TimestampedValue.of(1, new Instant(1)),
TimestampedValue.of(2, new Instant(9)),
TimestampedValue.of(3, new Instant(15)),
TimestampedValue.of(4, new Instant(19)),
TimestampedValue.of(5, new Instant(30)));
// Advance the watermark almost to the end of the first window.
tester.advanceProcessingTime(new Instant(500));
tester.advanceInputWatermark(new Instant(8));
assertThat(tester.extractOutput(), Matchers.emptyIterable());
// Advance watermark to 10 (past end of the window), which causes the first fixed window to
// be emitted
tester.advanceInputWatermark(new Instant(10));
assertThat(
tester.extractOutput(),
Matchers.contains(isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 0, 10)));
// Advance watermark to 100, which causes the remaining two windows to be emitted.
// Since their timers were at different timestamps, they should fire in order.
tester.advanceInputWatermark(new Instant(100));
assertThat(
tester.extractOutput(),
Matchers.contains(
isSingleWindowedValue(Matchers.containsInAnyOrder(3, 4), 15, 10, 20),
isSingleWindowedValue(Matchers.contains(5), 30, 30, 40)));
assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(30), new Instant(40))));
tester.assertHasOnlyGlobalAndPaneInfoFor(
new IntervalWindow(new Instant(0), new Instant(10)),
new IntervalWindow(new Instant(10), new Instant(20)),
new IntervalWindow(new Instant(30), new Instant(40)));
}
@Test
public void testAfterPaneWithMerging() throws Exception {
Duration windowDuration = Duration.millis(10);
ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
ReduceFnTester.nonCombining(
Sessions.withGapDuration(windowDuration),
AfterPane.<IntervalWindow>elementCountAtLeast(2),
AccumulationMode.DISCARDING_FIRED_PANES,
Duration.millis(100));
assertThat(tester.extractOutput(), Matchers.emptyIterable());
tester.injectElements(
TimestampedValue.of(1, new Instant(1)), // in [1, 11)
TimestampedValue.of(2, new Instant(2))); // in [2, 12)
assertThat(
tester.extractOutput(),
Matchers.contains(
WindowMatchers.isSingleWindowedValue(Matchers.containsInAnyOrder(1, 2), 1, 1, 12)));
// Because we closed the previous window, we don't have it around to merge with.
tester.injectElements(
TimestampedValue.of(3, new Instant(7)), // in [7, 17)
TimestampedValue.of(4, new Instant(8))); // in [8, 18)
assertThat(
tester.extractOutput(),
Matchers.contains(
WindowMatchers.isSingleWindowedValue(Matchers.containsInAnyOrder(3, 4), 7, 7, 18)));
assertTrue(tester.isMarkedFinished(new IntervalWindow(new Instant(1), new Instant(12))));
tester.assertHasOnlyGlobalAndFinishedSetsFor(
new IntervalWindow(new Instant(1), new Instant(12)),
new IntervalWindow(new Instant(7), new Instant(18)));
}
/**
* GithubProfile can accept asset from repo name that contains a dot.
*
* @throws Exception In case of error.
*/
@Test
public void acceptsAssetsFromDotRepo() throws Exception {
final Github github = new MkGithub("jeff");
final String name = "te.st";
final Repo repo = github.repos().create(new Repos.RepoCreate(name, false));
repo.contents()
.create(
Json.createObjectBuilder()
.add("path", ".rultor.yml")
.add("message", "just test")
.add(
"content",
Base64.encodeBase64String(
Joiner.on('\n')
.join(
"assets: ",
String.format(" something.xml: jeff/%s#.rultor.yml", name),
"friends:",
String.format(" - jeff/%s", name))
.getBytes()))
.build());
MatcherAssert.assertThat(
new GithubProfile(repo).assets().entrySet(), Matchers.not(Matchers.emptyIterable()));
}
/**
* MkGitignores can iterate over templates.
*
* @throws Exception if something goes wrong.
*/
@Test
public void canIterateOverTemplates() throws Exception {
final Gitignores gitignores = new MkGithub().gitignores();
MatcherAssert.assertThat(gitignores.iterate(), Matchers.not(Matchers.emptyIterable()));
}
@Test
public void testSearchAndRelocateConcurrently() throws Exception {
final int numShards = between(10, 20);
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("loc")
.field("type", "geo_point")
.endObject()
.startObject("test")
.field("type", "string")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
settingsBuilder()
.put("index.number_of_shards", numShards)
.put("index.number_of_replicas", 0))
.addMapping("type1", mapping)
.execute()
.actionGet();
ensureYellow();
List<IndexRequestBuilder> indexBuilders = new ArrayList<IndexRequestBuilder>();
final int numDocs = between(10, 20);
for (int i = 0; i < numDocs; i++) {
indexBuilders.add(
new IndexRequestBuilder(client())
.setType("type")
.setId(Integer.toString(i))
.setIndex("test")
.setSource(
jsonBuilder()
.startObject()
.field("test", "value")
.startObject("loc")
.field("lat", 11)
.field("lon", 21)
.endObject()
.endObject()));
}
indexRandom("test", true, indexBuilders.toArray(new IndexRequestBuilder[indexBuilders.size()]));
final int numIters = atLeast(3);
for (int i = 0; i < numIters; i++) {
allowNodes("test", between(1, 3));
client().admin().cluster().prepareReroute().get();
final AtomicBoolean stop = new AtomicBoolean(false);
final List<Throwable> thrownExceptions = new CopyOnWriteArrayList<Throwable>();
final Thread t =
new Thread() {
public void run() {
final List<Float> lonlat = new ArrayList<Float>();
lonlat.add(new Float(20));
lonlat.add(new Float(11));
try {
while (!stop.get()) {
SearchResponse sr =
client()
.search(
searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(
searchSource()
.size(numDocs)
.query(
functionScoreQuery(
termQuery("test", "value"),
gaussDecayFunction("loc", lonlat, "1000km"))
.boostMode(CombineFunction.MULT.getName()))))
.get();
final SearchHits sh = sr.getHits();
assertThat(
"Expect num docs in getTotalHits() ",
sh.getTotalHits(),
equalTo((long) (numDocs)));
assertThat(
"Expected hits to be the same size the actual hits array",
sh.getTotalHits(),
equalTo((long) (sh.getHits().length)));
}
} catch (Throwable t) {
thrownExceptions.add(t);
}
}
};
t.start();
ClusterHealthResponse resp =
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForRelocatingShards(0)
.execute()
.actionGet();
stop.set(true);
t.join();
assertThat(resp.isTimedOut(), equalTo(false));
assertThat("failed in iteration " + i, thrownExceptions, Matchers.emptyIterable());
}
}