public void testCloseAndReopenOrDeleteWithActiveScroll() throws IOException {
createIndex("test");
for (int i = 0; i < 100; i++) {
client()
.prepareIndex("test", "type1", Integer.toString(i))
.setSource(jsonBuilder().startObject().field("field", i).endObject())
.execute()
.actionGet();
}
refresh();
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(35)
.setScroll(TimeValue.timeValueMinutes(2))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
long counter = 0;
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
if (randomBoolean()) {
client().admin().indices().prepareClose("test").get();
client().admin().indices().prepareOpen("test").get();
ensureGreen("test");
} else {
client().admin().indices().prepareDelete("test").get();
}
}
public static CandidateResult parseIndexDocId(final SearchHit hit, boolean isGeo) {
final String documentId = hit.getId();
final double distance = isGeo ? (double) hit.sortValues()[0] : -1;
return parseIndexDocId(documentId, distance);
}
@Test
public void testParentWithMultipleBuckets() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("test")
.setQuery(matchQuery("randomized", false))
.addAggregation(
terms("category")
.field("category")
.size(0)
.subAggregation(
children("to_comment")
.childType("comment")
.subAggregation(topHits("top_comments").addSort("_id", SortOrder.ASC))))
.get();
assertSearchResponse(searchResponse);
Terms categoryTerms = searchResponse.getAggregations().get("category");
assertThat(categoryTerms.getBuckets().size(), equalTo(3));
for (Terms.Bucket bucket : categoryTerms.getBuckets()) {
logger.info("bucket=" + bucket.getKey());
Children childrenBucket = bucket.getAggregations().get("to_comment");
TopHits topHits = childrenBucket.getAggregations().get("top_comments");
logger.info("total_hits={}", topHits.getHits().getTotalHits());
for (SearchHit searchHit : topHits.getHits()) {
logger.info(
"hit= {} {} {}", searchHit.sortValues()[0], searchHit.getType(), searchHit.getId());
}
}
Terms.Bucket categoryBucket = categoryTerms.getBucketByKey("a");
assertThat(categoryBucket.getKey(), equalTo("a"));
assertThat(categoryBucket.getDocCount(), equalTo(3l));
Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(2l));
TopHits topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(2l));
assertThat(topHits.getHits().getAt(0).sortValues()[0].toString(), equalTo("a"));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("a"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
assertThat(topHits.getHits().getAt(1).sortValues()[0].toString(), equalTo("c"));
assertThat(topHits.getHits().getAt(1).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(1).getType(), equalTo("comment"));
categoryBucket = categoryTerms.getBucketByKey("b");
assertThat(categoryBucket.getKey(), equalTo("b"));
assertThat(categoryBucket.getDocCount(), equalTo(2l));
childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(1l));
topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(1l));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
categoryBucket = categoryTerms.getBucketByKey("c");
assertThat(categoryBucket.getKey(), equalTo("c"));
assertThat(categoryBucket.getDocCount(), equalTo(2l));
childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(1l));
topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(1l));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
}
@Test
public void testSimpleScrollQueryThenFetch_clearAllScrollIds() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
for (int i = 0; i < 100; i++) {
client()
.prepareIndex("test", "type1", Integer.toString(i))
.setSource(jsonBuilder().startObject().field("field", i).endObject())
.execute()
.actionGet();
}
client().admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse1 =
client()
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(35)
.setScroll(TimeValue.timeValueMinutes(2))
.setSearchType(SearchType.QUERY_THEN_FETCH)
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
SearchResponse searchResponse2 =
client()
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(35)
.setScroll(TimeValue.timeValueMinutes(2))
.setSearchType(SearchType.QUERY_THEN_FETCH)
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
long counter1 = 0;
long counter2 = 0;
assertThat(searchResponse1.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse1.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse1.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter1++));
}
assertThat(searchResponse2.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse2.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse2.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter2++));
}
searchResponse1 =
client()
.prepareSearchScroll(searchResponse1.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
searchResponse2 =
client()
.prepareSearchScroll(searchResponse2.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse1.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse1.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse1.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter1++));
}
assertThat(searchResponse2.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse2.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse2.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter2++));
}
ClearScrollResponse clearResponse =
client().prepareClearScroll().addScrollId("_all").execute().actionGet();
assertThat(clearResponse.isSucceeded(), is(true));
assertThat(clearResponse.getNumFreed(), greaterThan(0));
assertThat(clearResponse.status(), equalTo(RestStatus.OK));
assertThrows(
internalCluster()
.transportClient()
.prepareSearchScroll(searchResponse1.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2)),
RestStatus.NOT_FOUND);
assertThrows(
internalCluster()
.transportClient()
.prepareSearchScroll(searchResponse2.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2)),
RestStatus.NOT_FOUND);
}
@Test
public void testSimpleScrollQueryThenFetch() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
for (int i = 0; i < 100; i++) {
client()
.prepareIndex("test", "type1", Integer.toString(i))
.setSource(jsonBuilder().startObject().field("field", i).endObject())
.execute()
.actionGet();
}
client().admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(35)
.setScroll(TimeValue.timeValueMinutes(2))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
try {
long counter = 0;
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(30));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
} finally {
clearScroll(searchResponse.getScrollId());
}
}
@Test
public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
for (int i = 0; i < 100; i++) {
String routing = "0";
if (i > 90) {
routing = "1";
} else if (i > 60) {
routing = "2";
}
client()
.prepareIndex("test", "type1", Integer.toString(i))
.setSource("field", i)
.setRouting(routing)
.execute()
.actionGet();
}
client().admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse =
client()
.prepareSearch()
.setSearchType(SearchType.QUERY_THEN_FETCH)
.setQuery(matchAllQuery())
.setSize(3)
.setScroll(TimeValue.timeValueMinutes(2))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
try {
long counter = 0;
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
for (int i = 0; i < 32; i++) {
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
}
// and now, the last one is one
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(1));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
// a the last is zero
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(0));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
} finally {
clearScroll(searchResponse.getScrollId());
}
}