public void testCloseAndReopenOrDeleteWithActiveScroll() throws IOException {
createIndex("test");
for (int i = 0; i < 100; i++) {
client()
.prepareIndex("test", "type1", Integer.toString(i))
.setSource(jsonBuilder().startObject().field("field", i).endObject())
.execute()
.actionGet();
}
refresh();
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(35)
.setScroll(TimeValue.timeValueMinutes(2))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
long counter = 0;
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
if (randomBoolean()) {
client().admin().indices().prepareClose("test").get();
client().admin().indices().prepareOpen("test").get();
ensureGreen("test");
} else {
client().admin().indices().prepareDelete("test").get();
}
}
public void testDeepScrollingDoesNotBlowUp() throws Exception {
client()
.prepareIndex("index", "type", "1")
.setSource("field", "value")
.setRefresh(true)
.execute()
.get();
for (SearchType searchType : SearchType.values()) {
SearchRequestBuilder builder =
client()
.prepareSearch("index")
.setSearchType(searchType)
.setQuery(QueryBuilders.matchAllQuery())
.setSize(Integer.MAX_VALUE)
.setScroll("1m");
SearchResponse response = builder.execute().actionGet();
try {
ElasticsearchAssertions.assertHitCount(response, 1l);
} finally {
String scrollId = response.getScrollId();
if (scrollId != null) {
clearScroll(scrollId);
}
}
}
}
@Test
public void testThatNonExistingScrollIdReturnsCorrectException() throws Exception {
client().prepareIndex("index", "type", "1").setSource("field", "value").execute().get();
refresh();
SearchResponse searchResponse =
client().prepareSearch("index").setSize(1).setScroll("1m").get();
assertThat(searchResponse.getScrollId(), is(notNullValue()));
ClearScrollResponse clearScrollResponse =
client().prepareClearScroll().addScrollId(searchResponse.getScrollId()).get();
assertThat(clearScrollResponse.isSucceeded(), is(true));
assertThrows(
internalCluster().transportClient().prepareSearchScroll(searchResponse.getScrollId()),
RestStatus.NOT_FOUND);
}
@Test
public void testStringSortMissingAscTerminates() throws Exception {
assertAcked(
prepareCreate("test")
.setSettings(
Settings.settingsBuilder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
.addMapping("test", "no_field", "type=string", "some_field", "type=string"));
client().prepareIndex("test", "test", "1").setSource("some_field", "test").get();
refresh();
SearchResponse response =
client()
.prepareSearch("test")
.setTypes("test")
.addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_last"))
.setScroll("1m")
.get();
assertHitCount(response, 1);
assertSearchHits(response, "1");
response = client().prepareSearchScroll(response.getScrollId()).get();
assertSearchResponse(response);
assertHitCount(response, 1);
assertNoSearchHits(response);
response =
client()
.prepareSearch("test")
.setTypes("test")
.addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_first"))
.setScroll("1m")
.get();
assertHitCount(response, 1);
assertSearchHits(response, "1");
response = client().prepareSearchScroll(response.getScrollId()).get();
assertHitCount(response, 1);
assertThat(response.getHits().getHits().length, equalTo(0));
}
@Test
public void testSimpleScrollQueryThenFetch_clearAllScrollIds() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
for (int i = 0; i < 100; i++) {
client()
.prepareIndex("test", "type1", Integer.toString(i))
.setSource(jsonBuilder().startObject().field("field", i).endObject())
.execute()
.actionGet();
}
client().admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse1 =
client()
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(35)
.setScroll(TimeValue.timeValueMinutes(2))
.setSearchType(SearchType.QUERY_THEN_FETCH)
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
SearchResponse searchResponse2 =
client()
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(35)
.setScroll(TimeValue.timeValueMinutes(2))
.setSearchType(SearchType.QUERY_THEN_FETCH)
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
long counter1 = 0;
long counter2 = 0;
assertThat(searchResponse1.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse1.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse1.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter1++));
}
assertThat(searchResponse2.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse2.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse2.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter2++));
}
searchResponse1 =
client()
.prepareSearchScroll(searchResponse1.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
searchResponse2 =
client()
.prepareSearchScroll(searchResponse2.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse1.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse1.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse1.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter1++));
}
assertThat(searchResponse2.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse2.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse2.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter2++));
}
ClearScrollResponse clearResponse =
client().prepareClearScroll().addScrollId("_all").execute().actionGet();
assertThat(clearResponse.isSucceeded(), is(true));
assertThat(clearResponse.getNumFreed(), greaterThan(0));
assertThat(clearResponse.status(), equalTo(RestStatus.OK));
assertThrows(
internalCluster()
.transportClient()
.prepareSearchScroll(searchResponse1.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2)),
RestStatus.NOT_FOUND);
assertThrows(
internalCluster()
.transportClient()
.prepareSearchScroll(searchResponse2.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2)),
RestStatus.NOT_FOUND);
}
@Test
public void testSimpleScrollQueryThenFetch() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
for (int i = 0; i < 100; i++) {
client()
.prepareIndex("test", "type1", Integer.toString(i))
.setSource(jsonBuilder().startObject().field("field", i).endObject())
.execute()
.actionGet();
}
client().admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(35)
.setScroll(TimeValue.timeValueMinutes(2))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
try {
long counter = 0;
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(35));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(30));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
} finally {
clearScroll(searchResponse.getScrollId());
}
}
@Test
public void testScrollAndUpdateIndex() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(Settings.settingsBuilder().put("index.number_of_shards", 5))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
for (int i = 0; i < 500; i++) {
client()
.prepareIndex("test", "tweet", Integer.toString(i))
.setSource(
jsonBuilder()
.startObject()
.field("user", "kimchy")
.field("postDate", System.currentTimeMillis())
.field("message", "test")
.endObject())
.execute()
.actionGet();
}
client().admin().indices().prepareRefresh().execute().actionGet();
assertThat(
client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(),
equalTo(500l));
assertThat(
client()
.prepareCount()
.setQuery(termQuery("message", "test"))
.execute()
.actionGet()
.getCount(),
equalTo(500l));
assertThat(
client()
.prepareCount()
.setQuery(termQuery("message", "test"))
.execute()
.actionGet()
.getCount(),
equalTo(500l));
assertThat(
client()
.prepareCount()
.setQuery(termQuery("message", "update"))
.execute()
.actionGet()
.getCount(),
equalTo(0l));
assertThat(
client()
.prepareCount()
.setQuery(termQuery("message", "update"))
.execute()
.actionGet()
.getCount(),
equalTo(0l));
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(queryStringQuery("user:kimchy"))
.setSize(35)
.setScroll(TimeValue.timeValueMinutes(2))
.addSort("postDate", SortOrder.ASC)
.execute()
.actionGet();
try {
do {
for (SearchHit searchHit : searchResponse.getHits().hits()) {
Map<String, Object> map = searchHit.sourceAsMap();
map.put("message", "update");
client()
.prepareIndex("test", "tweet", searchHit.id())
.setSource(map)
.execute()
.actionGet();
}
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
} while (searchResponse.getHits().hits().length > 0);
client().admin().indices().prepareRefresh().execute().actionGet();
assertThat(
client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(),
equalTo(500l));
assertThat(
client()
.prepareCount()
.setQuery(termQuery("message", "test"))
.execute()
.actionGet()
.getCount(),
equalTo(0l));
assertThat(
client()
.prepareCount()
.setQuery(termQuery("message", "test"))
.execute()
.actionGet()
.getCount(),
equalTo(0l));
assertThat(
client()
.prepareCount()
.setQuery(termQuery("message", "update"))
.execute()
.actionGet()
.getCount(),
equalTo(500l));
assertThat(
client()
.prepareCount()
.setQuery(termQuery("message", "update"))
.execute()
.actionGet()
.getCount(),
equalTo(500l));
} finally {
clearScroll(searchResponse.getScrollId());
}
}
@Test
public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
for (int i = 0; i < 100; i++) {
String routing = "0";
if (i > 90) {
routing = "1";
} else if (i > 60) {
routing = "2";
}
client()
.prepareIndex("test", "type1", Integer.toString(i))
.setSource("field", i)
.setRouting(routing)
.execute()
.actionGet();
}
client().admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse =
client()
.prepareSearch()
.setSearchType(SearchType.QUERY_THEN_FETCH)
.setQuery(matchAllQuery())
.setSize(3)
.setScroll(TimeValue.timeValueMinutes(2))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
try {
long counter = 0;
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
for (int i = 0; i < 32; i++) {
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
}
// and now, the last one is one
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(1));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
// a the last is zero
searchResponse =
client()
.prepareSearchScroll(searchResponse.getScrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
assertThat(searchResponse.getHits().hits().length, equalTo(0));
for (SearchHit hit : searchResponse.getHits()) {
assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
}
} finally {
clearScroll(searchResponse.getScrollId());
}
}