public void testCloseAndReopenOrDeleteWithActiveScroll() throws IOException {
   createIndex("test");
   for (int i = 0; i < 100; i++) {
     client()
         .prepareIndex("test", "type1", Integer.toString(i))
         .setSource(jsonBuilder().startObject().field("field", i).endObject())
         .execute()
         .actionGet();
   }
   refresh();
   SearchResponse searchResponse =
       client()
           .prepareSearch()
           .setQuery(matchAllQuery())
           .setSize(35)
           .setScroll(TimeValue.timeValueMinutes(2))
           .addSort("field", SortOrder.ASC)
           .execute()
           .actionGet();
   long counter = 0;
   assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
   assertThat(searchResponse.getHits().hits().length, equalTo(35));
   for (SearchHit hit : searchResponse.getHits()) {
     assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
   }
   if (randomBoolean()) {
     client().admin().indices().prepareClose("test").get();
     client().admin().indices().prepareOpen("test").get();
     ensureGreen("test");
   } else {
     client().admin().indices().prepareDelete("test").get();
   }
 }
  public void testDeepScrollingDoesNotBlowUp() throws Exception {
    client()
        .prepareIndex("index", "type", "1")
        .setSource("field", "value")
        .setRefresh(true)
        .execute()
        .get();

    for (SearchType searchType : SearchType.values()) {
      SearchRequestBuilder builder =
          client()
              .prepareSearch("index")
              .setSearchType(searchType)
              .setQuery(QueryBuilders.matchAllQuery())
              .setSize(Integer.MAX_VALUE)
              .setScroll("1m");

      SearchResponse response = builder.execute().actionGet();
      try {
        ElasticsearchAssertions.assertHitCount(response, 1l);
      } finally {
        String scrollId = response.getScrollId();
        if (scrollId != null) {
          clearScroll(scrollId);
        }
      }
    }
  }
  @Test
  public void testThatNonExistingScrollIdReturnsCorrectException() throws Exception {
    client().prepareIndex("index", "type", "1").setSource("field", "value").execute().get();
    refresh();

    SearchResponse searchResponse =
        client().prepareSearch("index").setSize(1).setScroll("1m").get();
    assertThat(searchResponse.getScrollId(), is(notNullValue()));

    ClearScrollResponse clearScrollResponse =
        client().prepareClearScroll().addScrollId(searchResponse.getScrollId()).get();
    assertThat(clearScrollResponse.isSucceeded(), is(true));

    assertThrows(
        internalCluster().transportClient().prepareSearchScroll(searchResponse.getScrollId()),
        RestStatus.NOT_FOUND);
  }
  @Test
  public void testStringSortMissingAscTerminates() throws Exception {
    assertAcked(
        prepareCreate("test")
            .setSettings(
                Settings.settingsBuilder()
                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
            .addMapping("test", "no_field", "type=string", "some_field", "type=string"));
    client().prepareIndex("test", "test", "1").setSource("some_field", "test").get();
    refresh();

    SearchResponse response =
        client()
            .prepareSearch("test")
            .setTypes("test")
            .addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_last"))
            .setScroll("1m")
            .get();
    assertHitCount(response, 1);
    assertSearchHits(response, "1");

    response = client().prepareSearchScroll(response.getScrollId()).get();
    assertSearchResponse(response);
    assertHitCount(response, 1);
    assertNoSearchHits(response);

    response =
        client()
            .prepareSearch("test")
            .setTypes("test")
            .addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_first"))
            .setScroll("1m")
            .get();
    assertHitCount(response, 1);
    assertSearchHits(response, "1");

    response = client().prepareSearchScroll(response.getScrollId()).get();
    assertHitCount(response, 1);
    assertThat(response.getHits().getHits().length, equalTo(0));
  }
  @Test
  public void testSimpleScrollQueryThenFetch_clearAllScrollIds() throws Exception {
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3))
        .execute()
        .actionGet();
    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForEvents(Priority.LANGUID)
        .setWaitForGreenStatus()
        .execute()
        .actionGet();

    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForEvents(Priority.LANGUID)
        .setWaitForGreenStatus()
        .execute()
        .actionGet();

    for (int i = 0; i < 100; i++) {
      client()
          .prepareIndex("test", "type1", Integer.toString(i))
          .setSource(jsonBuilder().startObject().field("field", i).endObject())
          .execute()
          .actionGet();
    }

    client().admin().indices().prepareRefresh().execute().actionGet();

    SearchResponse searchResponse1 =
        client()
            .prepareSearch()
            .setQuery(matchAllQuery())
            .setSize(35)
            .setScroll(TimeValue.timeValueMinutes(2))
            .setSearchType(SearchType.QUERY_THEN_FETCH)
            .addSort("field", SortOrder.ASC)
            .execute()
            .actionGet();

    SearchResponse searchResponse2 =
        client()
            .prepareSearch()
            .setQuery(matchAllQuery())
            .setSize(35)
            .setScroll(TimeValue.timeValueMinutes(2))
            .setSearchType(SearchType.QUERY_THEN_FETCH)
            .addSort("field", SortOrder.ASC)
            .execute()
            .actionGet();

    long counter1 = 0;
    long counter2 = 0;

    assertThat(searchResponse1.getHits().getTotalHits(), equalTo(100l));
    assertThat(searchResponse1.getHits().hits().length, equalTo(35));
    for (SearchHit hit : searchResponse1.getHits()) {
      assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter1++));
    }

    assertThat(searchResponse2.getHits().getTotalHits(), equalTo(100l));
    assertThat(searchResponse2.getHits().hits().length, equalTo(35));
    for (SearchHit hit : searchResponse2.getHits()) {
      assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter2++));
    }

    searchResponse1 =
        client()
            .prepareSearchScroll(searchResponse1.getScrollId())
            .setScroll(TimeValue.timeValueMinutes(2))
            .execute()
            .actionGet();

    searchResponse2 =
        client()
            .prepareSearchScroll(searchResponse2.getScrollId())
            .setScroll(TimeValue.timeValueMinutes(2))
            .execute()
            .actionGet();

    assertThat(searchResponse1.getHits().getTotalHits(), equalTo(100l));
    assertThat(searchResponse1.getHits().hits().length, equalTo(35));
    for (SearchHit hit : searchResponse1.getHits()) {
      assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter1++));
    }

    assertThat(searchResponse2.getHits().getTotalHits(), equalTo(100l));
    assertThat(searchResponse2.getHits().hits().length, equalTo(35));
    for (SearchHit hit : searchResponse2.getHits()) {
      assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter2++));
    }

    ClearScrollResponse clearResponse =
        client().prepareClearScroll().addScrollId("_all").execute().actionGet();
    assertThat(clearResponse.isSucceeded(), is(true));
    assertThat(clearResponse.getNumFreed(), greaterThan(0));
    assertThat(clearResponse.status(), equalTo(RestStatus.OK));

    assertThrows(
        internalCluster()
            .transportClient()
            .prepareSearchScroll(searchResponse1.getScrollId())
            .setScroll(TimeValue.timeValueMinutes(2)),
        RestStatus.NOT_FOUND);
    assertThrows(
        internalCluster()
            .transportClient()
            .prepareSearchScroll(searchResponse2.getScrollId())
            .setScroll(TimeValue.timeValueMinutes(2)),
        RestStatus.NOT_FOUND);
  }
  @Test
  public void testSimpleScrollQueryThenFetch() throws Exception {
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3))
        .execute()
        .actionGet();
    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForEvents(Priority.LANGUID)
        .setWaitForGreenStatus()
        .execute()
        .actionGet();

    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForEvents(Priority.LANGUID)
        .setWaitForGreenStatus()
        .execute()
        .actionGet();

    for (int i = 0; i < 100; i++) {
      client()
          .prepareIndex("test", "type1", Integer.toString(i))
          .setSource(jsonBuilder().startObject().field("field", i).endObject())
          .execute()
          .actionGet();
    }

    client().admin().indices().prepareRefresh().execute().actionGet();

    SearchResponse searchResponse =
        client()
            .prepareSearch()
            .setQuery(matchAllQuery())
            .setSize(35)
            .setScroll(TimeValue.timeValueMinutes(2))
            .addSort("field", SortOrder.ASC)
            .execute()
            .actionGet();
    try {
      long counter = 0;

      assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
      assertThat(searchResponse.getHits().hits().length, equalTo(35));
      for (SearchHit hit : searchResponse.getHits()) {
        assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
      }

      searchResponse =
          client()
              .prepareSearchScroll(searchResponse.getScrollId())
              .setScroll(TimeValue.timeValueMinutes(2))
              .execute()
              .actionGet();

      assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
      assertThat(searchResponse.getHits().hits().length, equalTo(35));
      for (SearchHit hit : searchResponse.getHits()) {
        assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
      }

      searchResponse =
          client()
              .prepareSearchScroll(searchResponse.getScrollId())
              .setScroll(TimeValue.timeValueMinutes(2))
              .execute()
              .actionGet();

      assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
      assertThat(searchResponse.getHits().hits().length, equalTo(30));
      for (SearchHit hit : searchResponse.getHits()) {
        assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
      }
    } finally {
      clearScroll(searchResponse.getScrollId());
    }
  }
  @Test
  public void testScrollAndUpdateIndex() throws Exception {
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(Settings.settingsBuilder().put("index.number_of_shards", 5))
        .execute()
        .actionGet();
    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForEvents(Priority.LANGUID)
        .setWaitForGreenStatus()
        .execute()
        .actionGet();

    for (int i = 0; i < 500; i++) {
      client()
          .prepareIndex("test", "tweet", Integer.toString(i))
          .setSource(
              jsonBuilder()
                  .startObject()
                  .field("user", "kimchy")
                  .field("postDate", System.currentTimeMillis())
                  .field("message", "test")
                  .endObject())
          .execute()
          .actionGet();
    }

    client().admin().indices().prepareRefresh().execute().actionGet();

    assertThat(
        client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(),
        equalTo(500l));
    assertThat(
        client()
            .prepareCount()
            .setQuery(termQuery("message", "test"))
            .execute()
            .actionGet()
            .getCount(),
        equalTo(500l));
    assertThat(
        client()
            .prepareCount()
            .setQuery(termQuery("message", "test"))
            .execute()
            .actionGet()
            .getCount(),
        equalTo(500l));
    assertThat(
        client()
            .prepareCount()
            .setQuery(termQuery("message", "update"))
            .execute()
            .actionGet()
            .getCount(),
        equalTo(0l));
    assertThat(
        client()
            .prepareCount()
            .setQuery(termQuery("message", "update"))
            .execute()
            .actionGet()
            .getCount(),
        equalTo(0l));

    SearchResponse searchResponse =
        client()
            .prepareSearch()
            .setQuery(queryStringQuery("user:kimchy"))
            .setSize(35)
            .setScroll(TimeValue.timeValueMinutes(2))
            .addSort("postDate", SortOrder.ASC)
            .execute()
            .actionGet();
    try {
      do {
        for (SearchHit searchHit : searchResponse.getHits().hits()) {
          Map<String, Object> map = searchHit.sourceAsMap();
          map.put("message", "update");
          client()
              .prepareIndex("test", "tweet", searchHit.id())
              .setSource(map)
              .execute()
              .actionGet();
        }
        searchResponse =
            client()
                .prepareSearchScroll(searchResponse.getScrollId())
                .setScroll(TimeValue.timeValueMinutes(2))
                .execute()
                .actionGet();
      } while (searchResponse.getHits().hits().length > 0);

      client().admin().indices().prepareRefresh().execute().actionGet();
      assertThat(
          client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(),
          equalTo(500l));
      assertThat(
          client()
              .prepareCount()
              .setQuery(termQuery("message", "test"))
              .execute()
              .actionGet()
              .getCount(),
          equalTo(0l));
      assertThat(
          client()
              .prepareCount()
              .setQuery(termQuery("message", "test"))
              .execute()
              .actionGet()
              .getCount(),
          equalTo(0l));
      assertThat(
          client()
              .prepareCount()
              .setQuery(termQuery("message", "update"))
              .execute()
              .actionGet()
              .getCount(),
          equalTo(500l));
      assertThat(
          client()
              .prepareCount()
              .setQuery(termQuery("message", "update"))
              .execute()
              .actionGet()
              .getCount(),
          equalTo(500l));
    } finally {
      clearScroll(searchResponse.getScrollId());
    }
  }
  @Test
  public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws Exception {
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3))
        .execute()
        .actionGet();
    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForEvents(Priority.LANGUID)
        .setWaitForGreenStatus()
        .execute()
        .actionGet();

    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForEvents(Priority.LANGUID)
        .setWaitForGreenStatus()
        .execute()
        .actionGet();

    for (int i = 0; i < 100; i++) {
      String routing = "0";
      if (i > 90) {
        routing = "1";
      } else if (i > 60) {
        routing = "2";
      }
      client()
          .prepareIndex("test", "type1", Integer.toString(i))
          .setSource("field", i)
          .setRouting(routing)
          .execute()
          .actionGet();
    }

    client().admin().indices().prepareRefresh().execute().actionGet();

    SearchResponse searchResponse =
        client()
            .prepareSearch()
            .setSearchType(SearchType.QUERY_THEN_FETCH)
            .setQuery(matchAllQuery())
            .setSize(3)
            .setScroll(TimeValue.timeValueMinutes(2))
            .addSort("field", SortOrder.ASC)
            .execute()
            .actionGet();
    try {
      long counter = 0;

      assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
      assertThat(searchResponse.getHits().hits().length, equalTo(3));
      for (SearchHit hit : searchResponse.getHits()) {
        assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
      }

      for (int i = 0; i < 32; i++) {
        searchResponse =
            client()
                .prepareSearchScroll(searchResponse.getScrollId())
                .setScroll(TimeValue.timeValueMinutes(2))
                .execute()
                .actionGet();

        assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
        assertThat(searchResponse.getHits().hits().length, equalTo(3));
        for (SearchHit hit : searchResponse.getHits()) {
          assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
        }
      }

      // and now, the last one is one
      searchResponse =
          client()
              .prepareSearchScroll(searchResponse.getScrollId())
              .setScroll(TimeValue.timeValueMinutes(2))
              .execute()
              .actionGet();

      assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
      assertThat(searchResponse.getHits().hits().length, equalTo(1));
      for (SearchHit hit : searchResponse.getHits()) {
        assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
      }

      // a the last is zero
      searchResponse =
          client()
              .prepareSearchScroll(searchResponse.getScrollId())
              .setScroll(TimeValue.timeValueMinutes(2))
              .execute()
              .actionGet();

      assertThat(searchResponse.getHits().getTotalHits(), equalTo(100l));
      assertThat(searchResponse.getHits().hits().length, equalTo(0));
      for (SearchHit hit : searchResponse.getHits()) {
        assertThat(((Number) hit.sortValues()[0]).longValue(), equalTo(counter++));
      }

    } finally {
      clearScroll(searchResponse.getScrollId());
    }
  }