@Test
  public void testClearCacheFilterKeys() {
    client().admin().indices().prepareDelete().execute().actionGet();
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1))
        .execute()
        .actionGet();
    client().prepareIndex("test", "type", "1").setSource("field", "value").execute().actionGet();
    client().admin().indices().prepareRefresh().execute().actionGet();

    NodesStatsResponse nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
    IndicesStatsResponse indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFilterCache(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));

    SearchResponse searchResponse =
        client()
            .prepareSearch()
            .setQuery(
                filteredQuery(
                    matchAllQuery(),
                    FilterBuilders.termFilter("field", "value").cacheKey("test_key")))
            .execute()
            .actionGet();
    assertThat(searchResponse.getHits().getHits().length, equalTo(1));
    nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(),
        greaterThan(0l));
    indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFilterCache(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));

    client().admin().indices().prepareClearCache().setFilterKeys("test_key").execute().actionGet();
    nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
    indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFilterCache(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
  }
  @Test
  public void testOpenContextsAfterRejections() throws InterruptedException {
    createIndex("test");
    ensureGreen("test");
    final int docs = scaledRandomIntBetween(20, 50);
    for (int i = 0; i < docs; i++) {
      client()
          .prepareIndex("test", "type", Integer.toString(i))
          .setSource("field", "value")
          .execute()
          .actionGet();
    }
    IndicesStatsResponse indicesStats =
        client().admin().indices().prepareStats().execute().actionGet();
    assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0l));
    refresh();

    int numSearches = 10;
    Future<SearchResponse>[] responses = new Future[numSearches];
    SearchType searchType =
        randomFrom(
            SearchType.DEFAULT,
            SearchType.QUERY_AND_FETCH,
            SearchType.QUERY_THEN_FETCH,
            SearchType.DFS_QUERY_AND_FETCH,
            SearchType.DFS_QUERY_THEN_FETCH);
    logger.info("search type is {}", searchType);
    for (int i = 0; i < numSearches; i++) {
      responses[i] =
          client().prepareSearch().setQuery(matchAllQuery()).setSearchType(searchType).execute();
    }
    for (int i = 0; i < numSearches; i++) {
      try {
        responses[i].get();
      } catch (Throwable t) {
      }
    }
    awaitBusy(
        new Predicate<Object>() {
          @Override
          public boolean apply(Object input) {
            // we must wait here because the requests to release search contexts might still be in
            // flight
            // although the search request has already returned
            return client()
                    .admin()
                    .indices()
                    .prepareStats()
                    .execute()
                    .actionGet()
                    .getTotal()
                    .getSearch()
                    .getOpenContexts()
                == 0;
          }
        },
        1,
        TimeUnit.SECONDS);
    indicesStats = client().admin().indices().prepareStats().execute().actionGet();
    assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0l));
  }
  @Test
  public void testClearAllCaches() throws Exception {
    client().admin().indices().prepareDelete().execute().actionGet();
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(
            ImmutableSettings.settingsBuilder()
                .put("index.number_of_replicas", 0)
                .put("index.number_of_shards", 1))
        .execute()
        .actionGet();
    client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
    client().prepareIndex("test", "type", "1").setSource("field", "value1").execute().actionGet();
    client().prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet();
    client().admin().indices().prepareRefresh().execute().actionGet();

    NodesStatsResponse nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));

    IndicesStatsResponse indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .setFilterCache(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));

    // sort to load it to field data and filter to load filter cache
    client()
        .prepareSearch()
        .setFilter(FilterBuilders.termFilter("field", "value1"))
        .addSort("field", SortOrder.ASC)
        .execute()
        .actionGet();
    client()
        .prepareSearch()
        .setFilter(FilterBuilders.termFilter("field", "value2"))
        .addSort("field", SortOrder.ASC)
        .execute()
        .actionGet();

    nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
        greaterThan(0l));
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(),
        greaterThan(0l));

    indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .setFilterCache(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
    assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));

    client().admin().indices().prepareClearCache().execute().actionGet();
    Thread.sleep(100); // Make sure the filter cache entries have been removed...
    nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));

    indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .setFilterCache(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
  }
  @Test
  public void testFieldDataStats() {
    client().admin().indices().prepareDelete().execute().actionGet();
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1))
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type", "1")
        .setSource("field", "value1", "field2", "value1")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type", "2")
        .setSource("field", "value2", "field2", "value2")
        .execute()
        .actionGet();
    client().admin().indices().prepareRefresh().execute().actionGet();

    NodesStatsResponse nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    IndicesStatsResponse indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));

    // sort to load it to field data...
    client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
    client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();

    nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
        greaterThan(0l));
    indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));

    // sort to load it to field data...
    client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();
    client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();

    // now check the per field stats
    nodesStats =
        client()
            .admin()
            .cluster()
            .prepareNodesStats()
            .setIndices(
                new CommonStatsFlags()
                    .set(CommonStatsFlags.Flag.FieldData, true)
                    .fieldDataFields("*"))
            .execute()
            .actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
        greaterThan(0l));
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getFields().get("field"),
        greaterThan(0l));
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getFields().get("field"),
        lessThan(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes()));

    indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .setFieldDataFields("*")
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
    assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), greaterThan(0l));
    assertThat(
        indicesStats.getTotal().getFieldData().getFields().get("field"),
        lessThan(indicesStats.getTotal().getFieldData().getMemorySizeInBytes()));

    client().admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet();
    nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
  }
  @Test
  public void simpleStats() throws Exception {
    // rely on 1 replica for this tests
    createIndex("test1");
    createIndex("test2");

    ClusterHealthResponse clusterHealthResponse =
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setWaitForGreenStatus()
            .execute()
            .actionGet();
    assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));

    client()
        .prepareIndex("test1", "type1", Integer.toString(1))
        .setSource("field", "value")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test1", "type2", Integer.toString(1))
        .setSource("field", "value")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test2", "type", Integer.toString(1))
        .setSource("field", "value")
        .execute()
        .actionGet();

    client().admin().indices().prepareRefresh().execute().actionGet();

    IndicesStatsResponse stats = client().admin().indices().prepareStats().execute().actionGet();
    assertThat(stats.getPrimaries().getDocs().getCount(), equalTo(3l));
    assertThat(stats.getTotal().getDocs().getCount(), equalTo(6l));
    assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexCount(), equalTo(3l));
    assertThat(stats.getTotal().getIndexing().getTotal().getIndexCount(), equalTo(6l));
    assertThat(stats.getTotal().getStore(), notNullValue());
    // verify nulls
    assertThat(stats.getTotal().getMerge(), nullValue());
    assertThat(stats.getTotal().getFlush(), nullValue());
    assertThat(stats.getTotal().getRefresh(), nullValue());

    assertThat(stats.getIndex("test1").getPrimaries().getDocs().getCount(), equalTo(2l));
    assertThat(stats.getIndex("test1").getTotal().getDocs().getCount(), equalTo(4l));
    assertThat(stats.getIndex("test1").getPrimaries().getStore(), notNullValue());
    assertThat(stats.getIndex("test1").getPrimaries().getMerge(), nullValue());
    assertThat(stats.getIndex("test1").getPrimaries().getFlush(), nullValue());
    assertThat(stats.getIndex("test1").getPrimaries().getRefresh(), nullValue());

    assertThat(stats.getIndex("test2").getPrimaries().getDocs().getCount(), equalTo(1l));
    assertThat(stats.getIndex("test2").getTotal().getDocs().getCount(), equalTo(2l));

    // make sure that number of requests in progress is 0
    assertThat(
        stats.getIndex("test1").getTotal().getIndexing().getTotal().getIndexCurrent(), equalTo(0l));
    assertThat(
        stats.getIndex("test1").getTotal().getIndexing().getTotal().getDeleteCurrent(),
        equalTo(0l));
    assertThat(
        stats.getIndex("test1").getTotal().getSearch().getTotal().getFetchCurrent(), equalTo(0l));
    assertThat(
        stats.getIndex("test1").getTotal().getSearch().getTotal().getQueryCurrent(), equalTo(0l));

    // check flags
    stats =
        client()
            .admin()
            .indices()
            .prepareStats()
            .setDocs(false)
            .setStore(false)
            .setIndexing(false)
            .setFlush(true)
            .setRefresh(true)
            .setMerge(true)
            .execute()
            .actionGet();

    assertThat(stats.getTotal().getDocs(), nullValue());
    assertThat(stats.getTotal().getStore(), nullValue());
    assertThat(stats.getTotal().getIndexing(), nullValue());
    assertThat(stats.getTotal().getMerge(), notNullValue());
    assertThat(stats.getTotal().getFlush(), notNullValue());
    assertThat(stats.getTotal().getRefresh(), notNullValue());

    // check types
    stats =
        client().admin().indices().prepareStats().setTypes("type1", "type").execute().actionGet();
    assertThat(
        stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCount(),
        equalTo(1l));
    assertThat(
        stats.getPrimaries().getIndexing().getTypeStats().get("type").getIndexCount(), equalTo(1l));
    assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2"), nullValue());
    assertThat(
        stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCurrent(),
        equalTo(0l));
    assertThat(
        stats.getPrimaries().getIndexing().getTypeStats().get("type1").getDeleteCurrent(),
        equalTo(0l));

    assertThat(stats.getTotal().getGet().getCount(), equalTo(0l));
    // check get
    GetResponse getResponse = client().prepareGet("test1", "type1", "1").execute().actionGet();
    assertThat(getResponse.isExists(), equalTo(true));

    stats = client().admin().indices().prepareStats().execute().actionGet();
    assertThat(stats.getTotal().getGet().getCount(), equalTo(1l));
    assertThat(stats.getTotal().getGet().getExistsCount(), equalTo(1l));
    assertThat(stats.getTotal().getGet().getMissingCount(), equalTo(0l));

    // missing get
    getResponse = client().prepareGet("test1", "type1", "2").execute().actionGet();
    assertThat(getResponse.isExists(), equalTo(false));

    stats = client().admin().indices().prepareStats().execute().actionGet();
    assertThat(stats.getTotal().getGet().getCount(), equalTo(2l));
    assertThat(stats.getTotal().getGet().getExistsCount(), equalTo(1l));
    assertThat(stats.getTotal().getGet().getMissingCount(), equalTo(1l));

    // clear all
    stats =
        client()
            .admin()
            .indices()
            .prepareStats()
            .setDocs(false)
            .setStore(false)
            .setIndexing(false)
            .setFlush(true)
            .setRefresh(true)
            .setMerge(true)
            .clear() // reset defaults
            .execute()
            .actionGet();

    assertThat(stats.getTotal().getDocs(), nullValue());
    assertThat(stats.getTotal().getStore(), nullValue());
    assertThat(stats.getTotal().getIndexing(), nullValue());
    assertThat(stats.getTotal().getGet(), nullValue());
    assertThat(stats.getTotal().getSearch(), nullValue());
  }
  @Test
  public void testAllFlags() throws Exception {
    // rely on 1 replica for this tests
    createIndex("test1");
    createIndex("test2");

    ClusterHealthResponse clusterHealthResponse =
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setWaitForGreenStatus()
            .execute()
            .actionGet();
    assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));

    client()
        .prepareIndex("test1", "type1", Integer.toString(1))
        .setSource("field", "value")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test1", "type2", Integer.toString(1))
        .setSource("field", "value")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test2", "type", Integer.toString(1))
        .setSource("field", "value")
        .execute()
        .actionGet();

    client().admin().indices().prepareRefresh().execute().actionGet();
    IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
    Flag[] values = CommonStatsFlags.Flag.values();
    for (Flag flag : values) {
      set(flag, builder, false);
    }

    IndicesStatsResponse stats = builder.execute().actionGet();
    for (Flag flag : values) {
      assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
      assertThat(isSet(flag, stats.getTotal()), equalTo(false));
    }

    for (Flag flag : values) {
      set(flag, builder, true);
    }
    stats = builder.execute().actionGet();
    for (Flag flag : values) {
      assertThat(isSet(flag, stats.getPrimaries()), equalTo(true));
      assertThat(isSet(flag, stats.getTotal()), equalTo(true));
    }
    Random random = getRandom();
    EnumSet<Flag> flags = EnumSet.noneOf(Flag.class);
    for (Flag flag : values) {
      if (random.nextBoolean()) {
        flags.add(flag);
      }
    }

    for (Flag flag : values) {
      set(flag, builder, false); // clear all
    }

    for (Flag flag : flags) { // set the flags
      set(flag, builder, true);
    }
    stats = builder.execute().actionGet();
    for (Flag flag : flags) { // check the flags
      assertThat(isSet(flag, stats.getPrimaries()), equalTo(true));
      assertThat(isSet(flag, stats.getTotal()), equalTo(true));
    }

    for (Flag flag : EnumSet.complementOf(flags)) { // check the complement
      assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
      assertThat(isSet(flag, stats.getTotal()), equalTo(false));
    }
  }
  @Test
  public void testMergeStats() {
    // rely on 1 replica for this tests
    createIndex("test1");

    ClusterHealthResponse clusterHealthResponse =
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setWaitForGreenStatus()
            .execute()
            .actionGet();
    assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));

    // clear all
    IndicesStatsResponse stats =
        client()
            .admin()
            .indices()
            .prepareStats()
            .setDocs(false)
            .setStore(false)
            .setIndexing(false)
            .setFlush(true)
            .setRefresh(true)
            .setMerge(true)
            .clear() // reset defaults
            .execute()
            .actionGet();

    assertThat(stats.getTotal().getDocs(), nullValue());
    assertThat(stats.getTotal().getStore(), nullValue());
    assertThat(stats.getTotal().getIndexing(), nullValue());
    assertThat(stats.getTotal().getGet(), nullValue());
    assertThat(stats.getTotal().getSearch(), nullValue());

    for (int i = 0; i < 20; i++) {
      client()
          .prepareIndex("test1", "type1", Integer.toString(i))
          .setSource("field", "value")
          .execute()
          .actionGet();
      client()
          .prepareIndex("test1", "type2", Integer.toString(i))
          .setSource("field", "value")
          .execute()
          .actionGet();
      client().admin().indices().prepareFlush().execute().actionGet();
    }
    client()
        .admin()
        .indices()
        .prepareOptimize()
        .setWaitForMerge(true)
        .setMaxNumSegments(1)
        .execute()
        .actionGet();
    stats = client().admin().indices().prepareStats().setMerge(true).execute().actionGet();

    assertThat(stats.getTotal().getMerge(), notNullValue());
    assertThat(stats.getTotal().getMerge().getTotal(), greaterThan(0l));
  }