@Test
public void testClearCacheFilterKeys() {
client().admin().indices().prepareDelete().execute().actionGet();
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1))
.execute()
.actionGet();
client().prepareIndex("test", "type", "1").setSource("field", "value").execute().actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
NodesStatsResponse nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
IndicesStatsResponse indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(
filteredQuery(
matchAllQuery(),
FilterBuilders.termFilter("field", "value").cacheKey("test_key")))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getHits().length, equalTo(1));
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(),
greaterThan(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
client().admin().indices().prepareClearCache().setFilterKeys("test_key").execute().actionGet();
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
}
@Test
public void testOpenContextsAfterRejections() throws InterruptedException {
createIndex("test");
ensureGreen("test");
final int docs = scaledRandomIntBetween(20, 50);
for (int i = 0; i < docs; i++) {
client()
.prepareIndex("test", "type", Integer.toString(i))
.setSource("field", "value")
.execute()
.actionGet();
}
IndicesStatsResponse indicesStats =
client().admin().indices().prepareStats().execute().actionGet();
assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0l));
refresh();
int numSearches = 10;
Future<SearchResponse>[] responses = new Future[numSearches];
SearchType searchType =
randomFrom(
SearchType.DEFAULT,
SearchType.QUERY_AND_FETCH,
SearchType.QUERY_THEN_FETCH,
SearchType.DFS_QUERY_AND_FETCH,
SearchType.DFS_QUERY_THEN_FETCH);
logger.info("search type is {}", searchType);
for (int i = 0; i < numSearches; i++) {
responses[i] =
client().prepareSearch().setQuery(matchAllQuery()).setSearchType(searchType).execute();
}
for (int i = 0; i < numSearches; i++) {
try {
responses[i].get();
} catch (Throwable t) {
}
}
awaitBusy(
new Predicate<Object>() {
@Override
public boolean apply(Object input) {
// we must wait here because the requests to release search contexts might still be in
// flight
// although the search request has already returned
return client()
.admin()
.indices()
.prepareStats()
.execute()
.actionGet()
.getTotal()
.getSearch()
.getOpenContexts()
== 0;
}
},
1,
TimeUnit.SECONDS);
indicesStats = client().admin().indices().prepareStats().execute().actionGet();
assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0l));
}
@Test
public void testClearAllCaches() throws Exception {
client().admin().indices().prepareDelete().execute().actionGet();
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
ImmutableSettings.settingsBuilder()
.put("index.number_of_replicas", 0)
.put("index.number_of_shards", 1))
.execute()
.actionGet();
client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
client().prepareIndex("test", "type", "1").setSource("field", "value1").execute().actionGet();
client().prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
NodesStatsResponse nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
IndicesStatsResponse indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
// sort to load it to field data and filter to load filter cache
client()
.prepareSearch()
.setFilter(FilterBuilders.termFilter("field", "value1"))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
client()
.prepareSearch()
.setFilter(FilterBuilders.termFilter("field", "value2"))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
greaterThan(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(),
greaterThan(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
client().admin().indices().prepareClearCache().execute().actionGet();
Thread.sleep(100); // Make sure the filter cache entries have been removed...
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
}
@Test
public void testFieldDataStats() {
client().admin().indices().prepareDelete().execute().actionGet();
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1))
.execute()
.actionGet();
client()
.prepareIndex("test", "type", "1")
.setSource("field", "value1", "field2", "value1")
.execute()
.actionGet();
client()
.prepareIndex("test", "type", "2")
.setSource("field", "value2", "field2", "value2")
.execute()
.actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
NodesStatsResponse nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
IndicesStatsResponse indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
// sort to load it to field data...
client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
greaterThan(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
// sort to load it to field data...
client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();
client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();
// now check the per field stats
nodesStats =
client()
.admin()
.cluster()
.prepareNodesStats()
.setIndices(
new CommonStatsFlags()
.set(CommonStatsFlags.Flag.FieldData, true)
.fieldDataFields("*"))
.execute()
.actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
greaterThan(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getFields().get("field"),
greaterThan(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getFields().get("field"),
lessThan(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes()));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.setFieldDataFields("*")
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), greaterThan(0l));
assertThat(
indicesStats.getTotal().getFieldData().getFields().get("field"),
lessThan(indicesStats.getTotal().getFieldData().getMemorySizeInBytes()));
client().admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet();
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
}
@Test
public void simpleStats() throws Exception {
// rely on 1 replica for this tests
createIndex("test1");
createIndex("test2");
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
client()
.prepareIndex("test1", "type1", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test1", "type2", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test2", "type", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
IndicesStatsResponse stats = client().admin().indices().prepareStats().execute().actionGet();
assertThat(stats.getPrimaries().getDocs().getCount(), equalTo(3l));
assertThat(stats.getTotal().getDocs().getCount(), equalTo(6l));
assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexCount(), equalTo(3l));
assertThat(stats.getTotal().getIndexing().getTotal().getIndexCount(), equalTo(6l));
assertThat(stats.getTotal().getStore(), notNullValue());
// verify nulls
assertThat(stats.getTotal().getMerge(), nullValue());
assertThat(stats.getTotal().getFlush(), nullValue());
assertThat(stats.getTotal().getRefresh(), nullValue());
assertThat(stats.getIndex("test1").getPrimaries().getDocs().getCount(), equalTo(2l));
assertThat(stats.getIndex("test1").getTotal().getDocs().getCount(), equalTo(4l));
assertThat(stats.getIndex("test1").getPrimaries().getStore(), notNullValue());
assertThat(stats.getIndex("test1").getPrimaries().getMerge(), nullValue());
assertThat(stats.getIndex("test1").getPrimaries().getFlush(), nullValue());
assertThat(stats.getIndex("test1").getPrimaries().getRefresh(), nullValue());
assertThat(stats.getIndex("test2").getPrimaries().getDocs().getCount(), equalTo(1l));
assertThat(stats.getIndex("test2").getTotal().getDocs().getCount(), equalTo(2l));
// make sure that number of requests in progress is 0
assertThat(
stats.getIndex("test1").getTotal().getIndexing().getTotal().getIndexCurrent(), equalTo(0l));
assertThat(
stats.getIndex("test1").getTotal().getIndexing().getTotal().getDeleteCurrent(),
equalTo(0l));
assertThat(
stats.getIndex("test1").getTotal().getSearch().getTotal().getFetchCurrent(), equalTo(0l));
assertThat(
stats.getIndex("test1").getTotal().getSearch().getTotal().getQueryCurrent(), equalTo(0l));
// check flags
stats =
client()
.admin()
.indices()
.prepareStats()
.setDocs(false)
.setStore(false)
.setIndexing(false)
.setFlush(true)
.setRefresh(true)
.setMerge(true)
.execute()
.actionGet();
assertThat(stats.getTotal().getDocs(), nullValue());
assertThat(stats.getTotal().getStore(), nullValue());
assertThat(stats.getTotal().getIndexing(), nullValue());
assertThat(stats.getTotal().getMerge(), notNullValue());
assertThat(stats.getTotal().getFlush(), notNullValue());
assertThat(stats.getTotal().getRefresh(), notNullValue());
// check types
stats =
client().admin().indices().prepareStats().setTypes("type1", "type").execute().actionGet();
assertThat(
stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCount(),
equalTo(1l));
assertThat(
stats.getPrimaries().getIndexing().getTypeStats().get("type").getIndexCount(), equalTo(1l));
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2"), nullValue());
assertThat(
stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCurrent(),
equalTo(0l));
assertThat(
stats.getPrimaries().getIndexing().getTypeStats().get("type1").getDeleteCurrent(),
equalTo(0l));
assertThat(stats.getTotal().getGet().getCount(), equalTo(0l));
// check get
GetResponse getResponse = client().prepareGet("test1", "type1", "1").execute().actionGet();
assertThat(getResponse.isExists(), equalTo(true));
stats = client().admin().indices().prepareStats().execute().actionGet();
assertThat(stats.getTotal().getGet().getCount(), equalTo(1l));
assertThat(stats.getTotal().getGet().getExistsCount(), equalTo(1l));
assertThat(stats.getTotal().getGet().getMissingCount(), equalTo(0l));
// missing get
getResponse = client().prepareGet("test1", "type1", "2").execute().actionGet();
assertThat(getResponse.isExists(), equalTo(false));
stats = client().admin().indices().prepareStats().execute().actionGet();
assertThat(stats.getTotal().getGet().getCount(), equalTo(2l));
assertThat(stats.getTotal().getGet().getExistsCount(), equalTo(1l));
assertThat(stats.getTotal().getGet().getMissingCount(), equalTo(1l));
// clear all
stats =
client()
.admin()
.indices()
.prepareStats()
.setDocs(false)
.setStore(false)
.setIndexing(false)
.setFlush(true)
.setRefresh(true)
.setMerge(true)
.clear() // reset defaults
.execute()
.actionGet();
assertThat(stats.getTotal().getDocs(), nullValue());
assertThat(stats.getTotal().getStore(), nullValue());
assertThat(stats.getTotal().getIndexing(), nullValue());
assertThat(stats.getTotal().getGet(), nullValue());
assertThat(stats.getTotal().getSearch(), nullValue());
}
@Test
public void testAllFlags() throws Exception {
// rely on 1 replica for this tests
createIndex("test1");
createIndex("test2");
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
client()
.prepareIndex("test1", "type1", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test1", "type2", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test2", "type", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
Flag[] values = CommonStatsFlags.Flag.values();
for (Flag flag : values) {
set(flag, builder, false);
}
IndicesStatsResponse stats = builder.execute().actionGet();
for (Flag flag : values) {
assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
assertThat(isSet(flag, stats.getTotal()), equalTo(false));
}
for (Flag flag : values) {
set(flag, builder, true);
}
stats = builder.execute().actionGet();
for (Flag flag : values) {
assertThat(isSet(flag, stats.getPrimaries()), equalTo(true));
assertThat(isSet(flag, stats.getTotal()), equalTo(true));
}
Random random = getRandom();
EnumSet<Flag> flags = EnumSet.noneOf(Flag.class);
for (Flag flag : values) {
if (random.nextBoolean()) {
flags.add(flag);
}
}
for (Flag flag : values) {
set(flag, builder, false); // clear all
}
for (Flag flag : flags) { // set the flags
set(flag, builder, true);
}
stats = builder.execute().actionGet();
for (Flag flag : flags) { // check the flags
assertThat(isSet(flag, stats.getPrimaries()), equalTo(true));
assertThat(isSet(flag, stats.getTotal()), equalTo(true));
}
for (Flag flag : EnumSet.complementOf(flags)) { // check the complement
assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
assertThat(isSet(flag, stats.getTotal()), equalTo(false));
}
}
@Test
public void testMergeStats() {
// rely on 1 replica for this tests
createIndex("test1");
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
// clear all
IndicesStatsResponse stats =
client()
.admin()
.indices()
.prepareStats()
.setDocs(false)
.setStore(false)
.setIndexing(false)
.setFlush(true)
.setRefresh(true)
.setMerge(true)
.clear() // reset defaults
.execute()
.actionGet();
assertThat(stats.getTotal().getDocs(), nullValue());
assertThat(stats.getTotal().getStore(), nullValue());
assertThat(stats.getTotal().getIndexing(), nullValue());
assertThat(stats.getTotal().getGet(), nullValue());
assertThat(stats.getTotal().getSearch(), nullValue());
for (int i = 0; i < 20; i++) {
client()
.prepareIndex("test1", "type1", Integer.toString(i))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test1", "type2", Integer.toString(i))
.setSource("field", "value")
.execute()
.actionGet();
client().admin().indices().prepareFlush().execute().actionGet();
}
client()
.admin()
.indices()
.prepareOptimize()
.setWaitForMerge(true)
.setMaxNumSegments(1)
.execute()
.actionGet();
stats = client().admin().indices().prepareStats().setMerge(true).execute().actionGet();
assertThat(stats.getTotal().getMerge(), notNullValue());
assertThat(stats.getTotal().getMerge().getTotal(), greaterThan(0l));
}