public void testAutoCreateIndexWithDateMathExpression() throws Exception {
DateTime now = new DateTime(DateTimeZone.UTC);
String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now);
String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1));
String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2));
String dateMathExp1 = "<.marvel-{now/d}>";
String dateMathExp2 = "<.marvel-{now/d-1d}>";
String dateMathExp3 = "<.marvel-{now/d-2d}>";
client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
refresh();
SearchResponse searchResponse =
client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
assertHitCount(searchResponse, 3);
assertSearchHits(searchResponse, "1", "2", "3");
IndicesStatsResponse indicesStatsResponse =
client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get();
assertThat(indicesStatsResponse.getIndex(index1), notNullValue());
assertThat(indicesStatsResponse.getIndex(index2), notNullValue());
assertThat(indicesStatsResponse.getIndex(index3), notNullValue());
}
public void testIndicesStats() {
createIndex("test");
ensureYellow("test");
IndicesStatsResponse indicesStatsResponse =
client().admin().indices().prepareStats().all().get();
assertThat(indicesStatsResponse.getIndices().size(), equalTo(1));
assertThat(indicesStatsResponse.getIndices().containsKey("test"), equalTo(true));
}
private long getWarmerRuns() {
IndicesStatsResponse indicesStatsResponse =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setWarmer(true)
.execute()
.actionGet();
return indicesStatsResponse.getIndex("test").getPrimaries().warmer.total();
}
public void testIndexNameDateMathExpressions() {
DateTime now = new DateTime(DateTimeZone.UTC);
String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now);
String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1));
String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2));
createIndex(index1, index2, index3);
String dateMathExp1 = "<.marvel-{now/d}>";
String dateMathExp2 = "<.marvel-{now/d-1d}>";
String dateMathExp3 = "<.marvel-{now/d-2d}>";
client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
refresh();
SearchResponse searchResponse =
client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
assertHitCount(searchResponse, 3);
assertSearchHits(searchResponse, "1", "2", "3");
GetResponse getResponse = client().prepareGet(dateMathExp1, "type", "1").get();
assertThat(getResponse.isExists(), is(true));
assertThat(getResponse.getId(), equalTo("1"));
getResponse = client().prepareGet(dateMathExp2, "type", "2").get();
assertThat(getResponse.isExists(), is(true));
assertThat(getResponse.getId(), equalTo("2"));
getResponse = client().prepareGet(dateMathExp3, "type", "3").get();
assertThat(getResponse.isExists(), is(true));
assertThat(getResponse.getId(), equalTo("3"));
IndicesStatsResponse indicesStatsResponse =
client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get();
assertThat(indicesStatsResponse.getIndex(index1), notNullValue());
assertThat(indicesStatsResponse.getIndex(index2), notNullValue());
assertThat(indicesStatsResponse.getIndex(index3), notNullValue());
DeleteResponse deleteResponse = client().prepareDelete(dateMathExp1, "type", "1").get();
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
assertThat(deleteResponse.getId(), equalTo("1"));
deleteResponse = client().prepareDelete(dateMathExp2, "type", "2").get();
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
assertThat(deleteResponse.getId(), equalTo("2"));
deleteResponse = client().prepareDelete(dateMathExp3, "type", "3").get();
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
assertThat(deleteResponse.getId(), equalTo("3"));
}
@Override
protected void beforeIndexDeletion() throws Exception {
super.beforeIndexDeletion();
assertBusy(
() -> {
IndicesStatsResponse stats = client().admin().indices().prepareStats().clear().get();
for (IndexStats indexStats : stats.getIndices().values()) {
for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) {
Optional<ShardStats> maybePrimary =
Stream.of(indexShardStats.getShards())
.filter(s -> s.getShardRouting().active() && s.getShardRouting().primary())
.findFirst();
if (maybePrimary.isPresent() == false) {
continue;
}
ShardStats primary = maybePrimary.get();
final SeqNoStats primarySeqNoStats = primary.getSeqNoStats();
assertThat(
primary.getShardRouting() + " should have set the global checkpoint",
primarySeqNoStats.getGlobalCheckpoint(),
not(equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO)));
for (ShardStats shardStats : indexShardStats) {
final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
assertThat(
shardStats.getShardRouting() + " local checkpoint mismatch",
seqNoStats.getLocalCheckpoint(),
equalTo(primarySeqNoStats.getLocalCheckpoint()));
assertThat(
shardStats.getShardRouting() + " global checkpoint mismatch",
seqNoStats.getGlobalCheckpoint(),
equalTo(primarySeqNoStats.getGlobalCheckpoint()));
assertThat(
shardStats.getShardRouting() + " max seq no mismatch",
seqNoStats.getMaxSeqNo(),
equalTo(primarySeqNoStats.getMaxSeqNo()));
}
}
}
});
}
@Test
public void testOpenContextsAfterRejections() throws InterruptedException {
createIndex("test");
ensureGreen("test");
final int docs = scaledRandomIntBetween(20, 50);
for (int i = 0; i < docs; i++) {
client()
.prepareIndex("test", "type", Integer.toString(i))
.setSource("field", "value")
.execute()
.actionGet();
}
IndicesStatsResponse indicesStats =
client().admin().indices().prepareStats().execute().actionGet();
assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0l));
refresh();
int numSearches = 10;
Future<SearchResponse>[] responses = new Future[numSearches];
SearchType searchType =
randomFrom(
SearchType.DEFAULT,
SearchType.QUERY_AND_FETCH,
SearchType.QUERY_THEN_FETCH,
SearchType.DFS_QUERY_AND_FETCH,
SearchType.DFS_QUERY_THEN_FETCH);
logger.info("search type is {}", searchType);
for (int i = 0; i < numSearches; i++) {
responses[i] =
client().prepareSearch().setQuery(matchAllQuery()).setSearchType(searchType).execute();
}
for (int i = 0; i < numSearches; i++) {
try {
responses[i].get();
} catch (Throwable t) {
}
}
awaitBusy(
new Predicate<Object>() {
@Override
public boolean apply(Object input) {
// we must wait here because the requests to release search contexts might still be in
// flight
// although the search request has already returned
return client()
.admin()
.indices()
.prepareStats()
.execute()
.actionGet()
.getTotal()
.getSearch()
.getOpenContexts()
== 0;
}
},
1,
TimeUnit.SECONDS);
indicesStats = client().admin().indices().prepareStats().execute().actionGet();
assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0l));
}
@Test
public void testClearCacheFilterKeys() {
client().admin().indices().prepareDelete().execute().actionGet();
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1))
.execute()
.actionGet();
client().prepareIndex("test", "type", "1").setSource("field", "value").execute().actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
NodesStatsResponse nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
IndicesStatsResponse indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(
filteredQuery(
matchAllQuery(),
FilterBuilders.termFilter("field", "value").cacheKey("test_key")))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getHits().length, equalTo(1));
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(),
greaterThan(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
client().admin().indices().prepareClearCache().setFilterKeys("test_key").execute().actionGet();
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
}
@Test
public void testClearAllCaches() throws Exception {
client().admin().indices().prepareDelete().execute().actionGet();
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
ImmutableSettings.settingsBuilder()
.put("index.number_of_replicas", 0)
.put("index.number_of_shards", 1))
.execute()
.actionGet();
client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
client().prepareIndex("test", "type", "1").setSource("field", "value1").execute().actionGet();
client().prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
NodesStatsResponse nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
IndicesStatsResponse indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
// sort to load it to field data and filter to load filter cache
client()
.prepareSearch()
.setFilter(FilterBuilders.termFilter("field", "value1"))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
client()
.prepareSearch()
.setFilter(FilterBuilders.termFilter("field", "value2"))
.addSort("field", SortOrder.ASC)
.execute()
.actionGet();
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
greaterThan(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(),
greaterThan(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
client().admin().indices().prepareClearCache().execute().actionGet();
Thread.sleep(100); // Make sure the filter cache entries have been removed...
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.setFilterCache(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
}
@Test
public void testFieldDataStats() {
client().admin().indices().prepareDelete().execute().actionGet();
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1))
.execute()
.actionGet();
client()
.prepareIndex("test", "type", "1")
.setSource("field", "value1", "field2", "value1")
.execute()
.actionGet();
client()
.prepareIndex("test", "type", "2")
.setSource("field", "value2", "field2", "value2")
.execute()
.actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
NodesStatsResponse nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
IndicesStatsResponse indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
// sort to load it to field data...
client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
greaterThan(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
// sort to load it to field data...
client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();
client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();
// now check the per field stats
nodesStats =
client()
.admin()
.cluster()
.prepareNodesStats()
.setIndices(
new CommonStatsFlags()
.set(CommonStatsFlags.Flag.FieldData, true)
.fieldDataFields("*"))
.execute()
.actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
greaterThan(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getFields().get("field"),
greaterThan(0l));
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getFields().get("field"),
lessThan(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes()));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.setFieldDataFields("*")
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), greaterThan(0l));
assertThat(
indicesStats.getTotal().getFieldData().getFields().get("field"),
lessThan(indicesStats.getTotal().getFieldData().getMemorySizeInBytes()));
client().admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet();
nodesStats =
client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(
nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
indicesStats =
client()
.admin()
.indices()
.prepareStats("test")
.clear()
.setFieldData(true)
.execute()
.actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
}
@Test
public void testSimpleTTL() throws Exception {
assertAcked(
prepareCreate("test")
.addMapping(
"type1",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.endObject()
.endObject()
.endObject())
.addMapping(
"type2",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type2")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.field("default", "1d")
.endObject()
.endObject()
.endObject()));
ensureYellow("test");
final NumShards test = getNumShards("test");
long providedTTLValue = 3000;
logger.info("--> checking ttl");
// Index one doc without routing, one doc with routing, one doc with not TTL and no default and
// one doc with default TTL
long now = System.currentTimeMillis();
IndexResponse indexResponse =
client()
.prepareIndex("test", "type1", "1")
.setSource("field1", "value1")
.setTimestamp(String.valueOf(now))
.setTTL(providedTTLValue)
.setRefresh(true)
.get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client()
.prepareIndex("test", "type1", "with_routing")
.setSource("field1", "value1")
.setTimestamp(String.valueOf(now))
.setTTL(providedTTLValue)
.setRouting("routing")
.setRefresh(true)
.get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client().prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client().prepareIndex("test", "type2", "default_ttl").setSource("field1", "value1").get();
assertThat(indexResponse.isCreated(), is(true));
// realtime get check
long currentTime = System.currentTimeMillis();
GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
long ttl0;
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// verify the ttl is still decreasing when going to the replica
currentTime = System.currentTimeMillis();
getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// non realtime get (stored)
currentTime = System.currentTimeMillis();
getResponse =
client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// non realtime get going the replica
currentTime = System.currentTimeMillis();
getResponse =
client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// no TTL provided so no TTL fetched
getResponse =
client()
.prepareGet("test", "type1", "no_ttl")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.getField("_ttl"), nullValue());
// no TTL provided make sure it has default TTL
getResponse =
client()
.prepareGet("test", "type2", "default_ttl")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, greaterThan(0L));
IndicesStatsResponse response =
client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
assertThat(
response.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount(),
equalTo(0L));
// make sure the purger has done its job for all indexed docs that are expired
long shouldBeExpiredDate = now + providedTTLValue + PURGE_INTERVAL + 2000;
currentTime = System.currentTimeMillis();
if (shouldBeExpiredDate - currentTime > 0) {
Thread.sleep(shouldBeExpiredDate - currentTime);
}
// We can't assume that after waiting for ttl + purgeInterval (waitTime) that the document have
// actually been deleted.
// The ttl purging happens in the background in a different thread, and might not have been
// completed after waiting for waitTime.
// But we can use index statistics' delete count to be sure that deletes have been executed,
// that must be incremented before
// ttl purging has finished.
logger.info("--> checking purger");
assertThat(
awaitBusy(
new Predicate<Object>() {
@Override
public boolean apply(Object input) {
if (rarely()) {
client().admin().indices().prepareFlush("test").get();
} else if (rarely()) {
client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get();
}
IndicesStatsResponse response =
client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
// TTL deletes two docs, but it is indexed in the primary shard and replica shard.
return response
.getIndices()
.get("test")
.getTotal()
.getIndexing()
.getTotal()
.getDeleteCount()
== 2L * test.dataCopies;
}
},
5,
TimeUnit.SECONDS),
equalTo(true));
// realtime get check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// replica realtime get check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// Need to run a refresh, in order for the non realtime get to work.
client().admin().indices().prepareRefresh("test").execute().actionGet();
// non realtime get (stored) check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// non realtime get going the replica check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
}
@TestLogging("org.elasticsearch.gateway:TRACE")
public void testIndexWithFewDocuments() throws Exception {
final Path dataPath = createTempDir();
Settings nodeSettings = nodeSettings(dataPath);
internalCluster().startNodesAsync(3, nodeSettings).get();
final String IDX = "test";
Settings idxSettings =
Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2)
.put(
IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
new ByteSizeValue(1, ByteSizeUnit.PB))
.put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString())
.put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
.put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true)
.build();
prepareCreate(IDX).setSettings(idxSettings).addMapping("doc", "foo", "type=text").get();
ensureGreen(IDX);
// So basically, the primary should fail and the replica will need to
// replay the translog, this is what this tests
client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get();
client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get();
IndicesStatsResponse indicesStatsResponse =
client().admin().indices().prepareStats(IDX).clear().setTranslog(true).get();
assertEquals(
2,
indicesStatsResponse
.getIndex(IDX)
.getPrimaries()
.getTranslog()
.estimatedNumberOfOperations());
assertEquals(
2,
indicesStatsResponse.getIndex(IDX).getTotal().getTranslog().estimatedNumberOfOperations());
Index index = resolveIndex(IDX);
for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
IndexService indexService = service.indexService(index);
if (indexService != null) {
IndexShard shard = indexService.getShard(0);
TranslogStats translogStats = shard.translogStats();
assertTrue(translogStats != null || shard instanceof ShadowIndexShard);
if (translogStats != null) {
assertEquals(2, translogStats.estimatedNumberOfOperations());
}
}
}
// Check that we can get doc 1 and 2, because we are doing realtime
// gets and getting from the primary
GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get();
GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get();
assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
flushAndRefresh(IDX);
client().prepareIndex(IDX, "doc", "3").setSource("foo", "bar").get();
client().prepareIndex(IDX, "doc", "4").setSource("foo", "bar").get();
refresh();
// Check that we can get doc 1 and 2 without realtime
gResp1 = client().prepareGet(IDX, "doc", "1").setRealtime(false).get();
gResp2 = client().prepareGet(IDX, "doc", "2").setRealtime(false).get();
assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
logger.info("--> restarting all nodes");
if (randomBoolean()) {
logger.info("--> rolling restart");
internalCluster().rollingRestart();
} else {
logger.info("--> full restart");
internalCluster().fullRestart();
}
client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
ensureGreen(IDX);
flushAndRefresh(IDX);
logger.info("--> performing query");
SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
assertHitCount(resp, 4);
logger.info("--> deleting index");
assertAcked(client().admin().indices().prepareDelete(IDX));
}
@Test
public void simpleStats() throws Exception {
// rely on 1 replica for this tests
createIndex("test1");
createIndex("test2");
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
client()
.prepareIndex("test1", "type1", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test1", "type2", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test2", "type", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
IndicesStatsResponse stats = client().admin().indices().prepareStats().execute().actionGet();
assertThat(stats.getPrimaries().getDocs().getCount(), equalTo(3l));
assertThat(stats.getTotal().getDocs().getCount(), equalTo(6l));
assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexCount(), equalTo(3l));
assertThat(stats.getTotal().getIndexing().getTotal().getIndexCount(), equalTo(6l));
assertThat(stats.getTotal().getStore(), notNullValue());
// verify nulls
assertThat(stats.getTotal().getMerge(), nullValue());
assertThat(stats.getTotal().getFlush(), nullValue());
assertThat(stats.getTotal().getRefresh(), nullValue());
assertThat(stats.getIndex("test1").getPrimaries().getDocs().getCount(), equalTo(2l));
assertThat(stats.getIndex("test1").getTotal().getDocs().getCount(), equalTo(4l));
assertThat(stats.getIndex("test1").getPrimaries().getStore(), notNullValue());
assertThat(stats.getIndex("test1").getPrimaries().getMerge(), nullValue());
assertThat(stats.getIndex("test1").getPrimaries().getFlush(), nullValue());
assertThat(stats.getIndex("test1").getPrimaries().getRefresh(), nullValue());
assertThat(stats.getIndex("test2").getPrimaries().getDocs().getCount(), equalTo(1l));
assertThat(stats.getIndex("test2").getTotal().getDocs().getCount(), equalTo(2l));
// make sure that number of requests in progress is 0
assertThat(
stats.getIndex("test1").getTotal().getIndexing().getTotal().getIndexCurrent(), equalTo(0l));
assertThat(
stats.getIndex("test1").getTotal().getIndexing().getTotal().getDeleteCurrent(),
equalTo(0l));
assertThat(
stats.getIndex("test1").getTotal().getSearch().getTotal().getFetchCurrent(), equalTo(0l));
assertThat(
stats.getIndex("test1").getTotal().getSearch().getTotal().getQueryCurrent(), equalTo(0l));
// check flags
stats =
client()
.admin()
.indices()
.prepareStats()
.setDocs(false)
.setStore(false)
.setIndexing(false)
.setFlush(true)
.setRefresh(true)
.setMerge(true)
.execute()
.actionGet();
assertThat(stats.getTotal().getDocs(), nullValue());
assertThat(stats.getTotal().getStore(), nullValue());
assertThat(stats.getTotal().getIndexing(), nullValue());
assertThat(stats.getTotal().getMerge(), notNullValue());
assertThat(stats.getTotal().getFlush(), notNullValue());
assertThat(stats.getTotal().getRefresh(), notNullValue());
// check types
stats =
client().admin().indices().prepareStats().setTypes("type1", "type").execute().actionGet();
assertThat(
stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCount(),
equalTo(1l));
assertThat(
stats.getPrimaries().getIndexing().getTypeStats().get("type").getIndexCount(), equalTo(1l));
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2"), nullValue());
assertThat(
stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCurrent(),
equalTo(0l));
assertThat(
stats.getPrimaries().getIndexing().getTypeStats().get("type1").getDeleteCurrent(),
equalTo(0l));
assertThat(stats.getTotal().getGet().getCount(), equalTo(0l));
// check get
GetResponse getResponse = client().prepareGet("test1", "type1", "1").execute().actionGet();
assertThat(getResponse.isExists(), equalTo(true));
stats = client().admin().indices().prepareStats().execute().actionGet();
assertThat(stats.getTotal().getGet().getCount(), equalTo(1l));
assertThat(stats.getTotal().getGet().getExistsCount(), equalTo(1l));
assertThat(stats.getTotal().getGet().getMissingCount(), equalTo(0l));
// missing get
getResponse = client().prepareGet("test1", "type1", "2").execute().actionGet();
assertThat(getResponse.isExists(), equalTo(false));
stats = client().admin().indices().prepareStats().execute().actionGet();
assertThat(stats.getTotal().getGet().getCount(), equalTo(2l));
assertThat(stats.getTotal().getGet().getExistsCount(), equalTo(1l));
assertThat(stats.getTotal().getGet().getMissingCount(), equalTo(1l));
// clear all
stats =
client()
.admin()
.indices()
.prepareStats()
.setDocs(false)
.setStore(false)
.setIndexing(false)
.setFlush(true)
.setRefresh(true)
.setMerge(true)
.clear() // reset defaults
.execute()
.actionGet();
assertThat(stats.getTotal().getDocs(), nullValue());
assertThat(stats.getTotal().getStore(), nullValue());
assertThat(stats.getTotal().getIndexing(), nullValue());
assertThat(stats.getTotal().getGet(), nullValue());
assertThat(stats.getTotal().getSearch(), nullValue());
}
@Test
public void testAllFlags() throws Exception {
// rely on 1 replica for this tests
createIndex("test1");
createIndex("test2");
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
client()
.prepareIndex("test1", "type1", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test1", "type2", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test2", "type", Integer.toString(1))
.setSource("field", "value")
.execute()
.actionGet();
client().admin().indices().prepareRefresh().execute().actionGet();
IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
Flag[] values = CommonStatsFlags.Flag.values();
for (Flag flag : values) {
set(flag, builder, false);
}
IndicesStatsResponse stats = builder.execute().actionGet();
for (Flag flag : values) {
assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
assertThat(isSet(flag, stats.getTotal()), equalTo(false));
}
for (Flag flag : values) {
set(flag, builder, true);
}
stats = builder.execute().actionGet();
for (Flag flag : values) {
assertThat(isSet(flag, stats.getPrimaries()), equalTo(true));
assertThat(isSet(flag, stats.getTotal()), equalTo(true));
}
Random random = getRandom();
EnumSet<Flag> flags = EnumSet.noneOf(Flag.class);
for (Flag flag : values) {
if (random.nextBoolean()) {
flags.add(flag);
}
}
for (Flag flag : values) {
set(flag, builder, false); // clear all
}
for (Flag flag : flags) { // set the flags
set(flag, builder, true);
}
stats = builder.execute().actionGet();
for (Flag flag : flags) { // check the flags
assertThat(isSet(flag, stats.getPrimaries()), equalTo(true));
assertThat(isSet(flag, stats.getTotal()), equalTo(true));
}
for (Flag flag : EnumSet.complementOf(flags)) { // check the complement
assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
assertThat(isSet(flag, stats.getTotal()), equalTo(false));
}
}
@Test
public void testMergeStats() {
// rely on 1 replica for this tests
createIndex("test1");
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
// clear all
IndicesStatsResponse stats =
client()
.admin()
.indices()
.prepareStats()
.setDocs(false)
.setStore(false)
.setIndexing(false)
.setFlush(true)
.setRefresh(true)
.setMerge(true)
.clear() // reset defaults
.execute()
.actionGet();
assertThat(stats.getTotal().getDocs(), nullValue());
assertThat(stats.getTotal().getStore(), nullValue());
assertThat(stats.getTotal().getIndexing(), nullValue());
assertThat(stats.getTotal().getGet(), nullValue());
assertThat(stats.getTotal().getSearch(), nullValue());
for (int i = 0; i < 20; i++) {
client()
.prepareIndex("test1", "type1", Integer.toString(i))
.setSource("field", "value")
.execute()
.actionGet();
client()
.prepareIndex("test1", "type2", Integer.toString(i))
.setSource("field", "value")
.execute()
.actionGet();
client().admin().indices().prepareFlush().execute().actionGet();
}
client()
.admin()
.indices()
.prepareOptimize()
.setWaitForMerge(true)
.setMaxNumSegments(1)
.execute()
.actionGet();
stats = client().admin().indices().prepareStats().setMerge(true).execute().actionGet();
assertThat(stats.getTotal().getMerge(), notNullValue());
assertThat(stats.getTotal().getMerge().getTotal(), greaterThan(0l));
}