public void testIndicesStats() {
createIndex("test");
ensureYellow("test");
IndicesStatsResponse indicesStatsResponse =
client().admin().indices().prepareStats().all().get();
assertThat(indicesStatsResponse.getIndices().size(), equalTo(1));
assertThat(indicesStatsResponse.getIndices().containsKey("test"), equalTo(true));
}
@Override
protected void beforeIndexDeletion() throws Exception {
super.beforeIndexDeletion();
assertBusy(
() -> {
IndicesStatsResponse stats = client().admin().indices().prepareStats().clear().get();
for (IndexStats indexStats : stats.getIndices().values()) {
for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) {
Optional<ShardStats> maybePrimary =
Stream.of(indexShardStats.getShards())
.filter(s -> s.getShardRouting().active() && s.getShardRouting().primary())
.findFirst();
if (maybePrimary.isPresent() == false) {
continue;
}
ShardStats primary = maybePrimary.get();
final SeqNoStats primarySeqNoStats = primary.getSeqNoStats();
assertThat(
primary.getShardRouting() + " should have set the global checkpoint",
primarySeqNoStats.getGlobalCheckpoint(),
not(equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO)));
for (ShardStats shardStats : indexShardStats) {
final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
assertThat(
shardStats.getShardRouting() + " local checkpoint mismatch",
seqNoStats.getLocalCheckpoint(),
equalTo(primarySeqNoStats.getLocalCheckpoint()));
assertThat(
shardStats.getShardRouting() + " global checkpoint mismatch",
seqNoStats.getGlobalCheckpoint(),
equalTo(primarySeqNoStats.getGlobalCheckpoint()));
assertThat(
shardStats.getShardRouting() + " max seq no mismatch",
seqNoStats.getMaxSeqNo(),
equalTo(primarySeqNoStats.getMaxSeqNo()));
}
}
}
});
}
@Test
public void testSimpleTTL() throws Exception {
assertAcked(
prepareCreate("test")
.addMapping(
"type1",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.endObject()
.endObject()
.endObject())
.addMapping(
"type2",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type2")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.field("default", "1d")
.endObject()
.endObject()
.endObject()));
ensureYellow("test");
final NumShards test = getNumShards("test");
long providedTTLValue = 3000;
logger.info("--> checking ttl");
// Index one doc without routing, one doc with routing, one doc with not TTL and no default and
// one doc with default TTL
long now = System.currentTimeMillis();
IndexResponse indexResponse =
client()
.prepareIndex("test", "type1", "1")
.setSource("field1", "value1")
.setTimestamp(String.valueOf(now))
.setTTL(providedTTLValue)
.setRefresh(true)
.get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client()
.prepareIndex("test", "type1", "with_routing")
.setSource("field1", "value1")
.setTimestamp(String.valueOf(now))
.setTTL(providedTTLValue)
.setRouting("routing")
.setRefresh(true)
.get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client().prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client().prepareIndex("test", "type2", "default_ttl").setSource("field1", "value1").get();
assertThat(indexResponse.isCreated(), is(true));
// realtime get check
long currentTime = System.currentTimeMillis();
GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
long ttl0;
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// verify the ttl is still decreasing when going to the replica
currentTime = System.currentTimeMillis();
getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// non realtime get (stored)
currentTime = System.currentTimeMillis();
getResponse =
client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// non realtime get going the replica
currentTime = System.currentTimeMillis();
getResponse =
client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// no TTL provided so no TTL fetched
getResponse =
client()
.prepareGet("test", "type1", "no_ttl")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.getField("_ttl"), nullValue());
// no TTL provided make sure it has default TTL
getResponse =
client()
.prepareGet("test", "type2", "default_ttl")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, greaterThan(0L));
IndicesStatsResponse response =
client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
assertThat(
response.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount(),
equalTo(0L));
// make sure the purger has done its job for all indexed docs that are expired
long shouldBeExpiredDate = now + providedTTLValue + PURGE_INTERVAL + 2000;
currentTime = System.currentTimeMillis();
if (shouldBeExpiredDate - currentTime > 0) {
Thread.sleep(shouldBeExpiredDate - currentTime);
}
// We can't assume that after waiting for ttl + purgeInterval (waitTime) that the document have
// actually been deleted.
// The ttl purging happens in the background in a different thread, and might not have been
// completed after waiting for waitTime.
// But we can use index statistics' delete count to be sure that deletes have been executed,
// that must be incremented before
// ttl purging has finished.
logger.info("--> checking purger");
assertThat(
awaitBusy(
new Predicate<Object>() {
@Override
public boolean apply(Object input) {
if (rarely()) {
client().admin().indices().prepareFlush("test").get();
} else if (rarely()) {
client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get();
}
IndicesStatsResponse response =
client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
// TTL deletes two docs, but it is indexed in the primary shard and replica shard.
return response
.getIndices()
.get("test")
.getTotal()
.getIndexing()
.getTotal()
.getDeleteCount()
== 2L * test.dataCopies;
}
},
5,
TimeUnit.SECONDS),
equalTo(true));
// realtime get check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// replica realtime get check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// Need to run a refresh, in order for the non realtime get to work.
client().admin().indices().prepareRefresh("test").execute().actionGet();
// non realtime get (stored) check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// non realtime get going the replica check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
}
