public void testIndicesStats() {
    createIndex("test");
    ensureYellow("test");

    IndicesStatsResponse indicesStatsResponse =
        client().admin().indices().prepareStats().all().get();
    assertThat(indicesStatsResponse.getIndices().size(), equalTo(1));
    assertThat(indicesStatsResponse.getIndices().containsKey("test"), equalTo(true));
  }
Пример #2
0
  @Override
  protected void beforeIndexDeletion() throws Exception {
    super.beforeIndexDeletion();
    assertBusy(
        () -> {
          IndicesStatsResponse stats = client().admin().indices().prepareStats().clear().get();
          for (IndexStats indexStats : stats.getIndices().values()) {
            for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) {
              Optional<ShardStats> maybePrimary =
                  Stream.of(indexShardStats.getShards())
                      .filter(s -> s.getShardRouting().active() && s.getShardRouting().primary())
                      .findFirst();
              if (maybePrimary.isPresent() == false) {
                continue;
              }
              ShardStats primary = maybePrimary.get();
              final SeqNoStats primarySeqNoStats = primary.getSeqNoStats();
              assertThat(
                  primary.getShardRouting() + " should have set the global checkpoint",
                  primarySeqNoStats.getGlobalCheckpoint(),
                  not(equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO)));
              for (ShardStats shardStats : indexShardStats) {
                final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
                assertThat(
                    shardStats.getShardRouting() + " local checkpoint mismatch",
                    seqNoStats.getLocalCheckpoint(),
                    equalTo(primarySeqNoStats.getLocalCheckpoint()));

                assertThat(
                    shardStats.getShardRouting() + " global checkpoint mismatch",
                    seqNoStats.getGlobalCheckpoint(),
                    equalTo(primarySeqNoStats.getGlobalCheckpoint()));
                assertThat(
                    shardStats.getShardRouting() + " max seq no mismatch",
                    seqNoStats.getMaxSeqNo(),
                    equalTo(primarySeqNoStats.getMaxSeqNo()));
              }
            }
          }
        });
  }
Пример #3
0
  @Test
  public void testSimpleTTL() throws Exception {
    assertAcked(
        prepareCreate("test")
            .addMapping(
                "type1",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .startObject("type1")
                    .startObject("_timestamp")
                    .field("enabled", true)
                    .field("store", "yes")
                    .endObject()
                    .startObject("_ttl")
                    .field("enabled", true)
                    .endObject()
                    .endObject()
                    .endObject())
            .addMapping(
                "type2",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .startObject("type2")
                    .startObject("_timestamp")
                    .field("enabled", true)
                    .field("store", "yes")
                    .endObject()
                    .startObject("_ttl")
                    .field("enabled", true)
                    .field("default", "1d")
                    .endObject()
                    .endObject()
                    .endObject()));
    ensureYellow("test");

    final NumShards test = getNumShards("test");

    long providedTTLValue = 3000;
    logger.info("--> checking ttl");
    // Index one doc without routing, one doc with routing, one doc with not TTL and no default and
    // one doc with default TTL
    long now = System.currentTimeMillis();
    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type1", "1")
            .setSource("field1", "value1")
            .setTimestamp(String.valueOf(now))
            .setTTL(providedTTLValue)
            .setRefresh(true)
            .get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client()
            .prepareIndex("test", "type1", "with_routing")
            .setSource("field1", "value1")
            .setTimestamp(String.valueOf(now))
            .setTTL(providedTTLValue)
            .setRouting("routing")
            .setRefresh(true)
            .get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client().prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client().prepareIndex("test", "type2", "default_ttl").setSource("field1", "value1").get();
    assertThat(indexResponse.isCreated(), is(true));

    // realtime get check
    long currentTime = System.currentTimeMillis();
    GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
    long ttl0;
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // verify the ttl is still decreasing when going to the replica
    currentTime = System.currentTimeMillis();
    getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // non realtime get (stored)
    currentTime = System.currentTimeMillis();
    getResponse =
        client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // non realtime get going the replica
    currentTime = System.currentTimeMillis();
    getResponse =
        client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }

    // no TTL provided so no TTL fetched
    getResponse =
        client()
            .prepareGet("test", "type1", "no_ttl")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.getField("_ttl"), nullValue());
    // no TTL provided make sure it has default TTL
    getResponse =
        client()
            .prepareGet("test", "type2", "default_ttl")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
    assertThat(ttl0, greaterThan(0L));

    IndicesStatsResponse response =
        client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
    assertThat(
        response.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount(),
        equalTo(0L));

    // make sure the purger has done its job for all indexed docs that are expired
    long shouldBeExpiredDate = now + providedTTLValue + PURGE_INTERVAL + 2000;
    currentTime = System.currentTimeMillis();
    if (shouldBeExpiredDate - currentTime > 0) {
      Thread.sleep(shouldBeExpiredDate - currentTime);
    }

    // We can't assume that after waiting for ttl + purgeInterval (waitTime) that the document have
    // actually been deleted.
    // The ttl purging happens in the background in a different thread, and might not have been
    // completed after waiting for waitTime.
    // But we can use index statistics' delete count to be sure that deletes have been executed,
    // that must be incremented before
    // ttl purging has finished.
    logger.info("--> checking purger");
    assertThat(
        awaitBusy(
            new Predicate<Object>() {
              @Override
              public boolean apply(Object input) {
                if (rarely()) {
                  client().admin().indices().prepareFlush("test").get();
                } else if (rarely()) {
                  client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get();
                }
                IndicesStatsResponse response =
                    client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
                // TTL deletes two docs, but it is indexed in the primary shard and replica shard.
                return response
                        .getIndices()
                        .get("test")
                        .getTotal()
                        .getIndexing()
                        .getTotal()
                        .getDeleteCount()
                    == 2L * test.dataCopies;
              }
            },
            5,
            TimeUnit.SECONDS),
        equalTo(true));

    // realtime get check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    // replica realtime get check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));

    // Need to run a refresh, in order for the non realtime get to work.
    client().admin().indices().prepareRefresh("test").execute().actionGet();

    // non realtime get (stored) check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    // non realtime get going the replica check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
  }