public static void main(String[] args) throws Exception {
    System.setProperty("es.logger.prefix", "");
    Natives.tryMlockall();

    Settings settings =
        settingsBuilder()
            .put("gateway.type", "local")
            .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, "false")
            .put(SETTING_NUMBER_OF_SHARDS, 1)
            .put(SETTING_NUMBER_OF_REPLICAS, 0)
            .put(TransportModule.TRANSPORT_TYPE_KEY, "local")
            .build();

    String clusterName = ReplicaRecoveryBenchmark.class.getSimpleName();
    Node node1 =
        nodeBuilder().clusterName(clusterName).settings(settingsBuilder().put(settings)).node();

    final ESLogger logger = ESLoggerFactory.getLogger("benchmark");

    final Client client1 = node1.client();
    client1
        .admin()
        .cluster()
        .prepareUpdateSettings()
        .setPersistentSettings("logger.indices.recovery: TRACE")
        .get();
    final BackgroundIndexer indexer =
        new BackgroundIndexer(
            INDEX_NAME, TYPE_NAME, client1, 0, CONCURRENT_INDEXERS, false, new Random());
    indexer.setMinFieldSize(10);
    indexer.setMaxFieldSize(150);
    try {
      client1.admin().indices().prepareDelete(INDEX_NAME).get();
    } catch (IndexMissingException e) {
    }
    client1.admin().indices().prepareCreate(INDEX_NAME).get();
    indexer.start(DOC_COUNT / 2);
    while (indexer.totalIndexedDocs() < DOC_COUNT / 2) {
      Thread.sleep(5000);
      logger.info("--> indexed {} of {}", indexer.totalIndexedDocs(), DOC_COUNT);
    }
    client1.admin().indices().prepareFlush().get();
    indexer.continueIndexing(DOC_COUNT / 2);
    while (indexer.totalIndexedDocs() < DOC_COUNT) {
      Thread.sleep(5000);
      logger.info("--> indexed {} of {}", indexer.totalIndexedDocs(), DOC_COUNT);
    }

    logger.info("--> starting another node and allocating a shard on it");

    Node node2 =
        nodeBuilder().clusterName(clusterName).settings(settingsBuilder().put(settings)).node();

    client1
        .admin()
        .indices()
        .prepareUpdateSettings(INDEX_NAME)
        .setSettings(IndexMetaData.SETTING_NUMBER_OF_REPLICAS + ": 1")
        .get();

    final AtomicBoolean end = new AtomicBoolean(false);

    final Thread backgroundLogger =
        new Thread(
            new Runnable() {

              long lastTime = System.currentTimeMillis();
              long lastDocs = indexer.totalIndexedDocs();
              long lastBytes = 0;
              long lastTranslogOps = 0;

              @Override
              public void run() {
                while (true) {
                  try {
                    Thread.sleep(5000);
                  } catch (InterruptedException e) {

                  }
                  if (end.get()) {
                    return;
                  }
                  long currentTime = System.currentTimeMillis();
                  long currentDocs = indexer.totalIndexedDocs();
                  RecoveryResponse recoveryResponse =
                      client1
                          .admin()
                          .indices()
                          .prepareRecoveries(INDEX_NAME)
                          .setActiveOnly(true)
                          .get();
                  List<ShardRecoveryResponse> indexRecoveries =
                      recoveryResponse.shardResponses().get(INDEX_NAME);
                  long translogOps;
                  long bytes;
                  if (indexRecoveries.size() > 0) {
                    translogOps =
                        indexRecoveries.get(0).recoveryState().getTranslog().recoveredOperations();
                    bytes =
                        recoveryResponse
                            .shardResponses()
                            .get(INDEX_NAME)
                            .get(0)
                            .recoveryState()
                            .getIndex()
                            .recoveredBytes();
                  } else {
                    bytes = lastBytes = 0;
                    translogOps = lastTranslogOps = 0;
                  }
                  float seconds = (currentTime - lastTime) / 1000.0F;
                  logger.info(
                      "--> indexed [{}];[{}] doc/s, recovered [{}] MB/s , translog ops [{}]/s ",
                      currentDocs,
                      (currentDocs - lastDocs) / seconds,
                      (bytes - lastBytes) / 1024.0F / 1024F / seconds,
                      (translogOps - lastTranslogOps) / seconds);
                  lastBytes = bytes;
                  lastTranslogOps = translogOps;
                  lastTime = currentTime;
                  lastDocs = currentDocs;
                }
              }
            });

    backgroundLogger.start();

    client1.admin().cluster().prepareHealth().setWaitForGreenStatus().get();

    logger.info("--> green. starting relocation cycles");

    long startDocIndexed = indexer.totalIndexedDocs();
    indexer.continueIndexing(DOC_COUNT * 50);

    long totalRecoveryTime = 0;
    long startTime = System.currentTimeMillis();
    long[] recoveryTimes = new long[3];
    for (int iteration = 0; iteration < 3; iteration++) {
      logger.info("--> removing replicas");
      client1
          .admin()
          .indices()
          .prepareUpdateSettings(INDEX_NAME)
          .setSettings(IndexMetaData.SETTING_NUMBER_OF_REPLICAS + ": 0")
          .get();
      logger.info("--> adding replica again");
      long recoveryStart = System.currentTimeMillis();
      client1
          .admin()
          .indices()
          .prepareUpdateSettings(INDEX_NAME)
          .setSettings(IndexMetaData.SETTING_NUMBER_OF_REPLICAS + ": 1")
          .get();
      client1
          .admin()
          .cluster()
          .prepareHealth(INDEX_NAME)
          .setWaitForGreenStatus()
          .setTimeout("15m")
          .get();
      long recoveryTime = System.currentTimeMillis() - recoveryStart;
      totalRecoveryTime += recoveryTime;
      recoveryTimes[iteration] = recoveryTime;
      logger.info("--> recovery done in [{}]", new TimeValue(recoveryTime));

      // sleep some to let things clean up
      Thread.sleep(10000);
    }

    long endDocIndexed = indexer.totalIndexedDocs();
    long totalTime = System.currentTimeMillis() - startTime;
    indexer.stop();

    end.set(true);

    backgroundLogger.interrupt();

    backgroundLogger.join();

    logger.info(
        "average doc/s [{}], average relocation time [{}], taking [{}], [{}], [{}]",
        (endDocIndexed - startDocIndexed) * 1000.0 / totalTime,
        new TimeValue(totalRecoveryTime / 3),
        TimeValue.timeValueMillis(recoveryTimes[0]),
        TimeValue.timeValueMillis(recoveryTimes[1]),
        TimeValue.timeValueMillis(recoveryTimes[2]));

    client1.close();
    node1.close();
    node2.close();
  }
Esempio n. 2
0
  @Test
  public void testRelocationWhileIndexingRandom() throws Exception {
    int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4);
    int numberOfReplicas = randomBoolean() ? 0 : 1;
    int numberOfNodes = numberOfReplicas == 0 ? 2 : 3;

    logger.info(
        "testRelocationWhileIndexingRandom(numRelocations={}, numberOfReplicas={}, numberOfNodes={})",
        numberOfRelocations,
        numberOfReplicas,
        numberOfNodes);

    String[] nodes = new String[numberOfNodes];
    logger.info("--> starting [node1] ...");
    nodes[0] = internalCluster().startNode();

    logger.info("--> creating test index ...");
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(
            settingsBuilder()
                .put("index.number_of_shards", 1)
                .put("index.number_of_replicas", numberOfReplicas))
        .execute()
        .actionGet();

    for (int i = 1; i < numberOfNodes; i++) {
      logger.info("--> starting [node{}] ...", i + 1);
      nodes[i] = internalCluster().startNode();
      if (i != numberOfNodes - 1) {
        ClusterHealthResponse healthResponse =
            client()
                .admin()
                .cluster()
                .prepareHealth()
                .setWaitForEvents(Priority.LANGUID)
                .setWaitForNodes(Integer.toString(i + 1))
                .setWaitForGreenStatus()
                .execute()
                .actionGet();
        assertThat(healthResponse.isTimedOut(), equalTo(false));
      }
    }

    int numDocs = scaledRandomIntBetween(200, 2500);
    try (BackgroundIndexer indexer = new BackgroundIndexer("test", "type1", client(), numDocs)) {
      logger.info("--> waiting for {} docs to be indexed ...", numDocs);
      waitForDocs(numDocs, indexer);
      logger.info("--> {} docs indexed", numDocs);

      logger.info("--> starting relocations...");
      int nodeShiftBased = numberOfReplicas; // if we have replicas shift those
      for (int i = 0; i < numberOfRelocations; i++) {
        int fromNode = (i % 2);
        int toNode = fromNode == 0 ? 1 : 0;
        fromNode += nodeShiftBased;
        toNode += nodeShiftBased;
        numDocs = scaledRandomIntBetween(200, 1000);
        logger.debug("--> Allow indexer to index [{}] documents", numDocs);
        indexer.continueIndexing(numDocs);
        logger.info("--> START relocate the shard from {} to {}", nodes[fromNode], nodes[toNode]);
        client()
            .admin()
            .cluster()
            .prepareReroute()
            .add(new MoveAllocationCommand(new ShardId("test", 0), nodes[fromNode], nodes[toNode]))
            .get();
        if (rarely()) {
          logger.debug("--> flushing");
          client().admin().indices().prepareFlush().get();
        }
        ClusterHealthResponse clusterHealthResponse =
            client()
                .admin()
                .cluster()
                .prepareHealth()
                .setWaitForEvents(Priority.LANGUID)
                .setWaitForRelocatingShards(0)
                .setTimeout(ACCEPTABLE_RELOCATION_TIME)
                .execute()
                .actionGet();
        assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
        clusterHealthResponse =
            client()
                .admin()
                .cluster()
                .prepareHealth()
                .setWaitForEvents(Priority.LANGUID)
                .setWaitForRelocatingShards(0)
                .setTimeout(ACCEPTABLE_RELOCATION_TIME)
                .execute()
                .actionGet();
        assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
        indexer.pauseIndexing();
        logger.info("--> DONE relocate the shard from {} to {}", fromNode, toNode);
      }
      logger.info("--> done relocations");
      logger.info("--> waiting for indexing threads to stop ...");
      indexer.stop();
      logger.info("--> indexing threads stopped");

      logger.info("--> refreshing the index");
      client().admin().indices().prepareRefresh("test").execute().actionGet();
      logger.info("--> searching the index");
      boolean ranOnce = false;
      for (int i = 0; i < 10; i++) {
        try {
          logger.info("--> START search test round {}", i + 1);
          SearchHits hits =
              client()
                  .prepareSearch("test")
                  .setQuery(matchAllQuery())
                  .setSize((int) indexer.totalIndexedDocs())
                  .setNoFields()
                  .execute()
                  .actionGet()
                  .getHits();
          ranOnce = true;
          if (hits.totalHits() != indexer.totalIndexedDocs()) {
            int[] hitIds = new int[(int) indexer.totalIndexedDocs()];
            for (int hit = 0; hit < indexer.totalIndexedDocs(); hit++) {
              hitIds[hit] = hit + 1;
            }
            IntHashSet set = IntHashSet.from(hitIds);
            for (SearchHit hit : hits.hits()) {
              int id = Integer.parseInt(hit.id());
              if (!set.remove(id)) {
                logger.error("Extra id [{}]", id);
              }
            }
            set.forEach(
                new IntProcedure() {

                  @Override
                  public void apply(int value) {
                    logger.error("Missing id [{}]", value);
                  }
                });
          }
          assertThat(hits.totalHits(), equalTo(indexer.totalIndexedDocs()));
          logger.info("--> DONE search test round {}", i + 1);
        } catch (SearchPhaseExecutionException ex) {
          // TODO: the first run fails with this failure, waiting for relocating nodes set to 0 is
          // not enough?
          logger.warn("Got exception while searching.", ex);
        }
      }
      if (!ranOnce) {
        fail();
      }
    }
  }