@Override
 public void run() {
   while (true) {
     if (close) {
       closed = true;
       return;
     }
     try {
       indexDoc();
       Thread.sleep(indexerThrottle.millis());
     } catch (Exception e) {
       logger.warn("failed to index / sleep", e);
     }
   }
 }
  public void run() throws Exception {
    Node[] nodes = new Node[numberOfNodes];
    for (int i = 0; i < nodes.length; i++) {
      nodes[i] = NodeBuilder.nodeBuilder().settings(settings).node();
    }
    client = NodeBuilder.nodeBuilder().settings(settings).client(true).node();

    client
        .client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(
            settingsBuilder()
                .put("index.number_of_shards", numberOfShards)
                .put("index.number_of_replicas", numberOfReplicas))
        .execute()
        .actionGet();

    logger.info("********** [START] INDEXING INITIAL DOCS");
    for (long i = 0; i < initialNumberOfDocs; i++) {
      indexDoc();
    }
    logger.info("********** [DONE ] INDEXING INITIAL DOCS");

    Indexer[] indexerThreads = new Indexer[indexers];
    for (int i = 0; i < indexerThreads.length; i++) {
      indexerThreads[i] = new Indexer();
    }
    for (int i = 0; i < indexerThreads.length; i++) {
      indexerThreads[i].start();
    }

    long testStart = System.currentTimeMillis();

    // start doing the rolling restart
    int nodeIndex = 0;
    while (true) {
      File[] nodeData =
          ((InternalNode) nodes[nodeIndex])
              .injector()
              .getInstance(NodeEnvironment.class)
              .nodeDataLocations();
      nodes[nodeIndex].close();
      if (clearNodeData) {
        FileSystemUtils.deleteRecursively(nodeData);
      }

      try {
        ClusterHealthResponse clusterHealth =
            client
                .client()
                .admin()
                .cluster()
                .prepareHealth()
                .setWaitForGreenStatus()
                .setWaitForNodes(Integer.toString(numberOfNodes + 0 /* client node*/))
                .setWaitForRelocatingShards(0)
                .setTimeout("10m")
                .execute()
                .actionGet();
        if (clusterHealth.timedOut()) {
          logger.warn("timed out waiting for green status....");
        }
      } catch (Exception e) {
        logger.warn("failed to execute cluster health....");
      }

      nodes[nodeIndex] = NodeBuilder.nodeBuilder().settings(settings).node();

      Thread.sleep(1000);

      try {
        ClusterHealthResponse clusterHealth =
            client
                .client()
                .admin()
                .cluster()
                .prepareHealth()
                .setWaitForGreenStatus()
                .setWaitForNodes(Integer.toString(numberOfNodes + 1 /* client node*/))
                .setWaitForRelocatingShards(0)
                .setTimeout("10m")
                .execute()
                .actionGet();
        if (clusterHealth.timedOut()) {
          logger.warn("timed out waiting for green status....");
        }
      } catch (Exception e) {
        logger.warn("failed to execute cluster health....");
      }

      if (++nodeIndex == nodes.length) {
        nodeIndex = 0;
      }

      if ((System.currentTimeMillis() - testStart) > period.millis()) {
        logger.info("test finished");
        break;
      }
    }

    for (int i = 0; i < indexerThreads.length; i++) {
      indexerThreads[i].close = true;
    }

    Thread.sleep(indexerThrottle.millis() + 10000);

    for (int i = 0; i < indexerThreads.length; i++) {
      if (!indexerThreads[i].closed) {
        logger.warn("thread not closed!");
      }
    }

    client.client().admin().indices().prepareRefresh().execute().actionGet();

    // check the status
    IndicesStatusResponse status =
        client.client().admin().indices().prepareStatus("test").execute().actionGet();
    for (IndexShardStatus shardStatus : status.index("test")) {
      ShardStatus shard = shardStatus.shards()[0];
      logger.info("shard [{}], docs [{}]", shard.shardId(), shard.getDocs().numDocs());
      for (ShardStatus shardStatu : shardStatus) {
        if (shard.docs().numDocs() != shardStatu.docs().numDocs()) {
          logger.warn(
              "shard doc number does not match!, got {} and {}",
              shard.docs().numDocs(),
              shardStatu.docs().numDocs());
        }
      }
    }

    // check the count
    for (int i = 0; i < (nodes.length * 5); i++) {
      CountResponse count =
          client.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet();
      logger.info(
          "indexed [{}], count [{}], [{}]",
          count.count(),
          indexCounter.get(),
          count.count() == indexCounter.get() ? "OK" : "FAIL");
      if (count.count() != indexCounter.get()) {
        logger.warn("count does not match!");
      }
    }

    // scan all the docs, verify all have the same version based on the number of replicas
    SearchResponse searchResponse =
        client
            .client()
            .prepareSearch()
            .setSearchType(SearchType.SCAN)
            .setQuery(matchAllQuery())
            .setSize(50)
            .setScroll(TimeValue.timeValueMinutes(2))
            .execute()
            .actionGet();
    logger.info("Verifying versions for {} hits...", searchResponse.hits().totalHits());

    while (true) {
      searchResponse =
          client
              .client()
              .prepareSearchScroll(searchResponse.scrollId())
              .setScroll(TimeValue.timeValueMinutes(2))
              .execute()
              .actionGet();
      if (searchResponse.failedShards() > 0) {
        logger.warn("Search Failures " + Arrays.toString(searchResponse.shardFailures()));
      }
      for (SearchHit hit : searchResponse.hits()) {
        long version = -1;
        for (int i = 0; i < (numberOfReplicas + 1); i++) {
          GetResponse getResponse =
              client.client().prepareGet(hit.index(), hit.type(), hit.id()).execute().actionGet();
          if (version == -1) {
            version = getResponse.version();
          } else {
            if (version != getResponse.version()) {
              logger.warn(
                  "Doc {} has different version numbers {} and {}",
                  hit.id(),
                  version,
                  getResponse.version());
            }
          }
        }
      }
      if (searchResponse.hits().hits().length == 0) {
        break;
      }
    }
    logger.info("Done verifying versions");

    client.close();
    for (Node node : nodes) {
      node.close();
    }
  }