public static void main(String[] args) throws Exception {

    Settings settings = settingsBuilder().put("gateway.type", "none").build();

    Node node1 = nodeBuilder().settings(settings).node();
    Node node2 = nodeBuilder().settings(settings).node();
    final Node client = nodeBuilder().settings(settings).client(true).node();

    final int NUMBER_OF_DOCS = 10000;
    final int NUMBER_OF_THREADS = 10;
    final long NUMBER_OF_ITERATIONS = SizeValue.parseSizeValue("10k").singles();
    final long DELETE_EVERY = 10;

    final CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS);
    Thread[] threads = new Thread[NUMBER_OF_THREADS];
    for (int i = 0; i < threads.length; i++) {
      threads[i] =
          new Thread() {
            @Override
            public void run() {
              try {
                for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) {
                  if ((i % DELETE_EVERY) == 0) {
                    client
                        .client()
                        .prepareDelete(
                            "test",
                            "type1",
                            Integer.toString(ThreadLocalRandom.current().nextInt(NUMBER_OF_DOCS)))
                        .execute()
                        .actionGet();
                  } else {
                    client
                        .client()
                        .prepareIndex(
                            "test",
                            "type1",
                            Integer.toString(ThreadLocalRandom.current().nextInt(NUMBER_OF_DOCS)))
                        .setSource("field1", "value1")
                        .execute()
                        .actionGet();
                  }
                }
              } finally {
                latch.countDown();
              }
            }
          };
    }

    for (Thread thread : threads) {
      thread.start();
    }

    latch.await();
    System.out.println("done indexing, verifying docs");
    client.client().admin().indices().prepareRefresh().execute().actionGet();
    for (int i = 0; i < NUMBER_OF_DOCS; i++) {
      String id = Integer.toString(i);
      for (int j = 0; j < 5; j++) {
        SearchResponse response =
            client
                .client()
                .prepareSearch()
                .setQuery(QueryBuilders.termQuery("_id", id))
                .execute()
                .actionGet();
        if (response.getHits().totalHits() > 1) {
          System.err.println("[" + i + "] FAIL, HITS [" + response.getHits().totalHits() + "]");
        }
      }
      GetResponse getResponse =
          client.client().prepareGet("test", "type1", id).execute().actionGet();
      if (getResponse.exists()) {
        long version = getResponse.version();
        for (int j = 0; j < 5; j++) {
          getResponse = client.client().prepareGet("test", "type1", id).execute().actionGet();
          if (!getResponse.exists()) {
            System.err.println("[" + i + "] FAIL, EXISTED, and NOT_EXISTED");
            break;
          }
          if (version != getResponse.version()) {
            System.err.println(
                "["
                    + i
                    + "] FAIL, DIFFERENT VERSIONS: ["
                    + version
                    + "], ["
                    + getResponse.version()
                    + "]");
            break;
          }
        }
      } else {
        for (int j = 0; j < 5; j++) {
          getResponse = client.client().prepareGet("test", "type1", id).execute().actionGet();
          if (getResponse.exists()) {
            System.err.println("[" + i + "] FAIL, EXISTED, and NOT_EXISTED");
            break;
          }
        }
      }
    }
    System.out.println("done.");

    client.close();
    node1.close();
    node2.close();
  }
  public void run() throws Exception {
    Node[] nodes = new Node[numberOfNodes];
    for (int i = 0; i < nodes.length; i++) {
      nodes[i] = NodeBuilder.nodeBuilder().settings(settings).node();
    }
    client = NodeBuilder.nodeBuilder().settings(settings).client(true).node();

    client
        .client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(
            settingsBuilder()
                .put("index.number_of_shards", numberOfShards)
                .put("index.number_of_replicas", numberOfReplicas))
        .execute()
        .actionGet();

    logger.info("********** [START] INDEXING INITIAL DOCS");
    for (long i = 0; i < initialNumberOfDocs; i++) {
      indexDoc();
    }
    logger.info("********** [DONE ] INDEXING INITIAL DOCS");

    Indexer[] indexerThreads = new Indexer[indexers];
    for (int i = 0; i < indexerThreads.length; i++) {
      indexerThreads[i] = new Indexer();
    }
    for (int i = 0; i < indexerThreads.length; i++) {
      indexerThreads[i].start();
    }

    long testStart = System.currentTimeMillis();

    // start doing the rolling restart
    int nodeIndex = 0;
    while (true) {
      File[] nodeData =
          ((InternalNode) nodes[nodeIndex])
              .injector()
              .getInstance(NodeEnvironment.class)
              .nodeDataLocations();
      nodes[nodeIndex].close();
      if (clearNodeData) {
        FileSystemUtils.deleteRecursively(nodeData);
      }

      try {
        ClusterHealthResponse clusterHealth =
            client
                .client()
                .admin()
                .cluster()
                .prepareHealth()
                .setWaitForGreenStatus()
                .setWaitForNodes(Integer.toString(numberOfNodes + 0 /* client node*/))
                .setWaitForRelocatingShards(0)
                .setTimeout("10m")
                .execute()
                .actionGet();
        if (clusterHealth.timedOut()) {
          logger.warn("timed out waiting for green status....");
        }
      } catch (Exception e) {
        logger.warn("failed to execute cluster health....");
      }

      nodes[nodeIndex] = NodeBuilder.nodeBuilder().settings(settings).node();

      Thread.sleep(1000);

      try {
        ClusterHealthResponse clusterHealth =
            client
                .client()
                .admin()
                .cluster()
                .prepareHealth()
                .setWaitForGreenStatus()
                .setWaitForNodes(Integer.toString(numberOfNodes + 1 /* client node*/))
                .setWaitForRelocatingShards(0)
                .setTimeout("10m")
                .execute()
                .actionGet();
        if (clusterHealth.timedOut()) {
          logger.warn("timed out waiting for green status....");
        }
      } catch (Exception e) {
        logger.warn("failed to execute cluster health....");
      }

      if (++nodeIndex == nodes.length) {
        nodeIndex = 0;
      }

      if ((System.currentTimeMillis() - testStart) > period.millis()) {
        logger.info("test finished");
        break;
      }
    }

    for (int i = 0; i < indexerThreads.length; i++) {
      indexerThreads[i].close = true;
    }

    Thread.sleep(indexerThrottle.millis() + 10000);

    for (int i = 0; i < indexerThreads.length; i++) {
      if (!indexerThreads[i].closed) {
        logger.warn("thread not closed!");
      }
    }

    client.client().admin().indices().prepareRefresh().execute().actionGet();

    // check the status
    IndicesStatusResponse status =
        client.client().admin().indices().prepareStatus("test").execute().actionGet();
    for (IndexShardStatus shardStatus : status.index("test")) {
      ShardStatus shard = shardStatus.shards()[0];
      logger.info("shard [{}], docs [{}]", shard.shardId(), shard.getDocs().numDocs());
      for (ShardStatus shardStatu : shardStatus) {
        if (shard.docs().numDocs() != shardStatu.docs().numDocs()) {
          logger.warn(
              "shard doc number does not match!, got {} and {}",
              shard.docs().numDocs(),
              shardStatu.docs().numDocs());
        }
      }
    }

    // check the count
    for (int i = 0; i < (nodes.length * 5); i++) {
      CountResponse count =
          client.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet();
      logger.info(
          "indexed [{}], count [{}], [{}]",
          count.count(),
          indexCounter.get(),
          count.count() == indexCounter.get() ? "OK" : "FAIL");
      if (count.count() != indexCounter.get()) {
        logger.warn("count does not match!");
      }
    }

    // scan all the docs, verify all have the same version based on the number of replicas
    SearchResponse searchResponse =
        client
            .client()
            .prepareSearch()
            .setSearchType(SearchType.SCAN)
            .setQuery(matchAllQuery())
            .setSize(50)
            .setScroll(TimeValue.timeValueMinutes(2))
            .execute()
            .actionGet();
    logger.info("Verifying versions for {} hits...", searchResponse.hits().totalHits());

    while (true) {
      searchResponse =
          client
              .client()
              .prepareSearchScroll(searchResponse.scrollId())
              .setScroll(TimeValue.timeValueMinutes(2))
              .execute()
              .actionGet();
      if (searchResponse.failedShards() > 0) {
        logger.warn("Search Failures " + Arrays.toString(searchResponse.shardFailures()));
      }
      for (SearchHit hit : searchResponse.hits()) {
        long version = -1;
        for (int i = 0; i < (numberOfReplicas + 1); i++) {
          GetResponse getResponse =
              client.client().prepareGet(hit.index(), hit.type(), hit.id()).execute().actionGet();
          if (version == -1) {
            version = getResponse.version();
          } else {
            if (version != getResponse.version()) {
              logger.warn(
                  "Doc {} has different version numbers {} and {}",
                  hit.id(),
                  version,
                  getResponse.version());
            }
          }
        }
      }
      if (searchResponse.hits().hits().length == 0) {
        break;
      }
    }
    logger.info("Done verifying versions");

    client.close();
    for (Node node : nodes) {
      node.close();
    }
  }