@Test
  @Slow
  public void testIndexActions() throws Exception {
    startNode("server1");

    logger.info("Running Cluster Health (waiting for node to startup properly)");
    ClusterHealthResponse clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForGreenStatus())
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));

    client("server1").admin().indices().create(createIndexRequest("test")).actionGet();

    closeNode("server1");

    startNode("server1");
    Thread.sleep(500);
    try {
      client("server1").admin().indices().create(createIndexRequest("test")).actionGet();
      assert false : "index should exists";
    } catch (IndexAlreadyExistsException e) {
      // all is well
    }
  }
  @Test
  @TestLogging(value = "cluster.service:TRACE,action.get:TRACE")
  public void testSimpleRecovery() throws Exception {
    assertAcked(prepareCreate("test", 1).execute().actionGet(5000));

    NumShards numShards = getNumShards("test");

    logger.info("Running Cluster Health");
    ensureYellow();

    client()
        .index(indexRequest("test").type("type1").id("1").source(source("1", "test")))
        .actionGet();
    FlushResponse flushResponse =
        client().admin().indices().flush(flushRequest("test")).actionGet();
    assertThat(flushResponse.getTotalShards(), equalTo(numShards.totalNumShards));
    assertThat(flushResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
    assertThat(flushResponse.getFailedShards(), equalTo(0));
    client()
        .index(indexRequest("test").type("type1").id("2").source(source("2", "test")))
        .actionGet();
    RefreshResponse refreshResponse =
        client().admin().indices().refresh(refreshRequest("test")).actionGet();
    assertThat(refreshResponse.getTotalShards(), equalTo(numShards.totalNumShards));
    assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
    assertThat(refreshResponse.getFailedShards(), equalTo(0));

    allowNodes("test", 2);

    logger.info("Running Cluster Health");
    ensureGreen();

    GetResponse getResult;

    for (int i = 0; i < 5; i++) {
      getResult =
          client()
              .get(getRequest("test").type("type1").id("1").operationThreaded(false))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("1").operationThreaded(false))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
    }

    // now start another one so we move some primaries
    allowNodes("test", 3);
    Thread.sleep(200);
    logger.info("Running Cluster Health");
    ensureGreen();

    for (int i = 0; i < 5; i++) {
      getResult = client().get(getRequest("test").type("type1").id("1")).actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult = client().get(getRequest("test").type("type1").id("1")).actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult = client().get(getRequest("test").type("type1").id("1")).actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
    }
  }