void checkBulkAction(boolean indexShouldBeAutoCreated, BulkRequestBuilder builder) {
    // bulk operation do not throw MasterNotDiscoveredException exceptions. The only test that auto
    // create kicked in and failed is
    // via the timeout, as bulk operation do not wait on blocks.
    TimeValue timeout;
    if (indexShouldBeAutoCreated) {
      // we expect the bulk to fail because it will try to go to the master. Use small timeout and
      // detect it has passed
      timeout = new TimeValue(200);
    } else {
      // the request should fail very quickly - use a large timeout and make sure it didn't pass...
      timeout = new TimeValue(5000);
    }
    builder.setTimeout(timeout);
    long now = System.currentTimeMillis();
    try {
      builder.get();
      fail("Expected ClusterBlockException");
    } catch (ClusterBlockException e) {

      if (indexShouldBeAutoCreated) {
        // timeout is 200
        assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
        assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
      } else {
        // timeout is 5000
        assertThat(System.currentTimeMillis() - now, lessThan(timeout.millis() - 50));
      }
    }
  }
 void checkWriteAction(
     boolean autoCreateIndex, TimeValue timeout, ActionRequestBuilder<?, ?, ?, ?> builder) {
   // we clean the metadata when loosing a master, therefore all operations on indices will auto
   // create it, if allowed
   long now = System.currentTimeMillis();
   try {
     builder.get();
     fail("expected ClusterBlockException or MasterNotDiscoveredException");
   } catch (ClusterBlockException | MasterNotDiscoveredException e) {
     if (e instanceof MasterNotDiscoveredException) {
       assertTrue(autoCreateIndex);
     } else {
       assertFalse(autoCreateIndex);
     }
     // verify we waited before giving up...
     assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
     assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
   }
 }
  @Test
  public void testScrollAndUpdateIndex() throws Exception {
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(Settings.settingsBuilder().put("index.number_of_shards", 5))
        .execute()
        .actionGet();
    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForEvents(Priority.LANGUID)
        .setWaitForGreenStatus()
        .execute()
        .actionGet();

    for (int i = 0; i < 500; i++) {
      client()
          .prepareIndex("test", "tweet", Integer.toString(i))
          .setSource(
              jsonBuilder()
                  .startObject()
                  .field("user", "kimchy")
                  .field("postDate", System.currentTimeMillis())
                  .field("message", "test")
                  .endObject())
          .execute()
          .actionGet();
    }

    client().admin().indices().prepareRefresh().execute().actionGet();

    assertThat(
        client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(),
        equalTo(500l));
    assertThat(
        client()
            .prepareCount()
            .setQuery(termQuery("message", "test"))
            .execute()
            .actionGet()
            .getCount(),
        equalTo(500l));
    assertThat(
        client()
            .prepareCount()
            .setQuery(termQuery("message", "test"))
            .execute()
            .actionGet()
            .getCount(),
        equalTo(500l));
    assertThat(
        client()
            .prepareCount()
            .setQuery(termQuery("message", "update"))
            .execute()
            .actionGet()
            .getCount(),
        equalTo(0l));
    assertThat(
        client()
            .prepareCount()
            .setQuery(termQuery("message", "update"))
            .execute()
            .actionGet()
            .getCount(),
        equalTo(0l));

    SearchResponse searchResponse =
        client()
            .prepareSearch()
            .setQuery(queryStringQuery("user:kimchy"))
            .setSize(35)
            .setScroll(TimeValue.timeValueMinutes(2))
            .addSort("postDate", SortOrder.ASC)
            .execute()
            .actionGet();
    try {
      do {
        for (SearchHit searchHit : searchResponse.getHits().hits()) {
          Map<String, Object> map = searchHit.sourceAsMap();
          map.put("message", "update");
          client()
              .prepareIndex("test", "tweet", searchHit.id())
              .setSource(map)
              .execute()
              .actionGet();
        }
        searchResponse =
            client()
                .prepareSearchScroll(searchResponse.getScrollId())
                .setScroll(TimeValue.timeValueMinutes(2))
                .execute()
                .actionGet();
      } while (searchResponse.getHits().hits().length > 0);

      client().admin().indices().prepareRefresh().execute().actionGet();
      assertThat(
          client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(),
          equalTo(500l));
      assertThat(
          client()
              .prepareCount()
              .setQuery(termQuery("message", "test"))
              .execute()
              .actionGet()
              .getCount(),
          equalTo(0l));
      assertThat(
          client()
              .prepareCount()
              .setQuery(termQuery("message", "test"))
              .execute()
              .actionGet()
              .getCount(),
          equalTo(0l));
      assertThat(
          client()
              .prepareCount()
              .setQuery(termQuery("message", "update"))
              .execute()
              .actionGet()
              .getCount(),
          equalTo(500l));
      assertThat(
          client()
              .prepareCount()
              .setQuery(termQuery("message", "update"))
              .execute()
              .actionGet()
              .getCount(),
          equalTo(500l));
    } finally {
      clearScroll(searchResponse.getScrollId());
    }
  }
  @Test
  public void testNoMasterActions_writeMasterBlock() throws Exception {
    Settings settings =
        settingsBuilder()
            .put("discovery.type", "zen")
            .put("action.auto_create_index", false)
            .put("discovery.zen.minimum_master_nodes", 2)
            .put("discovery.zen.ping_timeout", "200ms")
            .put("discovery.initial_state_timeout", "500ms")
            .put(DiscoverySettings.NO_MASTER_BLOCK, "write")
            .build();

    internalCluster().startNode(settings);
    // start a second node, create an index, and then shut it down so we have no master block
    internalCluster().startNode(settings);
    prepareCreate("test1").setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).get();
    prepareCreate("test2")
        .setSettings(
            IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
        .get();
    client().admin().cluster().prepareHealth("_all").setWaitForGreenStatus().get();
    client().prepareIndex("test1", "type1", "1").setSource("field", "value1").get();
    client().prepareIndex("test2", "type1", "1").setSource("field", "value1").get();
    refresh();

    ensureSearchable("test1", "test2");

    ClusterStateResponse clusterState = client().admin().cluster().prepareState().get();
    logger.info("Cluster state:\n" + clusterState.getState().prettyPrint());

    internalCluster().stopRandomDataNode();
    assertThat(
        awaitBusy(
            new Predicate<Object>() {
              public boolean apply(Object o) {
                ClusterState state =
                    client().admin().cluster().prepareState().setLocal(true).get().getState();
                return state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID);
              }
            }),
        equalTo(true));

    GetResponse getResponse = client().prepareGet("test1", "type1", "1").get();
    assertExists(getResponse);

    CountResponse countResponse = client().prepareCount("test1").get();
    assertHitCount(countResponse, 1l);

    SearchResponse searchResponse = client().prepareSearch("test1").get();
    assertHitCount(searchResponse, 1l);

    countResponse = client().prepareCount("test2").get();
    assertThat(countResponse.getTotalShards(), equalTo(2));
    assertThat(countResponse.getSuccessfulShards(), equalTo(1));

    TimeValue timeout = TimeValue.timeValueMillis(200);
    long now = System.currentTimeMillis();
    try {
      client()
          .prepareUpdate("test1", "type1", "1")
          .setDoc("field", "value2")
          .setTimeout(timeout)
          .get();
      fail("Expected ClusterBlockException");
    } catch (ClusterBlockException e) {
      assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
      assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
    }

    now = System.currentTimeMillis();
    try {
      client()
          .prepareIndex("test1", "type1", "1")
          .setSource(XContentFactory.jsonBuilder().startObject().endObject())
          .setTimeout(timeout)
          .get();
      fail("Expected ClusterBlockException");
    } catch (ClusterBlockException e) {
      assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
      assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
    }

    internalCluster().startNode(settings);
    client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").get();
  }