public void testLocalNodeMasterListenerCallbacks() throws Exception {
    Settings settings =
        Settings.builder()
            .put("discovery.type", "zen")
            .put("discovery.zen.minimum_master_nodes", 1)
            .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "400ms")
            .put("discovery.initial_state_timeout", "500ms")
            .build();

    String node_0 = internalCluster().startNode(settings);
    ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
    MasterAwareService testService = internalCluster().getInstance(MasterAwareService.class);

    ClusterHealthResponse clusterHealth =
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setWaitForNodes("1")
            .get();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));

    // the first node should be a master as the minimum required is 1
    assertThat(clusterService.state().nodes().getMasterNode(), notNullValue());
    assertThat(clusterService.state().nodes().isLocalNodeElectedMaster(), is(true));
    assertThat(testService.master(), is(true));

    String node_1 = internalCluster().startNode(settings);
    final ClusterService clusterService1 =
        internalCluster().getInstance(ClusterService.class, node_1);
    MasterAwareService testService1 =
        internalCluster().getInstance(MasterAwareService.class, node_1);

    clusterHealth =
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setWaitForNodes("2")
            .get();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));

    // the second node should not be the master as node1 is already the master.
    assertThat(clusterService1.state().nodes().isLocalNodeElectedMaster(), is(false));
    assertThat(testService1.master(), is(false));

    internalCluster().stopCurrentMasterNode();
    clusterHealth =
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setWaitForNodes("1")
            .get();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));

    // now that node0 is closed, node1 should be elected as master
    assertThat(clusterService1.state().nodes().isLocalNodeElectedMaster(), is(true));
    assertThat(testService1.master(), is(true));

    // start another node and set min_master_node
    internalCluster().startNode(Settings.builder().put(settings));
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());

    Settings transientSettings =
        Settings.builder().put("discovery.zen.minimum_master_nodes", 2).build();
    client()
        .admin()
        .cluster()
        .prepareUpdateSettings()
        .setTransientSettings(transientSettings)
        .get();

    // and shutdown the second node
    internalCluster().stopRandomNonMasterNode();

    // there should not be any master as the minimum number of required eligible masters is not met
    awaitBusy(
        () ->
            clusterService1.state().nodes().getMasterNode() == null
                && clusterService1.state().status() == ClusterState.ClusterStateStatus.APPLIED);
    assertThat(testService1.master(), is(false));

    // bring the node back up
    String node_2 =
        internalCluster().startNode(Settings.builder().put(settings).put(transientSettings));
    ClusterService clusterService2 = internalCluster().getInstance(ClusterService.class, node_2);
    MasterAwareService testService2 =
        internalCluster().getInstance(MasterAwareService.class, node_2);

    // make sure both nodes see each other otherwise the masternode below could be null if node 2 is
    // master and node 1 did'r receive
    // the updated cluster state...
    assertThat(
        internalCluster()
            .client(node_1)
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setLocal(true)
            .setWaitForNodes("2")
            .get()
            .isTimedOut(),
        is(false));
    assertThat(
        internalCluster()
            .client(node_2)
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setLocal(true)
            .setWaitForNodes("2")
            .get()
            .isTimedOut(),
        is(false));

    // now that we started node1 again, a new master should be elected
    assertThat(clusterService2.state().nodes().getMasterNode(), is(notNullValue()));
    if (node_2.equals(clusterService2.state().nodes().getMasterNode().getName())) {
      assertThat(testService1.master(), is(false));
      assertThat(testService2.master(), is(true));
    } else {
      assertThat(testService1.master(), is(true));
      assertThat(testService2.master(), is(false));
    }
  }
  @Test
  public void testLocalNodeMasterListenerCallbacks() throws Exception {
    Settings settings =
        settingsBuilder()
            .put("discovery.type", "zen")
            .put("discovery.zen.minimum_master_nodes", 1)
            .put("discovery.zen.ping_timeout", "400ms")
            .put("discovery.initial_state_timeout", "500ms")
            .put("plugin.types", TestPlugin.class.getName())
            .build();

    internalCluster().startNode(settings);
    ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
    MasterAwareService testService = internalCluster().getInstance(MasterAwareService.class);

    ClusterHealthResponse clusterHealth =
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setWaitForNodes("1")
            .get();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));

    // the first node should be a master as the minimum required is 1
    assertThat(clusterService.state().nodes().masterNode(), notNullValue());
    assertThat(clusterService.state().nodes().localNodeMaster(), is(true));
    assertThat(testService.master(), is(true));

    String node_1 = internalCluster().startNode(settings);
    final ClusterService clusterService1 =
        internalCluster().getInstance(ClusterService.class, node_1);
    MasterAwareService testService1 =
        internalCluster().getInstance(MasterAwareService.class, node_1);

    clusterHealth =
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setWaitForNodes("2")
            .get();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));

    // the second node should not be the master as node1 is already the master.
    assertThat(clusterService1.state().nodes().localNodeMaster(), is(false));
    assertThat(testService1.master(), is(false));

    internalCluster().stopCurrentMasterNode();
    clusterHealth =
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setWaitForNodes("1")
            .get();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));

    // now that node1 is closed, node2 should be elected as master
    assertThat(clusterService1.state().nodes().localNodeMaster(), is(true));
    assertThat(testService1.master(), is(true));

    Settings transientSettings =
        settingsBuilder().put("discovery.zen.minimum_master_nodes", 2).build();
    client()
        .admin()
        .cluster()
        .prepareUpdateSettings()
        .setTransientSettings(transientSettings)
        .get();

    // there should not be any master as the minimum number of required eligible masters is not met
    awaitBusy(
        new Predicate<Object>() {
          public boolean apply(Object obj) {
            return clusterService1.state().nodes().masterNode() == null;
          }
        });
    assertThat(testService1.master(), is(false));

    String node_2 =
        internalCluster()
            .startNode(ImmutableSettings.builder().put(settings).put(transientSettings));
    ClusterService clusterService2 = internalCluster().getInstance(ClusterService.class, node_2);
    MasterAwareService testService2 =
        internalCluster().getInstance(MasterAwareService.class, node_2);

    // make sure both nodes see each other otherwise the masternode below could be null if node 2 is
    // master and node 1 did'r receive the updated cluster state...
    assertThat(
        internalCluster()
            .client(node_1)
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setLocal(true)
            .setWaitForNodes("2")
            .get()
            .isTimedOut(),
        is(false));
    assertThat(
        internalCluster()
            .client(node_2)
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForEvents(Priority.LANGUID)
            .setLocal(true)
            .setWaitForNodes("2")
            .get()
            .isTimedOut(),
        is(false));

    // now that we started node1 again, a new master should be elected
    assertThat(clusterService2.state().nodes().masterNode(), is(notNullValue()));
    if (node_2.equals(clusterService2.state().nodes().masterNode().name())) {
      assertThat(testService1.master(), is(false));
      assertThat(testService2.master(), is(true));
    } else {
      assertThat(testService1.master(), is(true));
      assertThat(testService2.master(), is(false));
    }
  }