@Test
  public void testPendingUpdateTask() throws Exception {
    Settings settings = settingsBuilder().put("discovery.type", "local").build();
    String node_0 = internalCluster().startNode(settings);
    internalCluster().startNodeClient(settings);

    final ClusterService clusterService =
        internalCluster().getInstance(ClusterService.class, node_0);
    final CountDownLatch block1 = new CountDownLatch(1);
    final CountDownLatch invoked1 = new CountDownLatch(1);
    clusterService.submitStateUpdateTask(
        "1",
        new ClusterStateUpdateTask() {
          @Override
          public ClusterState execute(ClusterState currentState) {
            invoked1.countDown();
            try {
              block1.await();
            } catch (InterruptedException e) {
              fail();
            }
            return currentState;
          }

          @Override
          public void onFailure(String source, Throwable t) {
            invoked1.countDown();
            fail();
          }
        });
    invoked1.await();
    final CountDownLatch invoked2 = new CountDownLatch(9);
    for (int i = 2; i <= 10; i++) {
      clusterService.submitStateUpdateTask(
          Integer.toString(i),
          new ProcessedClusterStateUpdateTask() {
            @Override
            public ClusterState execute(ClusterState currentState) {
              return currentState;
            }

            @Override
            public void onFailure(String source, Throwable t) {
              fail();
            }

            @Override
            public void clusterStateProcessed(
                String source, ClusterState oldState, ClusterState newState) {
              invoked2.countDown();
            }
          });
    }

    // there might be other tasks in this node, make sure to only take the ones we add into account
    // in this test

    // The tasks can be re-ordered, so we need to check out-of-order
    Set<String> controlSources =
        new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
    List<PendingClusterTask> pendingClusterTasks = clusterService.pendingTasks();
    assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(10));
    assertThat(pendingClusterTasks.get(0).getSource().string(), equalTo("1"));
    assertThat(pendingClusterTasks.get(0).isExecuting(), equalTo(true));
    for (PendingClusterTask task : pendingClusterTasks) {
      controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());

    controlSources =
        new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
    PendingClusterTasksResponse response =
        internalCluster()
            .clientNodeClient()
            .admin()
            .cluster()
            .preparePendingClusterTasks()
            .execute()
            .actionGet();
    assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(10));
    assertThat(response.pendingTasks().get(0).getSource().string(), equalTo("1"));
    assertThat(response.pendingTasks().get(0).isExecuting(), equalTo(true));
    for (PendingClusterTask task : response) {
      controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());
    block1.countDown();
    invoked2.await();

    // whenever we test for no tasks, we need to awaitBusy since this is a live node
    assertTrue(
        awaitBusy(
            new Predicate<Object>() {
              @Override
              public boolean apply(Object input) {
                return clusterService.pendingTasks().isEmpty();
              }
            }));
    waitNoPendingTasksOnAll();

    final CountDownLatch block2 = new CountDownLatch(1);
    final CountDownLatch invoked3 = new CountDownLatch(1);
    clusterService.submitStateUpdateTask(
        "1",
        new ClusterStateUpdateTask() {
          @Override
          public ClusterState execute(ClusterState currentState) {
            invoked3.countDown();
            try {
              block2.await();
            } catch (InterruptedException e) {
              fail();
            }
            return currentState;
          }

          @Override
          public void onFailure(String source, Throwable t) {
            invoked3.countDown();
            fail();
          }
        });
    invoked3.await();

    for (int i = 2; i <= 5; i++) {
      clusterService.submitStateUpdateTask(
          Integer.toString(i),
          new ClusterStateUpdateTask() {
            @Override
            public ClusterState execute(ClusterState currentState) {
              return currentState;
            }

            @Override
            public void onFailure(String source, Throwable t) {
              fail();
            }
          });
    }
    Thread.sleep(100);

    pendingClusterTasks = clusterService.pendingTasks();
    assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(5));
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
    for (PendingClusterTask task : pendingClusterTasks) {
      controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());

    response =
        internalCluster().clientNodeClient().admin().cluster().preparePendingClusterTasks().get();
    assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(5));
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
    for (PendingClusterTask task : response) {
      if (controlSources.remove(task.getSource().string())) {
        assertThat(task.getTimeInQueueInMillis(), greaterThan(0l));
      }
    }
    assertTrue(controlSources.isEmpty());
    block2.countDown();
  }