public void testScalingExecutorType() throws InterruptedException {
    String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING);
    ThreadPool threadPool = null;
    try {
      Settings nodeSettings =
          Settings.builder()
              .put("threadpool." + threadPoolName + ".size", 10)
              .put("node.name", "testScalingExecutorType")
              .build();
      threadPool = new ThreadPool(nodeSettings);
      ClusterSettings clusterSettings =
          new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
      threadPool.setClusterSettings(clusterSettings);
      final int expectedMinimum = "generic".equals(threadPoolName) ? 4 : 1;
      assertThat(info(threadPool, threadPoolName).getMin(), equalTo(expectedMinimum));
      assertThat(info(threadPool, threadPoolName).getMax(), equalTo(10));
      final long expectedKeepAlive = "generic".equals(threadPoolName) ? 30 : 300;
      assertThat(
          info(threadPool, threadPoolName).getKeepAlive().seconds(), equalTo(expectedKeepAlive));
      assertEquals(
          info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING);
      assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));

      // Change settings that doesn't require pool replacement
      Executor oldExecutor = threadPool.executor(threadPoolName);
      clusterSettings.applySettings(
          Settings.builder()
              .put("threadpool." + threadPoolName + ".keep_alive", "10m")
              .put("threadpool." + threadPoolName + ".min", "2")
              .put("threadpool." + threadPoolName + ".size", "15")
              .build());
      assertEquals(
          info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING);
      assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(),
          equalTo(2));
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(),
          equalTo(15));
      assertThat(info(threadPool, threadPoolName).getMin(), equalTo(2));
      assertThat(info(threadPool, threadPoolName).getMax(), equalTo(15));
      // Make sure keep alive value changed
      assertThat(info(threadPool, threadPoolName).getKeepAlive().minutes(), equalTo(10L));
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName))
              .getKeepAliveTime(TimeUnit.MINUTES),
          equalTo(10L));
      assertThat(threadPool.executor(threadPoolName), sameInstance(oldExecutor));
    } finally {
      terminateThreadPoolIfNeeded(threadPool);
    }
  }
 public FilterAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
   super(settings);
   setClusterRequireFilters(CLUSTER_ROUTING_REQUIRE_GROUP_SETTING.get(settings));
   setClusterExcludeFilters(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING.get(settings));
   setClusterIncludeFilters(CLUSTER_ROUTING_INCLUDE_GROUP_SETTING.get(settings));
   clusterSettings.addSettingsUpdateConsumer(
       CLUSTER_ROUTING_REQUIRE_GROUP_SETTING, this::setClusterRequireFilters);
   clusterSettings.addSettingsUpdateConsumer(
       CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING, this::setClusterExcludeFilters);
   clusterSettings.addSettingsUpdateConsumer(
       CLUSTER_ROUTING_INCLUDE_GROUP_SETTING, this::setClusterIncludeFilters);
 }
 public ConcurrentRebalanceAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
   super(settings);
   this.clusterConcurrentRebalance =
       CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING.get(settings);
   logger.debug("using [cluster_concurrent_rebalance] with [{}]", clusterConcurrentRebalance);
   clusterSettings.addSettingsUpdateConsumer(
       CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING,
       this::setClusterConcurrentRebalance);
 }
  public void testIndexingThreadPoolsMaxSize() throws InterruptedException {
    String threadPoolName = randomThreadPoolName();
    for (String name : new String[] {ThreadPool.Names.BULK, ThreadPool.Names.INDEX}) {
      ThreadPool threadPool = null;
      try {

        int maxSize = EsExecutors.boundedNumberOfProcessors(Settings.EMPTY);

        // try to create a too-big (maxSize+1) thread pool
        threadPool =
            new ThreadPool(
                Settings.builder()
                    .put("node.name", "testIndexingThreadPoolsMaxSize")
                    .put("threadpool." + name + ".size", maxSize + 1)
                    .build());

        // confirm it clipped us at the maxSize:
        assertEquals(
            maxSize, ((ThreadPoolExecutor) threadPool.executor(name)).getMaximumPoolSize());

        ClusterSettings clusterSettings =
            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
        threadPool.setClusterSettings(clusterSettings);

        // update it to a tiny size:
        clusterSettings.applySettings(
            Settings.builder().put("threadpool." + name + ".size", 1).build());

        // confirm it worked:
        assertEquals(1, ((ThreadPoolExecutor) threadPool.executor(name)).getMaximumPoolSize());

        // try to update to too-big size:
        clusterSettings.applySettings(
            Settings.builder().put("threadpool." + name + ".size", maxSize + 1).build());

        // confirm it clipped us at the maxSize:
        assertEquals(
            maxSize, ((ThreadPoolExecutor) threadPool.executor(name)).getMaximumPoolSize());
      } finally {
        terminateThreadPoolIfNeeded(threadPool);
      }
    }
  }
  public void testUpdateSettingsCanNotChangeThreadPoolType() throws InterruptedException {
    String threadPoolName = randomThreadPoolName();
    ThreadPool.ThreadPoolType invalidThreadPoolType = randomIncorrectThreadPoolType(threadPoolName);
    ThreadPool.ThreadPoolType validThreadPoolType =
        ThreadPool.THREAD_POOL_TYPES.get(threadPoolName);
    ThreadPool threadPool = null;
    try {
      threadPool =
          new ThreadPool(
              Settings.builder()
                  .put("node.name", "testUpdateSettingsCanNotChangeThreadPoolType")
                  .build());
      ClusterSettings clusterSettings =
          new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
      threadPool.setClusterSettings(clusterSettings);

      clusterSettings.applySettings(
          Settings.builder()
              .put("threadpool." + threadPoolName + ".type", invalidThreadPoolType.getType())
              .build());
      fail("expected IllegalArgumentException");
    } catch (IllegalArgumentException e) {
      assertEquals(
          "illegal value can't update [threadpool.] from [{}] to [{"
              + threadPoolName
              + ".type="
              + invalidThreadPoolType.getType()
              + "}]",
          e.getMessage());
      assertThat(
          e.getCause().getMessage(),
          is(
              "setting threadpool."
                  + threadPoolName
                  + ".type to "
                  + invalidThreadPoolType.getType()
                  + " is not permitted; must be "
                  + validThreadPoolType.getType()));
    } finally {
      terminateThreadPoolIfNeeded(threadPool);
    }
  }
  public void testShutdownNowInterrupts() throws Exception {
    String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.FIXED);
    ThreadPool threadPool = null;
    try {
      Settings nodeSettings =
          Settings.builder()
              .put("threadpool." + threadPoolName + ".queue_size", 1000)
              .put("node.name", "testShutdownNowInterrupts")
              .build();
      threadPool = new ThreadPool(nodeSettings);
      ClusterSettings clusterSettings =
          new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
      threadPool.setClusterSettings(clusterSettings);
      assertEquals(info(threadPool, threadPoolName).getQueueSize().getSingles(), 1000L);

      final CountDownLatch latch = new CountDownLatch(1);
      ThreadPoolExecutor oldExecutor = (ThreadPoolExecutor) threadPool.executor(threadPoolName);
      threadPool
          .executor(threadPoolName)
          .execute(
              () -> {
                try {
                  new CountDownLatch(1).await();
                } catch (InterruptedException ex) {
                  latch.countDown();
                  Thread.currentThread().interrupt();
                }
              });
      clusterSettings.applySettings(
          Settings.builder().put("threadpool." + threadPoolName + ".queue_size", 2000).build());
      assertThat(threadPool.executor(threadPoolName), not(sameInstance(oldExecutor)));
      assertThat(oldExecutor.isShutdown(), equalTo(true));
      assertThat(oldExecutor.isTerminating(), equalTo(true));
      assertThat(oldExecutor.isTerminated(), equalTo(false));
      threadPool.shutdownNow(); // should interrupt the thread
      latch.await(
          3, TimeUnit.SECONDS); // If this throws then ThreadPool#shutdownNow didn't interrupt
    } finally {
      terminateThreadPoolIfNeeded(threadPool);
    }
  }
 @Inject
 public IndicesTTLService(
     Settings settings,
     ClusterService clusterService,
     IndicesService indicesService,
     ClusterSettings clusterSettings,
     TransportBulkAction bulkAction) {
   super(settings);
   this.clusterService = clusterService;
   this.indicesService = indicesService;
   TimeValue interval = INDICES_TTL_INTERVAL_SETTING.get(settings);
   this.bulkAction = bulkAction;
   this.bulkSize = this.settings.getAsInt("indices.ttl.bulk_size", 10000);
   this.purgerThread =
       new PurgerThread(EsExecutors.threadName(settings, "[ttl_expire]"), interval);
   clusterSettings.addSettingsUpdateConsumer(
       INDICES_TTL_INTERVAL_SETTING, this.purgerThread::resetInterval);
 }
  @Inject
  public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
    super(settings);

    this.retryDelayStateSync = INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.get(settings);
    // doesn't have to be fast as nodes are reconnected every 10s by default (see
    // InternalClusterService.ReconnectToNodes)
    // and we want to give the master time to remove a faulty node
    this.retryDelayNetwork = INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.get(settings);

    this.internalActionTimeout = INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.get(settings);
    this.internalActionLongTimeout =
        INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.get(settings);

    this.activityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings);
    this.maxBytesPerSec = INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.get(settings);
    if (maxBytesPerSec.getBytes() <= 0) {
      rateLimiter = null;
    } else {
      rateLimiter = new SimpleRateLimiter(maxBytesPerSec.getMbFrac());
    }

    logger.debug("using max_bytes_per_sec[{}]", maxBytesPerSec);

    clusterSettings.addSettingsUpdateConsumer(
        INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING, this::setMaxBytesPerSec);
    clusterSettings.addSettingsUpdateConsumer(
        INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING, this::setRetryDelayStateSync);
    clusterSettings.addSettingsUpdateConsumer(
        INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING, this::setRetryDelayNetwork);
    clusterSettings.addSettingsUpdateConsumer(
        INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING, this::setInternalActionTimeout);
    clusterSettings.addSettingsUpdateConsumer(
        INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING, this::setInternalActionLongTimeout);
    clusterSettings.addSettingsUpdateConsumer(
        INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, this::setActivityTimeout);
  }
  public void testCustomThreadPool() throws Exception {
    ThreadPool threadPool = null;
    try {
      Settings nodeSettings =
          Settings.builder()
              .put("threadpool.my_pool1.type", "scaling")
              .put("threadpool.my_pool1.min", 1)
              .put(
                  "threadpool.my_pool1.size", EsExecutors.boundedNumberOfProcessors(Settings.EMPTY))
              .put("threadpool.my_pool1.keep_alive", "1m")
              .put("threadpool.my_pool2.type", "fixed")
              .put("threadpool.my_pool2.size", "1")
              .put("threadpool.my_pool2.queue_size", "1")
              .put("node.name", "testCustomThreadPool")
              .build();
      threadPool = new ThreadPool(nodeSettings);
      ClusterSettings clusterSettings =
          new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
      threadPool.setClusterSettings(clusterSettings);
      ThreadPoolInfo groups = threadPool.info();
      boolean foundPool1 = false;
      boolean foundPool2 = false;
      outer:
      for (ThreadPool.Info info : groups) {
        if ("my_pool1".equals(info.getName())) {
          foundPool1 = true;
          assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING);
        } else if ("my_pool2".equals(info.getName())) {
          foundPool2 = true;
          assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
          assertThat(info.getMin(), equalTo(1));
          assertThat(info.getMax(), equalTo(1));
          assertThat(info.getQueueSize().singles(), equalTo(1L));
        } else {
          for (Field field : Names.class.getFields()) {
            if (info.getName().equalsIgnoreCase(field.getName())) {
              // This is ok it is a default thread pool
              continue outer;
            }
          }
          fail("Unexpected pool name: " + info.getName());
        }
      }
      assertThat(foundPool1, is(true));
      assertThat(foundPool2, is(true));

      // Updating my_pool2
      Settings settings = Settings.builder().put("threadpool.my_pool2.size", "10").build();
      clusterSettings.applySettings(settings);

      groups = threadPool.info();
      foundPool1 = false;
      foundPool2 = false;
      outer:
      for (ThreadPool.Info info : groups) {
        if ("my_pool1".equals(info.getName())) {
          foundPool1 = true;
          assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING);
        } else if ("my_pool2".equals(info.getName())) {
          foundPool2 = true;
          assertThat(info.getMax(), equalTo(10));
          assertThat(info.getMin(), equalTo(10));
          assertThat(info.getQueueSize().singles(), equalTo(1L));
          assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
        } else {
          for (Field field : Names.class.getFields()) {
            if (info.getName().equalsIgnoreCase(field.getName())) {
              // This is ok it is a default thread pool
              continue outer;
            }
          }
          fail("Unexpected pool name: " + info.getName());
        }
      }
      assertThat(foundPool1, is(true));
      assertThat(foundPool2, is(true));
    } finally {
      terminateThreadPoolIfNeeded(threadPool);
    }
  }
  public void testFixedExecutorType() throws InterruptedException {
    String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.FIXED);
    ThreadPool threadPool = null;

    try {
      Settings nodeSettings = Settings.builder().put("node.name", "testFixedExecutorType").build();
      threadPool = new ThreadPool(nodeSettings);
      ClusterSettings clusterSettings =
          new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
      threadPool.setClusterSettings(clusterSettings);
      assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
      Settings settings =
          clusterSettings.applySettings(
              Settings.builder().put("threadpool." + threadPoolName + ".size", "15").build());

      int expectedSize = getExpectedThreadPoolSize(nodeSettings, threadPoolName, 15);
      assertEquals(
          info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
      assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(),
          equalTo(expectedSize));
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(),
          equalTo(expectedSize));
      assertThat(info(threadPool, threadPoolName).getMin(), equalTo(expectedSize));
      assertThat(info(threadPool, threadPoolName).getMax(), equalTo(expectedSize));
      // keep alive does not apply to fixed thread pools
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName))
              .getKeepAliveTime(TimeUnit.MINUTES),
          equalTo(0L));

      // Put old type back
      settings = clusterSettings.applySettings(Settings.EMPTY);
      assertEquals(
          info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
      // Make sure keep alive value is not used
      assertThat(info(threadPool, threadPoolName).getKeepAlive(), nullValue());
      // Make sure keep pool size value were reused
      assertThat(info(threadPool, threadPoolName).getMin(), equalTo(expectedSize));
      assertThat(info(threadPool, threadPoolName).getMax(), equalTo(expectedSize));
      assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(),
          equalTo(expectedSize));
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(),
          equalTo(expectedSize));

      // Change size
      Executor oldExecutor = threadPool.executor(threadPoolName);
      settings =
          clusterSettings.applySettings(
              Settings.builder()
                  .put(settings)
                  .put("threadpool." + threadPoolName + ".size", "10")
                  .build());

      expectedSize = getExpectedThreadPoolSize(nodeSettings, threadPoolName, 10);

      // Make sure size values changed
      assertThat(info(threadPool, threadPoolName).getMax(), equalTo(expectedSize));
      assertThat(info(threadPool, threadPoolName).getMin(), equalTo(expectedSize));
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(),
          equalTo(expectedSize));
      assertThat(
          ((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(),
          equalTo(expectedSize));
      // Make sure executor didn't change
      assertEquals(
          info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
      assertThat(threadPool.executor(threadPoolName), sameInstance(oldExecutor));

      // Change queue capacity
      clusterSettings.applySettings(
          Settings.builder()
              .put(settings)
              .put("threadpool." + threadPoolName + ".queue", "500")
              .build());
    } finally {
      terminateThreadPoolIfNeeded(threadPool);
    }
  }
  <T> void runTasksForExecutor(ClusterStateTaskExecutor<T> executor) {
    final ArrayList<UpdateTask<T>> toExecute = new ArrayList<>();
    final Map<String, ArrayList<T>> processTasksBySource = new HashMap<>();
    synchronized (updateTasksPerExecutor) {
      List<UpdateTask> pending = updateTasksPerExecutor.remove(executor);
      if (pending != null) {
        for (UpdateTask<T> task : pending) {
          if (task.processed.getAndSet(true) == false) {
            logger.trace("will process {}", task.toString(executor));
            toExecute.add(task);
            processTasksBySource
                .computeIfAbsent(task.source, s -> new ArrayList<>())
                .add(task.task);
          } else {
            logger.trace("skipping {}, already processed", task.toString(executor));
          }
        }
      }
    }
    if (toExecute.isEmpty()) {
      return;
    }
    final String tasksSummary =
        processTasksBySource
            .entrySet()
            .stream()
            .map(
                entry -> {
                  String tasks = executor.describeTasks(entry.getValue());
                  return tasks.isEmpty() ? entry.getKey() : entry.getKey() + "[" + tasks + "]";
                })
            .reduce((s1, s2) -> s1 + ", " + s2)
            .orElse("");

    if (!lifecycle.started()) {
      logger.debug("processing [{}]: ignoring, cluster_service not started", tasksSummary);
      return;
    }
    logger.debug("processing [{}]: execute", tasksSummary);
    ClusterState previousClusterState = clusterState;
    if (!previousClusterState.nodes().isLocalNodeElectedMaster() && executor.runOnlyOnMaster()) {
      logger.debug("failing [{}]: local node is no longer master", tasksSummary);
      toExecute.stream().forEach(task -> task.listener.onNoLongerMaster(task.source));
      return;
    }
    ClusterStateTaskExecutor.BatchResult<T> batchResult;
    long startTimeNS = currentTimeInNanos();
    try {
      List<T> inputs =
          toExecute.stream().map(tUpdateTask -> tUpdateTask.task).collect(Collectors.toList());
      batchResult = executor.execute(previousClusterState, inputs);
    } catch (Exception e) {
      TimeValue executionTime =
          TimeValue.timeValueMillis(
              Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
      if (logger.isTraceEnabled()) {
        logger.trace(
            (Supplier<?>)
                () ->
                    new ParameterizedMessage(
                        "failed to execute cluster state update in [{}], state:\nversion [{}], source [{}]\n{}{}{}",
                        executionTime,
                        previousClusterState.version(),
                        tasksSummary,
                        previousClusterState.nodes().prettyPrint(),
                        previousClusterState.routingTable().prettyPrint(),
                        previousClusterState.getRoutingNodes().prettyPrint()),
            e);
      }
      warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
      batchResult =
          ClusterStateTaskExecutor.BatchResult.<T>builder()
              .failures(toExecute.stream().map(updateTask -> updateTask.task)::iterator, e)
              .build(previousClusterState);
    }

    assert batchResult.executionResults != null;
    assert batchResult.executionResults.size() == toExecute.size()
        : String.format(
            Locale.ROOT,
            "expected [%d] task result%s but was [%d]",
            toExecute.size(),
            toExecute.size() == 1 ? "" : "s",
            batchResult.executionResults.size());
    boolean assertsEnabled = false;
    assert (assertsEnabled = true);
    if (assertsEnabled) {
      for (UpdateTask<T> updateTask : toExecute) {
        assert batchResult.executionResults.containsKey(updateTask.task)
            : "missing task result for " + updateTask.toString(executor);
      }
    }

    ClusterState newClusterState = batchResult.resultingState;
    final ArrayList<UpdateTask<T>> proccessedListeners = new ArrayList<>();
    // fail all tasks that have failed and extract those that are waiting for results
    for (UpdateTask<T> updateTask : toExecute) {
      assert batchResult.executionResults.containsKey(updateTask.task)
          : "missing " + updateTask.toString(executor);
      final ClusterStateTaskExecutor.TaskResult executionResult =
          batchResult.executionResults.get(updateTask.task);
      executionResult.handle(
          () -> proccessedListeners.add(updateTask),
          ex -> {
            logger.debug(
                (Supplier<?>)
                    () ->
                        new ParameterizedMessage(
                            "cluster state update task {} failed", updateTask.toString(executor)),
                ex);
            updateTask.listener.onFailure(updateTask.source, ex);
          });
    }

    if (previousClusterState == newClusterState) {
      for (UpdateTask<T> task : proccessedListeners) {
        if (task.listener instanceof AckedClusterStateTaskListener) {
          // no need to wait for ack if nothing changed, the update can be counted as acknowledged
          ((AckedClusterStateTaskListener) task.listener).onAllNodesAcked(null);
        }
        task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState);
      }
      TimeValue executionTime =
          TimeValue.timeValueMillis(
              Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
      logger.debug(
          "processing [{}]: took [{}] no change in cluster_state", tasksSummary, executionTime);
      warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
      return;
    }

    try {
      ArrayList<Discovery.AckListener> ackListeners = new ArrayList<>();
      if (newClusterState.nodes().isLocalNodeElectedMaster()) {
        // only the master controls the version numbers
        Builder builder = ClusterState.builder(newClusterState).incrementVersion();
        if (previousClusterState.routingTable() != newClusterState.routingTable()) {
          builder.routingTable(
              RoutingTable.builder(newClusterState.routingTable())
                  .version(newClusterState.routingTable().version() + 1)
                  .build());
        }
        if (previousClusterState.metaData() != newClusterState.metaData()) {
          builder.metaData(
              MetaData.builder(newClusterState.metaData())
                  .version(newClusterState.metaData().version() + 1));
        }
        newClusterState = builder.build();
        for (UpdateTask<T> task : proccessedListeners) {
          if (task.listener instanceof AckedClusterStateTaskListener) {
            final AckedClusterStateTaskListener ackedListener =
                (AckedClusterStateTaskListener) task.listener;
            if (ackedListener.ackTimeout() == null || ackedListener.ackTimeout().millis() == 0) {
              ackedListener.onAckTimeout();
            } else {
              try {
                ackListeners.add(
                    new AckCountDownListener(
                        ackedListener,
                        newClusterState.version(),
                        newClusterState.nodes(),
                        threadPool));
              } catch (EsRejectedExecutionException ex) {
                if (logger.isDebugEnabled()) {
                  logger.debug(
                      "Couldn't schedule timeout thread - node might be shutting down", ex);
                }
                // timeout straightaway, otherwise we could wait forever as the timeout thread has
                // not started
                ackedListener.onAckTimeout();
              }
            }
          }
        }
      }
      final Discovery.AckListener ackListener = new DelegetingAckListener(ackListeners);

      newClusterState.status(ClusterState.ClusterStateStatus.BEING_APPLIED);

      if (logger.isTraceEnabled()) {
        logger.trace(
            "cluster state updated, source [{}]\n{}", tasksSummary, newClusterState.prettyPrint());
      } else if (logger.isDebugEnabled()) {
        logger.debug(
            "cluster state updated, version [{}], source [{}]",
            newClusterState.version(),
            tasksSummary);
      }

      ClusterChangedEvent clusterChangedEvent =
          new ClusterChangedEvent(tasksSummary, newClusterState, previousClusterState);
      // new cluster state, notify all listeners
      final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta();
      if (nodesDelta.hasChanges() && logger.isInfoEnabled()) {
        String summary = nodesDelta.shortSummary();
        if (summary.length() > 0) {
          logger.info("{}, reason: {}", summary, tasksSummary);
        }
      }

      nodeConnectionsService.connectToAddedNodes(clusterChangedEvent);

      // if we are the master, publish the new state to all nodes
      // we publish here before we send a notification to all the listeners, since if it fails
      // we don't want to notify
      if (newClusterState.nodes().isLocalNodeElectedMaster()) {
        logger.debug("publishing cluster state version [{}]", newClusterState.version());
        try {
          clusterStatePublisher.accept(clusterChangedEvent, ackListener);
        } catch (Discovery.FailedToCommitClusterStateException t) {
          final long version = newClusterState.version();
          logger.warn(
              (Supplier<?>)
                  () ->
                      new ParameterizedMessage(
                          "failing [{}]: failed to commit cluster state version [{}]",
                          tasksSummary,
                          version),
              t);
          proccessedListeners.forEach(task -> task.listener.onFailure(task.source, t));
          return;
        }
      }

      // update the current cluster state
      clusterState = newClusterState;
      logger.debug("set local cluster state to version {}", newClusterState.version());
      try {
        // nothing to do until we actually recover from the gateway or any other block indicates we
        // need to disable persistency
        if (clusterChangedEvent.state().blocks().disableStatePersistence() == false
            && clusterChangedEvent.metaDataChanged()) {
          final Settings incomingSettings = clusterChangedEvent.state().metaData().settings();
          clusterSettings.applySettings(incomingSettings);
        }
      } catch (Exception ex) {
        logger.warn("failed to apply cluster settings", ex);
      }
      for (ClusterStateListener listener : preAppliedListeners) {
        try {
          listener.clusterChanged(clusterChangedEvent);
        } catch (Exception ex) {
          logger.warn("failed to notify ClusterStateListener", ex);
        }
      }

      nodeConnectionsService.disconnectFromRemovedNodes(clusterChangedEvent);

      newClusterState.status(ClusterState.ClusterStateStatus.APPLIED);

      for (ClusterStateListener listener : postAppliedListeners) {
        try {
          listener.clusterChanged(clusterChangedEvent);
        } catch (Exception ex) {
          logger.warn("failed to notify ClusterStateListener", ex);
        }
      }

      // manual ack only from the master at the end of the publish
      if (newClusterState.nodes().isLocalNodeElectedMaster()) {
        try {
          ackListener.onNodeAck(newClusterState.nodes().getLocalNode(), null);
        } catch (Exception e) {
          final DiscoveryNode localNode = newClusterState.nodes().getLocalNode();
          logger.debug(
              (Supplier<?>)
                  () ->
                      new ParameterizedMessage(
                          "error while processing ack for master node [{}]", localNode),
              e);
        }
      }

      for (UpdateTask<T> task : proccessedListeners) {
        task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState);
      }

      try {
        executor.clusterStatePublished(clusterChangedEvent);
      } catch (Exception e) {
        logger.error(
            (Supplier<?>)
                () ->
                    new ParameterizedMessage(
                        "exception thrown while notifying executor of new cluster state publication [{}]",
                        tasksSummary),
            e);
      }

      TimeValue executionTime =
          TimeValue.timeValueMillis(
              Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
      logger.debug(
          "processing [{}]: took [{}] done applying updated cluster_state (version: {}, uuid: {})",
          tasksSummary,
          executionTime,
          newClusterState.version(),
          newClusterState.stateUUID());
      warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
    } catch (Exception e) {
      TimeValue executionTime =
          TimeValue.timeValueMillis(
              Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
      final long version = newClusterState.version();
      final String stateUUID = newClusterState.stateUUID();
      final String prettyPrint = newClusterState.prettyPrint();
      logger.warn(
          (Supplier<?>)
              () ->
                  new ParameterizedMessage(
                      "failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]\n{}",
                      executionTime,
                      version,
                      stateUUID,
                      tasksSummary,
                      prettyPrint),
          e);
      // TODO: do we want to call updateTask.onFailure here?
    }
  }