@Override
 protected void doStop() throws ElasticsearchException {
   final boolean setStopped = started.compareAndSet(true, false);
   assert setStopped : "service has already been stopped";
   try {
     transport.stop();
   } finally {
     // in case the transport is not connected to our local node (thus cleaned on node disconnect)
     // make sure to clean any leftover on going handles
     for (Map.Entry<Long, RequestHolder> entry : clientHandlers.entrySet()) {
       final RequestHolder holderToNotify = clientHandlers.remove(entry.getKey());
       if (holderToNotify != null) {
         // callback that an exception happened, but on a different thread since we don't
         // want handlers to worry about stack overflows
         threadPool
             .generic()
             .execute(
                 new Runnable() {
                   @Override
                   public void run() {
                     holderToNotify
                         .handler()
                         .handleException(
                             new TransportException(
                                 "transport stopped, action: " + holderToNotify.action()));
                   }
                 });
       }
     }
   }
 }
 public void startRecovery(
     final IndexShard indexShard,
     final DiscoveryNode sourceNode,
     final RecoveryListener listener) {
   // create a new recovery status, and process...
   final long recoveryId =
       onGoingRecoveries.startRecovery(
           indexShard, sourceNode, listener, recoverySettings.activityTimeout());
   threadPool.generic().execute(new RecoveryRunner(recoveryId));
 }
 private void notifyDisconnectedFromMaster() {
   threadPool
       .generic()
       .execute(
           new Runnable() {
             @Override
             public void run() {
               for (Listener listener : listeners) {
                 listener.onDisconnectedFromMaster();
               }
             }
           });
 }
 private void notifyMasterFailure(final DiscoveryNode masterNode, final String reason) {
   if (notifiedMasterFailure.compareAndSet(false, true)) {
     threadPool
         .generic()
         .execute(
             new Runnable() {
               @Override
               public void run() {
                 for (Listener listener : listeners) {
                   listener.onMasterFailure(masterNode, reason);
                 }
               }
             });
     stop("master failure, " + reason);
   }
 }
    @Override
    public void performRequest(
        String method,
        String uri,
        Map<String, String> params,
        HttpEntity entity,
        ResponseListener listener) {
      /*
       * We use the generic thread pool here because this client is blocking the generic thread pool is sized appropriately for some
       * of the threads on it to be blocked, waiting on IO. It'd be a disaster if this ran on the listener thread pool, eating
       * valuable threads needed to handle responses. Most other thread pool would probably not mind running this either, but the
       * generic thread pool is the "most right" place for it to run. We could make our own thread pool for this but the generic
       * thread pool already has plenty of capacity.
       */
      threadPool
          .generic()
          .execute(
              new AbstractRunnable() {
                @Override
                protected void doRun() throws Exception {
                  try (org.elasticsearch.client.Response response =
                      restClient.performRequest(method, uri, params, entity)) {
                    InputStream markSupportedInputStream =
                        new BufferedInputStream(response.getEntity().getContent());
                    listener.onResponse(markSupportedInputStream);
                  }
                }

                @Override
                public void onFailure(Exception t) {
                  if (t instanceof ResponseException) {
                    ResponseException re = (ResponseException) t;
                    if (RestStatus.TOO_MANY_REQUESTS.getStatus()
                        == re.getResponse().getStatusLine().getStatusCode()) {
                      listener.onRetryableFailure(t);
                      return;
                    }
                  }
                  listener.onFailure(t);
                }
              });
    }
  private void performStateRecovery(boolean enforceRecoverAfterTime, String reason) {
    final Gateway.GatewayStateRecoveredListener recoveryListener = new GatewayRecoveryListener();

    if (enforceRecoverAfterTime && recoverAfterTime != null) {
      if (scheduledRecovery.compareAndSet(false, true)) {
        logger.info("delaying initial state recovery for [{}]. {}", recoverAfterTime, reason);
        threadPool.schedule(
            recoverAfterTime,
            ThreadPool.Names.GENERIC,
            () -> {
              if (recovered.compareAndSet(false, true)) {
                logger.info(
                    "recover_after_time [{}] elapsed. performing state recovery...",
                    recoverAfterTime);
                gateway.performStateRecovery(recoveryListener);
              }
            });
      }
    } else {
      if (recovered.compareAndSet(false, true)) {
        threadPool
            .generic()
            .execute(
                new AbstractRunnable() {
                  @Override
                  public void onFailure(Exception e) {
                    logger.warn("Recovery failed", e);
                    // we reset `recovered` in the listener don't reset it here otherwise there
                    // might be a race
                    // that resets it to false while a new recover is already running?
                    recoveryListener.onFailure("state recovery failed: " + e.getMessage());
                  }

                  @Override
                  protected void doRun() throws Exception {
                    gateway.performStateRecovery(recoveryListener);
                  }
                });
      }
    }
  }
 private void asyncJoinCluster() {
   if (currentJoinThread != null) {
     // we are already joining, ignore...
     logger.trace("a join thread already running");
     return;
   }
   threadPool
       .generic()
       .execute(
           new Runnable() {
             @Override
             public void run() {
               currentJoinThread = Thread.currentThread();
               try {
                 innerJoinCluster();
               } finally {
                 currentJoinThread = null;
               }
             }
           });
 }
 public void nodeIndexDeleted(final String index, final String nodeId)
     throws ElasticSearchException {
   DiscoveryNodes nodes = clusterService.state().nodes();
   if (nodes.localNodeMaster()) {
     threadPool
         .generic()
         .execute(
             new Runnable() {
               @Override
               public void run() {
                 innerNodeIndexDeleted(index, nodeId);
               }
             });
   } else {
     transportService.sendRequest(
         clusterService.state().nodes().masterNode(),
         NodeIndexDeletedTransportHandler.ACTION,
         new NodeIndexDeletedMessage(index, nodeId),
         EmptyTransportResponseHandler.INSTANCE_SAME);
   }
 }
  /**
   * Submits a batch of cluster state update tasks; submitted updates are guaranteed to be processed
   * together, potentially with more tasks of the same executor.
   *
   * @param source the source of the cluster state update task
   * @param tasks a map of update tasks and their corresponding listeners
   * @param config the cluster state update task configuration
   * @param executor the cluster state update task executor; tasks that share the same executor will
   *     be executed batches on this executor
   * @param <T> the type of the cluster state update task state
   */
  public <T> void submitStateUpdateTasks(
      final String source,
      final Map<T, ClusterStateTaskListener> tasks,
      final ClusterStateTaskConfig config,
      final ClusterStateTaskExecutor<T> executor) {
    if (!lifecycle.started()) {
      return;
    }
    if (tasks.isEmpty()) {
      return;
    }
    try {
      // convert to an identity map to check for dups based on update tasks semantics of using
      // identity instead of equal
      final IdentityHashMap<T, ClusterStateTaskListener> tasksIdentity =
          new IdentityHashMap<>(tasks);
      final List<UpdateTask<T>> updateTasks =
          tasksIdentity
              .entrySet()
              .stream()
              .map(
                  entry ->
                      new UpdateTask<>(
                          source, entry.getKey(), config, executor, safe(entry.getValue(), logger)))
              .collect(Collectors.toList());

      synchronized (updateTasksPerExecutor) {
        List<UpdateTask> existingTasks =
            updateTasksPerExecutor.computeIfAbsent(executor, k -> new ArrayList<>());
        for (@SuppressWarnings("unchecked") UpdateTask<T> existing : existingTasks) {
          if (tasksIdentity.containsKey(existing.task)) {
            throw new IllegalStateException(
                "task ["
                    + executor.describeTasks(Collections.singletonList(existing.task))
                    + "] with source ["
                    + source
                    + "] is already queued");
          }
        }
        existingTasks.addAll(updateTasks);
      }

      final UpdateTask<T> firstTask = updateTasks.get(0);

      if (config.timeout() != null) {
        updateTasksExecutor.execute(
            firstTask,
            threadPool.scheduler(),
            config.timeout(),
            () ->
                threadPool
                    .generic()
                    .execute(
                        () -> {
                          for (UpdateTask<T> task : updateTasks) {
                            if (task.processed.getAndSet(true) == false) {
                              logger.debug(
                                  "cluster state update task [{}] timed out after [{}]",
                                  source,
                                  config.timeout());
                              task.listener.onFailure(
                                  source,
                                  new ProcessClusterEventTimeoutException(
                                      config.timeout(), source));
                            }
                          }
                        }));
      } else {
        updateTasksExecutor.execute(firstTask);
      }
    } catch (EsRejectedExecutionException e) {
      // ignore cases where we are shutting down..., there is really nothing interesting
      // to be done here...
      if (!lifecycle.stoppedOrClosed()) {
        throw e;
      }
    }
  }
  /** Recovers the state of the shard from the gateway. */
  public void recover(final boolean indexShouldExists, final RecoveryListener listener)
      throws IndexShardGatewayRecoveryException, IgnoreGatewayRecoveryException {
    if (indexShard.state() == IndexShardState.CLOSED) {
      // got closed on us, just ignore this recovery
      listener.onIgnoreRecovery("shard closed");
      return;
    }
    if (!indexShard.routingEntry().primary()) {
      listener.onRecoveryFailed(
          new IndexShardGatewayRecoveryException(
              shardId, "Trying to recover when the shard is in backup state", null));
      return;
    }
    try {
      if (indexShard.routingEntry().restoreSource() != null) {
        indexShard.recovering("from snapshot");
      } else {
        indexShard.recovering("from gateway");
      }
    } catch (IllegalIndexShardStateException e) {
      // that's fine, since we might be called concurrently, just ignore this, we are already
      // recovering
      listener.onIgnoreRecovery("already in recovering process, " + e.getMessage());
      return;
    }

    threadPool
        .generic()
        .execute(
            new Runnable() {
              @Override
              public void run() {
                recoveryStatus = new RecoveryStatus();
                recoveryStatus.updateStage(RecoveryStatus.Stage.INIT);

                try {
                  if (indexShard.routingEntry().restoreSource() != null) {
                    logger.debug(
                        "restoring from {} ...", indexShard.routingEntry().restoreSource());
                    snapshotService.restore(recoveryStatus);
                  } else {
                    logger.debug("starting recovery from {} ...", shardGateway);
                    shardGateway.recover(indexShouldExists, recoveryStatus);
                  }

                  lastIndexVersion = recoveryStatus.index().version();
                  lastTranslogId = -1;
                  lastTranslogLength = 0;
                  lastTotalTranslogOperations =
                      recoveryStatus.translog().currentTranslogOperations();

                  // start the shard if the gateway has not started it already. Note that if the
                  // gateway
                  // moved shard to POST_RECOVERY, it may have been started as well if:
                  // 1) master sent a new cluster state indicating shard is initializing
                  // 2) IndicesClusterStateService#applyInitializingShard will send a shard started
                  // event
                  // 3) Master will mark shard as started and this will be processed locally.
                  IndexShardState shardState = indexShard.state();
                  if (shardState != IndexShardState.POST_RECOVERY
                      && shardState != IndexShardState.STARTED) {
                    indexShard.postRecovery("post recovery from gateway");
                  }
                  // refresh the shard
                  indexShard.refresh(new Engine.Refresh("post_gateway").force(true));

                  recoveryStatus.time(System.currentTimeMillis() - recoveryStatus.startTime());
                  recoveryStatus.updateStage(RecoveryStatus.Stage.DONE);

                  if (logger.isDebugEnabled()) {
                    logger.debug(
                        "recovery completed from [{}], took [{}]",
                        shardGateway,
                        timeValueMillis(recoveryStatus.time()));
                  } else if (logger.isTraceEnabled()) {
                    StringBuilder sb = new StringBuilder();
                    sb.append("recovery completed from ")
                        .append(shardGateway)
                        .append(", took [")
                        .append(timeValueMillis(recoveryStatus.time()))
                        .append("]\n");
                    sb.append("    index    : files           [")
                        .append(recoveryStatus.index().numberOfFiles())
                        .append("] with total_size [")
                        .append(new ByteSizeValue(recoveryStatus.index().totalSize()))
                        .append("], took[")
                        .append(TimeValue.timeValueMillis(recoveryStatus.index().time()))
                        .append("]\n");
                    sb.append("             : recovered_files [")
                        .append(recoveryStatus.index().numberOfRecoveredFiles())
                        .append("] with total_size [")
                        .append(new ByteSizeValue(recoveryStatus.index().recoveredTotalSize()))
                        .append("]\n");
                    sb.append("             : reusing_files   [")
                        .append(recoveryStatus.index().numberOfReusedFiles())
                        .append("] with total_size [")
                        .append(new ByteSizeValue(recoveryStatus.index().reusedTotalSize()))
                        .append("]\n");
                    sb.append("    start    : took [")
                        .append(TimeValue.timeValueMillis(recoveryStatus.start().time()))
                        .append("], check_index [")
                        .append(timeValueMillis(recoveryStatus.start().checkIndexTime()))
                        .append("]\n");
                    sb.append("    translog : number_of_operations [")
                        .append(recoveryStatus.translog().currentTranslogOperations())
                        .append("], took [")
                        .append(TimeValue.timeValueMillis(recoveryStatus.translog().time()))
                        .append("]");
                    logger.trace(sb.toString());
                  }
                  listener.onRecoveryDone();
                  scheduleSnapshotIfNeeded();
                } catch (IndexShardGatewayRecoveryException e) {
                  if (indexShard.state() == IndexShardState.CLOSED) {
                    // got closed on us, just ignore this recovery
                    listener.onIgnoreRecovery("shard closed");
                    return;
                  }
                  if ((e.getCause() instanceof IndexShardClosedException)
                      || (e.getCause() instanceof IndexShardNotStartedException)) {
                    // got closed on us, just ignore this recovery
                    listener.onIgnoreRecovery("shard closed");
                    return;
                  }
                  listener.onRecoveryFailed(e);
                } catch (IndexShardClosedException e) {
                  listener.onIgnoreRecovery("shard closed");
                } catch (IndexShardNotStartedException e) {
                  listener.onIgnoreRecovery("shard closed");
                } catch (Exception e) {
                  if (indexShard.state() == IndexShardState.CLOSED) {
                    // got closed on us, just ignore this recovery
                    listener.onIgnoreRecovery("shard closed");
                    return;
                  }
                  listener.onRecoveryFailed(
                      new IndexShardGatewayRecoveryException(shardId, "failed recovery", e));
                }
              }
            });
  }