void retry(@Nullable final Throwable failure) {
      if (observer.isTimedOut()) {
        // we running as a last attempt after a timeout has happened. don't retry
        Throwable listenFailure = failure;
        if (listenFailure == null) {
          if (shardIt == null) {
            listenFailure =
                new UnavailableShardsException(
                    request.concreteIndex(),
                    -1,
                    "Timeout waiting for [{}], request: {}",
                    request.timeout(),
                    actionName);
          } else {
            listenFailure =
                new UnavailableShardsException(
                    shardIt.shardId(),
                    "[{}] shardIt, [{}] active : Timeout waiting for [{}], request: {}",
                    shardIt.size(),
                    shardIt.sizeActive(),
                    request.timeout(),
                    actionName);
          }
        }
        listener.onFailure(listenFailure);
        return;
      }

      observer.waitForNextChange(
          new ClusterStateObserver.Listener() {
            @Override
            public void onNewClusterState(ClusterState state) {
              doStart();
            }

            @Override
            public void onClusterServiceClose() {
              listener.onFailure(new NodeClosedException(nodes.getLocalNode()));
            }

            @Override
            public void onTimeout(TimeValue timeout) {
              // just to be on the safe side, see if we can start it now?
              doStart();
            }
          },
          request.timeout());
    }
  public void execute() throws Exception {
    final String activeShardCountFailure = checkActiveShardCount();
    final ShardRouting primaryRouting = primary.routingEntry();
    final ShardId primaryId = primaryRouting.shardId();
    if (activeShardCountFailure != null) {
      finishAsFailed(
          new UnavailableShardsException(
              primaryId,
              "{} Timeout: [{}], request: [{}]",
              activeShardCountFailure,
              request.timeout(),
              request));
      return;
    }

    totalShards.incrementAndGet();
    pendingShards.incrementAndGet();
    primaryResult = primary.perform(request);
    final ReplicaRequest replicaRequest = primaryResult.replicaRequest();
    assert replicaRequest.primaryTerm() > 0 : "replicaRequest doesn't have a primary term";
    if (logger.isTraceEnabled()) {
      logger.trace(
          "[{}] op [{}] completed on primary for request [{}]", primaryId, opType, request);
    }

    performOnReplicas(primaryId, replicaRequest);

    successfulShards.incrementAndGet();
    decPendingAndFinishIfNeeded();
  }
 public void start() {
   this.observer = new ClusterStateObserver(clusterService, request.timeout(), logger);
   doStart();
 }
 public void start() {
   this.observer =
       new ClusterStateObserver(
           clusterService, request.timeout(), logger, threadPool.getThreadContext());
   doStart();
 }
    void retry(boolean fromClusterEvent, @Nullable final Throwable failure) {
      if (!fromClusterEvent) {
        // make it threaded operation so we fork on the discovery listener thread
        request.beforeLocalFork();
        request.operationThreaded(true);
        clusterService.add(
            request.timeout(),
            new TimeoutClusterStateListener() {
              @Override
              public void postAdded() {
                if (start(true)) {
                  // if we managed to start and perform the operation on the primary, we can remove
                  // this listener
                  clusterService.remove(this);
                }
              }

              @Override
              public void onClose() {
                clusterService.remove(this);
                listener.onFailure(new NodeClosedException(nodes.localNode()));
              }

              @Override
              public void clusterChanged(ClusterChangedEvent event) {
                if (start(true)) {
                  // if we managed to start and perform the operation on the primary, we can remove
                  // this listener
                  clusterService.remove(this);
                }
              }

              @Override
              public void onTimeout(TimeValue timeValue) {
                // just to be on the safe side, see if we can start it now?
                if (start(true)) {
                  clusterService.remove(this);
                  return;
                }
                clusterService.remove(this);
                Throwable listenerFailure = failure;
                if (listenerFailure == null) {
                  if (shardIt == null) {
                    listenerFailure =
                        new UnavailableShardsException(
                            null,
                            "no available shards: Timeout waiting for ["
                                + timeValue
                                + "], request: "
                                + request.toString());
                  } else {
                    listenerFailure =
                        new UnavailableShardsException(
                            shardIt.shardId(),
                            "["
                                + shardIt.size()
                                + "] shardIt, ["
                                + shardIt.sizeActive()
                                + "] active : Timeout waiting for ["
                                + timeValue
                                + "], request: "
                                + request.toString());
                  }
                }
                listener.onFailure(listenerFailure);
              }
            });
      }
    }