void setFailure(ShardIterator shardIt, int shardIndex, Throwable t) {
      // we don't aggregate shard failures on non active shards (but do keep the header counts
      // right)
      if (TransportActions.isShardNotAvailableException(t)) {
        return;
      }

      if (!(t instanceof BroadcastShardOperationFailedException)) {
        t = new BroadcastShardOperationFailedException(shardIt.shardId(), t);
      }

      Object response = shardsResponses.get(shardIndex);
      if (response == null) {
        // just override it and return
        shardsResponses.set(shardIndex, t);
      }

      if (!(response instanceof Throwable)) {
        // we should never really get here...
        return;
      }

      // the failure is already present, try and not override it with an exception that is less
      // meaningless
      // for example, getting illegal shard state
      if (TransportActions.isReadOverrideException(t)) {
        shardsResponses.set(shardIndex, t);
      }
    }
  protected final void addShardFailure(
      final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
    // we don't aggregate shard failures on non active shards (but do keep the header counts right)
    if (TransportActions.isShardNotAvailableException(e)) {
      return;
    }

    // lazily create shard failures, so we can early build the empty shard failure list in most
    // cases (no failures)
    if (shardFailures == null) {
      synchronized (shardFailuresMutex) {
        if (shardFailures == null) {
          shardFailures = new AtomicArray<>(shardsIts.size());
        }
      }
    }
    ShardSearchFailure failure = shardFailures.get(shardIndex);
    if (failure == null) {
      shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
    } else {
      // the failure is already present, try and not override it with an exception that is less
      // meaningless
      // for example, getting illegal shard state
      if (TransportActions.isReadOverrideException(e)) {
        shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
      }
    }
  }
    @SuppressWarnings({"unchecked"})
    void onOperation(
        @Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) {
      // we set the shard failure always, even if its the first in the replication group, and the
      // next one
      // will work (it will just override it...)
      setFailure(shardIt, shardIndex, t);

      ShardRouting nextShard = shardIt.nextOrNull();
      if (nextShard != null) {
        if (t != null) {
          if (logger.isTraceEnabled()) {
            if (!TransportActions.isShardNotAvailableException(t)) {
              if (shard != null) {
                logger.trace(shard.shortSummary() + ": Failed to execute [" + request + "]", t);
              } else {
                logger.trace(shardIt.shardId() + ": Failed to execute [" + request + "]", t);
              }
            }
          }
        }
        // we are not threaded here if we got here from the transport
        // or we possibly threaded if we got from a local threaded one,
        // in which case, the next shard in the partition will not be local one
        // so there is no meaning to this flag
        performOperation(shardIt, nextShard, shardIndex, true);
      } else {
        if (logger.isDebugEnabled()) {
          if (t != null) {
            if (!TransportActions.isShardNotAvailableException(t)) {
              if (shard != null) {
                logger.debug(shard.shortSummary() + ": Failed to execute [" + request + "]", t);
              } else {
                logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "]", t);
              }
            }
          }
        }
        if (expectedOps == counterOps.incrementAndGet()) {
          finishHim();
        }
      }
    }
 /** Should an exception be ignored when the operation is performed on the replica. */
 public static boolean ignoreReplicaException(Exception e) {
   if (TransportActions.isShardNotAvailableException(e)) {
     return true;
   }
   // on version conflict or document missing, it means
   // that a new change has crept into the replica, and it's fine
   if (isConflictException(e)) {
     return true;
   }
   return false;
 }
  @Override
  protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, int shardId)
      throws ElasticSearchException {
    IndexService indexService = indicesService.indexServiceSafe(request.index());
    IndexShard indexShard = indexService.shardSafe(shardId);

    if (request.refresh() && !request.realtime()) {
      indexShard.refresh(
          new Engine.Refresh("refresh_flag_mget").force(TransportGetAction.REFRESH_FORCE));
    }

    MultiGetShardResponse response = new MultiGetShardResponse();
    for (int i = 0; i < request.locations.size(); i++) {
      String type = request.types.get(i);
      String id = request.ids.get(i);
      String[] fields = request.fields.get(i);

      long version = request.versions.get(i);
      VersionType versionType = request.versionTypes.get(i);
      if (versionType == null) {
        versionType = VersionType.INTERNAL;
      }

      FetchSourceContext fetchSourceContext = request.fetchSourceContexts.get(i);
      try {
        GetResult getResult =
            indexShard
                .getService()
                .get(
                    type, id, fields, request.realtime(), version, versionType, fetchSourceContext);
        response.add(request.locations.get(i), new GetResponse(getResult));
      } catch (Throwable t) {
        if (TransportActions.isShardNotAvailableException(t)) {
          throw (ElasticSearchException) t;
        } else {
          logger.debug(
              "[{}][{}] failed to execute multi_get for [{}]/[{}]",
              t,
              request.index(),
              shardId,
              type,
              id);
          response.add(
              request.locations.get(i),
              new MultiGetResponse.Failure(
                  request.index(), type, id, ExceptionsHelper.detailedMessage(t)));
        }
      }
    }

    return response;
  }
 @SuppressWarnings({"unchecked"})
 void onOperation(
     @Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) {
   // we set the shard failure always, even if its the first in the replication group, and the
   // next one
   // will work (it will just override it...)
   setFailure(shardIt, shardIndex, t);
   ShardRouting nextShard = shardIt.nextOrNull();
   if (nextShard != null) {
     if (t != null) {
       if (logger.isTraceEnabled()) {
         if (!TransportActions.isShardNotAvailableException(t)) {
           logger.trace(
               "{}: failed to execute [{}]",
               t,
               shard != null ? shard.shortSummary() : shardIt.shardId(),
               request);
         }
       }
     }
     performOperation(shardIt, nextShard, shardIndex);
   } else {
     if (logger.isDebugEnabled()) {
       if (t != null) {
         if (!TransportActions.isShardNotAvailableException(t)) {
           logger.debug(
               "{}: failed to execute [{}]",
               t,
               shard != null ? shard.shortSummary() : shardIt.shardId(),
               request);
         }
       }
     }
     if (expectedOps == counterOps.incrementAndGet()) {
       finishHim();
     }
   }
 }
 private Response newResponse(
     Request request,
     AtomicReferenceArray responses,
     List<NoShardAvailableActionException> unavailableShardExceptions,
     Map<String, List<ShardRouting>> nodes,
     ClusterState clusterState) {
   int totalShards = 0;
   int successfulShards = 0;
   List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>();
   List<ShardOperationFailedException> exceptions = new ArrayList<>();
   for (int i = 0; i < responses.length(); i++) {
     if (responses.get(i) instanceof FailedNodeException) {
       FailedNodeException exception = (FailedNodeException) responses.get(i);
       totalShards += nodes.get(exception.nodeId()).size();
       for (ShardRouting shard : nodes.get(exception.nodeId())) {
         exceptions.add(
             new DefaultShardOperationFailedException(shard.getIndex(), shard.getId(), exception));
       }
     } else {
       NodeResponse response = (NodeResponse) responses.get(i);
       broadcastByNodeResponses.addAll(response.results);
       totalShards += response.getTotalShards();
       successfulShards += response.getSuccessfulShards();
       for (BroadcastShardOperationFailedException throwable : response.getExceptions()) {
         if (!TransportActions.isShardNotAvailableException(throwable)) {
           exceptions.add(
               new DefaultShardOperationFailedException(
                   throwable.getIndex(), throwable.getShardId().getId(), throwable));
         }
       }
     }
   }
   totalShards += unavailableShardExceptions.size();
   int failedShards = exceptions.size();
   return newResponse(
       request,
       totalShards,
       successfulShards,
       failedShards,
       broadcastByNodeResponses,
       exceptions,
       clusterState);
 }
  @Override
  protected ShardResponse processRequestItems(
      ShardId shardId, ShardUpsertRequest request, AtomicBoolean killed) {
    ShardResponse shardResponse = new ShardResponse();
    DocTableInfo tableInfo = schemas.getWritableTable(TableIdent.fromIndexName(request.index()));
    for (int i = 0; i < request.itemIndices().size(); i++) {
      int location = request.itemIndices().get(i);
      ShardUpsertRequest.Item item = request.items().get(i);
      if (killed.get()) {
        throw new CancellationException();
      }
      try {
        indexItem(
            tableInfo,
            request,
            item,
            shardId,
            item.insertValues() != null, // try insert first
            0);
        shardResponse.add(location);
      } catch (Throwable t) {
        if (!TransportActions.isShardNotAvailableException(t) && !request.continueOnError()) {
          throw t;
        } else {
          logger.debug(
              "{} failed to execute upsert for [{}]/[{}]",
              t,
              request.shardId(),
              request.type(),
              item.id());
          shardResponse.add(
              location,
              new ShardResponse.Failure(
                  item.id(),
                  ExceptionsHelper.detailedMessage(t),
                  (t instanceof VersionConflictEngineException)));
        }
      }
    }

    return shardResponse;
  }
  @Override
  protected MultiTermVectorsShardResponse shardOperation(
      MultiTermVectorsShardRequest request, int shardId) throws ElasticsearchException {

    MultiTermVectorsShardResponse response = new MultiTermVectorsShardResponse();
    for (int i = 0; i < request.locations.size(); i++) {
      TermVectorRequest termVectorRequest = request.requests.get(i);

      try {
        IndexService indexService = indicesService.indexServiceSafe(request.index());
        IndexShard indexShard = indexService.shardSafe(shardId);
        TermVectorResponse termVectorResponse =
            indexShard.termVectorService().getTermVector(termVectorRequest);
        response.add(request.locations.get(i), termVectorResponse);
      } catch (Throwable t) {
        if (TransportActions.isShardNotAvailableException(t)) {
          throw (ElasticsearchException) t;
        } else {
          logger.debug(
              "[{}][{}] failed to execute multi term vectors for [{}]/[{}]",
              t,
              request.index(),
              shardId,
              termVectorRequest.type(),
              termVectorRequest.id());
          response.add(
              request.locations.get(i),
              new MultiTermVectorsResponse.Failure(
                  request.index(),
                  termVectorRequest.type(),
                  termVectorRequest.id(),
                  ExceptionsHelper.detailedMessage(t)));
        }
      }
    }

    return response;
  }
 @Override
 protected Response shardOperation(Request request, ShardId shardId) {
   // TODO: Look into combining the shard req's docs into one in memory index.
   Response response = new Response();
   response.items = new ArrayList<>(request.items.size());
   for (Request.Item item : request.items) {
     Response.Item responseItem;
     int slot = item.slot;
     try {
       responseItem = new Response.Item(slot, percolatorService.percolate(item.request));
     } catch (Throwable t) {
       if (TransportActions.isShardNotAvailableException(t)) {
         throw (ElasticsearchException) t;
       } else {
         logger.debug("{} failed to multi percolate", t, request.shardId());
         responseItem =
             new Response.Item(slot, new StringText(ExceptionsHelper.detailedMessage(t)));
       }
     }
     response.items.add(responseItem);
   }
   return response;
 }
    private void perform(@Nullable final Throwable currentFailure) {
      Throwable lastFailure = this.lastFailure;
      if (lastFailure == null || TransportActions.isReadOverrideException(currentFailure)) {
        lastFailure = currentFailure;
        this.lastFailure = currentFailure;
      }
      final ShardRouting shardRouting = shardIt.nextOrNull();
      if (shardRouting == null) {
        Throwable failure = lastFailure;
        if (failure == null || isShardNotAvailableException(failure)) {
          failure =
              new NoShardAvailableActionException(
                  null,
                  LoggerMessageFormat.format(
                      "No shard available for [{}]", internalRequest.request()),
                  failure);
        } else {
          if (logger.isDebugEnabled()) {
            logger.debug("{}: failed to execute [{}]", failure, null, internalRequest.request());
          }
        }
        listener.onFailure(failure);
        return;
      }
      DiscoveryNode node = nodes.get(shardRouting.currentNodeId());
      if (node == null) {
        onFailure(shardRouting, new NoShardAvailableActionException(shardRouting.shardId()));
      } else {
        internalRequest.request().internalShardId = shardRouting.shardId();
        if (logger.isTraceEnabled()) {
          logger.trace(
              "sending request [{}] to shard [{}] on node [{}]",
              internalRequest.request(),
              internalRequest.request().internalShardId,
              node);
        }
        transportService.sendRequest(
            node,
            transportShardAction,
            internalRequest.request(),
            new BaseTransportResponseHandler<Response>() {

              @Override
              public Response newInstance() {
                return newResponse();
              }

              @Override
              public String executor() {
                return ThreadPool.Names.SAME;
              }

              @Override
              public void handleResponse(final Response response) {
                listener.onResponse(response);
              }

              @Override
              public void handleException(TransportException exp) {
                onFailure(shardRouting, exp);
              }
            });
      }
    }
  private void onFirstPhaseResult(
      final int shardIndex,
      @Nullable ShardRouting shard,
      @Nullable String nodeId,
      final ShardIterator shardIt,
      Exception e) {
    // we always add the shard failure for a specific shard instance
    // we do make sure to clean it on a successful response from a shard
    SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId());
    addShardFailure(shardIndex, shardTarget, e);

    if (totalOps.incrementAndGet() == expectedTotalOps) {
      if (logger.isDebugEnabled()) {
        if (e != null && !TransportActions.isShardNotAvailableException(e)) {
          logger.debug(
              (Supplier<?>)
                  () ->
                      new ParameterizedMessage(
                          "{}: Failed to execute [{}]",
                          shard != null ? shard.shortSummary() : shardIt.shardId(),
                          request),
              e);
        } else if (logger.isTraceEnabled()) {
          logger.trace(
              (Supplier<?>)
                  () -> new ParameterizedMessage("{}: Failed to execute [{}]", shard, request),
              e);
        }
      }
      final ShardSearchFailure[] shardSearchFailures = buildShardFailures();
      if (successfulOps.get() == 0) {
        if (logger.isDebugEnabled()) {
          logger.debug(
              (Supplier<?>)
                  () ->
                      new ParameterizedMessage(
                          "All shards failed for phase: [{}]", firstPhaseName()),
              e);
        }

        // no successful ops, raise an exception
        raiseEarlyFailure(
            new SearchPhaseExecutionException(
                firstPhaseName(), "all shards failed", e, shardSearchFailures));
      } else {
        try {
          innerMoveToSecondPhase();
        } catch (Exception inner) {
          inner.addSuppressed(e);
          raiseEarlyFailure(
              new ReduceSearchPhaseException(firstPhaseName(), "", inner, shardSearchFailures));
        }
      }
    } else {
      final ShardRouting nextShard = shardIt.nextOrNull();
      final boolean lastShard = nextShard == null;
      // trace log this exception
      logger.trace(
          (Supplier<?>)
              () ->
                  new ParameterizedMessage(
                      "{}: Failed to execute [{}] lastShard [{}]",
                      shard != null ? shard.shortSummary() : shardIt.shardId(),
                      request,
                      lastShard),
          e);
      if (!lastShard) {
        try {
          performFirstPhase(shardIndex, shardIt, nextShard);
        } catch (Exception inner) {
          inner.addSuppressed(e);
          onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, inner);
        }
      } else {
        // no more shards active, add a failure
        if (logger.isDebugEnabled()
            && !logger.isTraceEnabled()) { // do not double log this exception
          if (e != null && !TransportActions.isShardNotAvailableException(e)) {
            logger.debug(
                (Supplier<?>)
                    () ->
                        new ParameterizedMessage(
                            "{}: Failed to execute [{}] lastShard [{}]",
                            shard != null ? shard.shortSummary() : shardIt.shardId(),
                            request,
                            lastShard),
                e);
          }
        }
      }
    }
  }
 @Override
 protected boolean retryOnFailure(Throwable e) {
   return TransportActions.isShardNotAvailableException(e);
 }
 protected boolean retryPrimaryException(Throwable e) {
   return e.getClass() == RetryOnPrimaryException.class
       || TransportActions.isShardNotAvailableException(e);
 }