void setFailure(ShardIterator shardIt, int shardIndex, Throwable t) { // we don't aggregate shard failures on non active shards (but do keep the header counts // right) if (TransportActions.isShardNotAvailableException(t)) { return; } if (!(t instanceof BroadcastShardOperationFailedException)) { t = new BroadcastShardOperationFailedException(shardIt.shardId(), t); } Object response = shardsResponses.get(shardIndex); if (response == null) { // just override it and return shardsResponses.set(shardIndex, t); } if (!(response instanceof Throwable)) { // we should never really get here... return; } // the failure is already present, try and not override it with an exception that is less // meaningless // for example, getting illegal shard state if (TransportActions.isReadOverrideException(t)) { shardsResponses.set(shardIndex, t); } }
protected final void addShardFailure( final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) { // we don't aggregate shard failures on non active shards (but do keep the header counts right) if (TransportActions.isShardNotAvailableException(e)) { return; } // lazily create shard failures, so we can early build the empty shard failure list in most // cases (no failures) if (shardFailures == null) { synchronized (shardFailuresMutex) { if (shardFailures == null) { shardFailures = new AtomicArray<>(shardsIts.size()); } } } ShardSearchFailure failure = shardFailures.get(shardIndex); if (failure == null) { shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget)); } else { // the failure is already present, try and not override it with an exception that is less // meaningless // for example, getting illegal shard state if (TransportActions.isReadOverrideException(e)) { shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget)); } } }
@SuppressWarnings({"unchecked"}) void onOperation( @Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) { // we set the shard failure always, even if its the first in the replication group, and the // next one // will work (it will just override it...) setFailure(shardIt, shardIndex, t); ShardRouting nextShard = shardIt.nextOrNull(); if (nextShard != null) { if (t != null) { if (logger.isTraceEnabled()) { if (!TransportActions.isShardNotAvailableException(t)) { if (shard != null) { logger.trace(shard.shortSummary() + ": Failed to execute [" + request + "]", t); } else { logger.trace(shardIt.shardId() + ": Failed to execute [" + request + "]", t); } } } } // we are not threaded here if we got here from the transport // or we possibly threaded if we got from a local threaded one, // in which case, the next shard in the partition will not be local one // so there is no meaning to this flag performOperation(shardIt, nextShard, shardIndex, true); } else { if (logger.isDebugEnabled()) { if (t != null) { if (!TransportActions.isShardNotAvailableException(t)) { if (shard != null) { logger.debug(shard.shortSummary() + ": Failed to execute [" + request + "]", t); } else { logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "]", t); } } } } if (expectedOps == counterOps.incrementAndGet()) { finishHim(); } } }
/** Should an exception be ignored when the operation is performed on the replica. */ public static boolean ignoreReplicaException(Exception e) { if (TransportActions.isShardNotAvailableException(e)) { return true; } // on version conflict or document missing, it means // that a new change has crept into the replica, and it's fine if (isConflictException(e)) { return true; } return false; }
@Override protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, int shardId) throws ElasticSearchException { IndexService indexService = indicesService.indexServiceSafe(request.index()); IndexShard indexShard = indexService.shardSafe(shardId); if (request.refresh() && !request.realtime()) { indexShard.refresh( new Engine.Refresh("refresh_flag_mget").force(TransportGetAction.REFRESH_FORCE)); } MultiGetShardResponse response = new MultiGetShardResponse(); for (int i = 0; i < request.locations.size(); i++) { String type = request.types.get(i); String id = request.ids.get(i); String[] fields = request.fields.get(i); long version = request.versions.get(i); VersionType versionType = request.versionTypes.get(i); if (versionType == null) { versionType = VersionType.INTERNAL; } FetchSourceContext fetchSourceContext = request.fetchSourceContexts.get(i); try { GetResult getResult = indexShard .getService() .get( type, id, fields, request.realtime(), version, versionType, fetchSourceContext); response.add(request.locations.get(i), new GetResponse(getResult)); } catch (Throwable t) { if (TransportActions.isShardNotAvailableException(t)) { throw (ElasticSearchException) t; } else { logger.debug( "[{}][{}] failed to execute multi_get for [{}]/[{}]", t, request.index(), shardId, type, id); response.add( request.locations.get(i), new MultiGetResponse.Failure( request.index(), type, id, ExceptionsHelper.detailedMessage(t))); } } } return response; }
@SuppressWarnings({"unchecked"}) void onOperation( @Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) { // we set the shard failure always, even if its the first in the replication group, and the // next one // will work (it will just override it...) setFailure(shardIt, shardIndex, t); ShardRouting nextShard = shardIt.nextOrNull(); if (nextShard != null) { if (t != null) { if (logger.isTraceEnabled()) { if (!TransportActions.isShardNotAvailableException(t)) { logger.trace( "{}: failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request); } } } performOperation(shardIt, nextShard, shardIndex); } else { if (logger.isDebugEnabled()) { if (t != null) { if (!TransportActions.isShardNotAvailableException(t)) { logger.debug( "{}: failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request); } } } if (expectedOps == counterOps.incrementAndGet()) { finishHim(); } } }
private Response newResponse( Request request, AtomicReferenceArray responses, List<NoShardAvailableActionException> unavailableShardExceptions, Map<String, List<ShardRouting>> nodes, ClusterState clusterState) { int totalShards = 0; int successfulShards = 0; List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>(); List<ShardOperationFailedException> exceptions = new ArrayList<>(); for (int i = 0; i < responses.length(); i++) { if (responses.get(i) instanceof FailedNodeException) { FailedNodeException exception = (FailedNodeException) responses.get(i); totalShards += nodes.get(exception.nodeId()).size(); for (ShardRouting shard : nodes.get(exception.nodeId())) { exceptions.add( new DefaultShardOperationFailedException(shard.getIndex(), shard.getId(), exception)); } } else { NodeResponse response = (NodeResponse) responses.get(i); broadcastByNodeResponses.addAll(response.results); totalShards += response.getTotalShards(); successfulShards += response.getSuccessfulShards(); for (BroadcastShardOperationFailedException throwable : response.getExceptions()) { if (!TransportActions.isShardNotAvailableException(throwable)) { exceptions.add( new DefaultShardOperationFailedException( throwable.getIndex(), throwable.getShardId().getId(), throwable)); } } } } totalShards += unavailableShardExceptions.size(); int failedShards = exceptions.size(); return newResponse( request, totalShards, successfulShards, failedShards, broadcastByNodeResponses, exceptions, clusterState); }
@Override protected ShardResponse processRequestItems( ShardId shardId, ShardUpsertRequest request, AtomicBoolean killed) { ShardResponse shardResponse = new ShardResponse(); DocTableInfo tableInfo = schemas.getWritableTable(TableIdent.fromIndexName(request.index())); for (int i = 0; i < request.itemIndices().size(); i++) { int location = request.itemIndices().get(i); ShardUpsertRequest.Item item = request.items().get(i); if (killed.get()) { throw new CancellationException(); } try { indexItem( tableInfo, request, item, shardId, item.insertValues() != null, // try insert first 0); shardResponse.add(location); } catch (Throwable t) { if (!TransportActions.isShardNotAvailableException(t) && !request.continueOnError()) { throw t; } else { logger.debug( "{} failed to execute upsert for [{}]/[{}]", t, request.shardId(), request.type(), item.id()); shardResponse.add( location, new ShardResponse.Failure( item.id(), ExceptionsHelper.detailedMessage(t), (t instanceof VersionConflictEngineException))); } } } return shardResponse; }
@Override protected MultiTermVectorsShardResponse shardOperation( MultiTermVectorsShardRequest request, int shardId) throws ElasticsearchException { MultiTermVectorsShardResponse response = new MultiTermVectorsShardResponse(); for (int i = 0; i < request.locations.size(); i++) { TermVectorRequest termVectorRequest = request.requests.get(i); try { IndexService indexService = indicesService.indexServiceSafe(request.index()); IndexShard indexShard = indexService.shardSafe(shardId); TermVectorResponse termVectorResponse = indexShard.termVectorService().getTermVector(termVectorRequest); response.add(request.locations.get(i), termVectorResponse); } catch (Throwable t) { if (TransportActions.isShardNotAvailableException(t)) { throw (ElasticsearchException) t; } else { logger.debug( "[{}][{}] failed to execute multi term vectors for [{}]/[{}]", t, request.index(), shardId, termVectorRequest.type(), termVectorRequest.id()); response.add( request.locations.get(i), new MultiTermVectorsResponse.Failure( request.index(), termVectorRequest.type(), termVectorRequest.id(), ExceptionsHelper.detailedMessage(t))); } } } return response; }
@Override protected Response shardOperation(Request request, ShardId shardId) { // TODO: Look into combining the shard req's docs into one in memory index. Response response = new Response(); response.items = new ArrayList<>(request.items.size()); for (Request.Item item : request.items) { Response.Item responseItem; int slot = item.slot; try { responseItem = new Response.Item(slot, percolatorService.percolate(item.request)); } catch (Throwable t) { if (TransportActions.isShardNotAvailableException(t)) { throw (ElasticsearchException) t; } else { logger.debug("{} failed to multi percolate", t, request.shardId()); responseItem = new Response.Item(slot, new StringText(ExceptionsHelper.detailedMessage(t))); } } response.items.add(responseItem); } return response; }
private void perform(@Nullable final Throwable currentFailure) { Throwable lastFailure = this.lastFailure; if (lastFailure == null || TransportActions.isReadOverrideException(currentFailure)) { lastFailure = currentFailure; this.lastFailure = currentFailure; } final ShardRouting shardRouting = shardIt.nextOrNull(); if (shardRouting == null) { Throwable failure = lastFailure; if (failure == null || isShardNotAvailableException(failure)) { failure = new NoShardAvailableActionException( null, LoggerMessageFormat.format( "No shard available for [{}]", internalRequest.request()), failure); } else { if (logger.isDebugEnabled()) { logger.debug("{}: failed to execute [{}]", failure, null, internalRequest.request()); } } listener.onFailure(failure); return; } DiscoveryNode node = nodes.get(shardRouting.currentNodeId()); if (node == null) { onFailure(shardRouting, new NoShardAvailableActionException(shardRouting.shardId())); } else { internalRequest.request().internalShardId = shardRouting.shardId(); if (logger.isTraceEnabled()) { logger.trace( "sending request [{}] to shard [{}] on node [{}]", internalRequest.request(), internalRequest.request().internalShardId, node); } transportService.sendRequest( node, transportShardAction, internalRequest.request(), new BaseTransportResponseHandler<Response>() { @Override public Response newInstance() { return newResponse(); } @Override public String executor() { return ThreadPool.Names.SAME; } @Override public void handleResponse(final Response response) { listener.onResponse(response); } @Override public void handleException(TransportException exp) { onFailure(shardRouting, exp); } }); } }
private void onFirstPhaseResult( final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId, final ShardIterator shardIt, Exception e) { // we always add the shard failure for a specific shard instance // we do make sure to clean it on a successful response from a shard SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId()); addShardFailure(shardIndex, shardTarget, e); if (totalOps.incrementAndGet() == expectedTotalOps) { if (logger.isDebugEnabled()) { if (e != null && !TransportActions.isShardNotAvailableException(e)) { logger.debug( (Supplier<?>) () -> new ParameterizedMessage( "{}: Failed to execute [{}]", shard != null ? shard.shortSummary() : shardIt.shardId(), request), e); } else if (logger.isTraceEnabled()) { logger.trace( (Supplier<?>) () -> new ParameterizedMessage("{}: Failed to execute [{}]", shard, request), e); } } final ShardSearchFailure[] shardSearchFailures = buildShardFailures(); if (successfulOps.get() == 0) { if (logger.isDebugEnabled()) { logger.debug( (Supplier<?>) () -> new ParameterizedMessage( "All shards failed for phase: [{}]", firstPhaseName()), e); } // no successful ops, raise an exception raiseEarlyFailure( new SearchPhaseExecutionException( firstPhaseName(), "all shards failed", e, shardSearchFailures)); } else { try { innerMoveToSecondPhase(); } catch (Exception inner) { inner.addSuppressed(e); raiseEarlyFailure( new ReduceSearchPhaseException(firstPhaseName(), "", inner, shardSearchFailures)); } } } else { final ShardRouting nextShard = shardIt.nextOrNull(); final boolean lastShard = nextShard == null; // trace log this exception logger.trace( (Supplier<?>) () -> new ParameterizedMessage( "{}: Failed to execute [{}] lastShard [{}]", shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard), e); if (!lastShard) { try { performFirstPhase(shardIndex, shardIt, nextShard); } catch (Exception inner) { inner.addSuppressed(e); onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, inner); } } else { // no more shards active, add a failure if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception if (e != null && !TransportActions.isShardNotAvailableException(e)) { logger.debug( (Supplier<?>) () -> new ParameterizedMessage( "{}: Failed to execute [{}] lastShard [{}]", shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard), e); } } } } }
@Override protected boolean retryOnFailure(Throwable e) { return TransportActions.isShardNotAvailableException(e); }
protected boolean retryPrimaryException(Throwable e) { return e.getClass() == RetryOnPrimaryException.class || TransportActions.isShardNotAvailableException(e); }