void setFailure(ShardIterator shardIt, int shardIndex, Throwable t) {
// we don't aggregate shard failures on non active shards (but do keep the header counts
// right)
if (TransportActions.isShardNotAvailableException(t)) {
return;
}
if (!(t instanceof BroadcastShardOperationFailedException)) {
t = new BroadcastShardOperationFailedException(shardIt.shardId(), t);
}
Object response = shardsResponses.get(shardIndex);
if (response == null) {
// just override it and return
shardsResponses.set(shardIndex, t);
}
if (!(response instanceof Throwable)) {
// we should never really get here...
return;
}
// the failure is already present, try and not override it with an exception that is less
// meaningless
// for example, getting illegal shard state
if (TransportActions.isReadOverrideException(t)) {
shardsResponses.set(shardIndex, t);
}
}
protected final void addShardFailure(
final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
// we don't aggregate shard failures on non active shards (but do keep the header counts right)
if (TransportActions.isShardNotAvailableException(e)) {
return;
}
// lazily create shard failures, so we can early build the empty shard failure list in most
// cases (no failures)
if (shardFailures == null) {
synchronized (shardFailuresMutex) {
if (shardFailures == null) {
shardFailures = new AtomicArray<>(shardsIts.size());
}
}
}
ShardSearchFailure failure = shardFailures.get(shardIndex);
if (failure == null) {
shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
} else {
// the failure is already present, try and not override it with an exception that is less
// meaningless
// for example, getting illegal shard state
if (TransportActions.isReadOverrideException(e)) {
shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
}
}
}
@SuppressWarnings({"unchecked"})
void onOperation(
@Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) {
// we set the shard failure always, even if its the first in the replication group, and the
// next one
// will work (it will just override it...)
setFailure(shardIt, shardIndex, t);
ShardRouting nextShard = shardIt.nextOrNull();
if (nextShard != null) {
if (t != null) {
if (logger.isTraceEnabled()) {
if (!TransportActions.isShardNotAvailableException(t)) {
if (shard != null) {
logger.trace(shard.shortSummary() + ": Failed to execute [" + request + "]", t);
} else {
logger.trace(shardIt.shardId() + ": Failed to execute [" + request + "]", t);
}
}
}
}
// we are not threaded here if we got here from the transport
// or we possibly threaded if we got from a local threaded one,
// in which case, the next shard in the partition will not be local one
// so there is no meaning to this flag
performOperation(shardIt, nextShard, shardIndex, true);
} else {
if (logger.isDebugEnabled()) {
if (t != null) {
if (!TransportActions.isShardNotAvailableException(t)) {
if (shard != null) {
logger.debug(shard.shortSummary() + ": Failed to execute [" + request + "]", t);
} else {
logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "]", t);
}
}
}
}
if (expectedOps == counterOps.incrementAndGet()) {
finishHim();
}
}
}
/** Should an exception be ignored when the operation is performed on the replica. */
public static boolean ignoreReplicaException(Exception e) {
if (TransportActions.isShardNotAvailableException(e)) {
return true;
}
// on version conflict or document missing, it means
// that a new change has crept into the replica, and it's fine
if (isConflictException(e)) {
return true;
}
return false;
}
@Override
protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, int shardId)
throws ElasticSearchException {
IndexService indexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = indexService.shardSafe(shardId);
if (request.refresh() && !request.realtime()) {
indexShard.refresh(
new Engine.Refresh("refresh_flag_mget").force(TransportGetAction.REFRESH_FORCE));
}
MultiGetShardResponse response = new MultiGetShardResponse();
for (int i = 0; i < request.locations.size(); i++) {
String type = request.types.get(i);
String id = request.ids.get(i);
String[] fields = request.fields.get(i);
long version = request.versions.get(i);
VersionType versionType = request.versionTypes.get(i);
if (versionType == null) {
versionType = VersionType.INTERNAL;
}
FetchSourceContext fetchSourceContext = request.fetchSourceContexts.get(i);
try {
GetResult getResult =
indexShard
.getService()
.get(
type, id, fields, request.realtime(), version, versionType, fetchSourceContext);
response.add(request.locations.get(i), new GetResponse(getResult));
} catch (Throwable t) {
if (TransportActions.isShardNotAvailableException(t)) {
throw (ElasticSearchException) t;
} else {
logger.debug(
"[{}][{}] failed to execute multi_get for [{}]/[{}]",
t,
request.index(),
shardId,
type,
id);
response.add(
request.locations.get(i),
new MultiGetResponse.Failure(
request.index(), type, id, ExceptionsHelper.detailedMessage(t)));
}
}
}
return response;
}
@SuppressWarnings({"unchecked"})
void onOperation(
@Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) {
// we set the shard failure always, even if its the first in the replication group, and the
// next one
// will work (it will just override it...)
setFailure(shardIt, shardIndex, t);
ShardRouting nextShard = shardIt.nextOrNull();
if (nextShard != null) {
if (t != null) {
if (logger.isTraceEnabled()) {
if (!TransportActions.isShardNotAvailableException(t)) {
logger.trace(
"{}: failed to execute [{}]",
t,
shard != null ? shard.shortSummary() : shardIt.shardId(),
request);
}
}
}
performOperation(shardIt, nextShard, shardIndex);
} else {
if (logger.isDebugEnabled()) {
if (t != null) {
if (!TransportActions.isShardNotAvailableException(t)) {
logger.debug(
"{}: failed to execute [{}]",
t,
shard != null ? shard.shortSummary() : shardIt.shardId(),
request);
}
}
}
if (expectedOps == counterOps.incrementAndGet()) {
finishHim();
}
}
}
private Response newResponse(
Request request,
AtomicReferenceArray responses,
List<NoShardAvailableActionException> unavailableShardExceptions,
Map<String, List<ShardRouting>> nodes,
ClusterState clusterState) {
int totalShards = 0;
int successfulShards = 0;
List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>();
List<ShardOperationFailedException> exceptions = new ArrayList<>();
for (int i = 0; i < responses.length(); i++) {
if (responses.get(i) instanceof FailedNodeException) {
FailedNodeException exception = (FailedNodeException) responses.get(i);
totalShards += nodes.get(exception.nodeId()).size();
for (ShardRouting shard : nodes.get(exception.nodeId())) {
exceptions.add(
new DefaultShardOperationFailedException(shard.getIndex(), shard.getId(), exception));
}
} else {
NodeResponse response = (NodeResponse) responses.get(i);
broadcastByNodeResponses.addAll(response.results);
totalShards += response.getTotalShards();
successfulShards += response.getSuccessfulShards();
for (BroadcastShardOperationFailedException throwable : response.getExceptions()) {
if (!TransportActions.isShardNotAvailableException(throwable)) {
exceptions.add(
new DefaultShardOperationFailedException(
throwable.getIndex(), throwable.getShardId().getId(), throwable));
}
}
}
}
totalShards += unavailableShardExceptions.size();
int failedShards = exceptions.size();
return newResponse(
request,
totalShards,
successfulShards,
failedShards,
broadcastByNodeResponses,
exceptions,
clusterState);
}
@Override
protected ShardResponse processRequestItems(
ShardId shardId, ShardUpsertRequest request, AtomicBoolean killed) {
ShardResponse shardResponse = new ShardResponse();
DocTableInfo tableInfo = schemas.getWritableTable(TableIdent.fromIndexName(request.index()));
for (int i = 0; i < request.itemIndices().size(); i++) {
int location = request.itemIndices().get(i);
ShardUpsertRequest.Item item = request.items().get(i);
if (killed.get()) {
throw new CancellationException();
}
try {
indexItem(
tableInfo,
request,
item,
shardId,
item.insertValues() != null, // try insert first
0);
shardResponse.add(location);
} catch (Throwable t) {
if (!TransportActions.isShardNotAvailableException(t) && !request.continueOnError()) {
throw t;
} else {
logger.debug(
"{} failed to execute upsert for [{}]/[{}]",
t,
request.shardId(),
request.type(),
item.id());
shardResponse.add(
location,
new ShardResponse.Failure(
item.id(),
ExceptionsHelper.detailedMessage(t),
(t instanceof VersionConflictEngineException)));
}
}
}
return shardResponse;
}
@Override
protected MultiTermVectorsShardResponse shardOperation(
MultiTermVectorsShardRequest request, int shardId) throws ElasticsearchException {
MultiTermVectorsShardResponse response = new MultiTermVectorsShardResponse();
for (int i = 0; i < request.locations.size(); i++) {
TermVectorRequest termVectorRequest = request.requests.get(i);
try {
IndexService indexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = indexService.shardSafe(shardId);
TermVectorResponse termVectorResponse =
indexShard.termVectorService().getTermVector(termVectorRequest);
response.add(request.locations.get(i), termVectorResponse);
} catch (Throwable t) {
if (TransportActions.isShardNotAvailableException(t)) {
throw (ElasticsearchException) t;
} else {
logger.debug(
"[{}][{}] failed to execute multi term vectors for [{}]/[{}]",
t,
request.index(),
shardId,
termVectorRequest.type(),
termVectorRequest.id());
response.add(
request.locations.get(i),
new MultiTermVectorsResponse.Failure(
request.index(),
termVectorRequest.type(),
termVectorRequest.id(),
ExceptionsHelper.detailedMessage(t)));
}
}
}
return response;
}
@Override
protected Response shardOperation(Request request, ShardId shardId) {
// TODO: Look into combining the shard req's docs into one in memory index.
Response response = new Response();
response.items = new ArrayList<>(request.items.size());
for (Request.Item item : request.items) {
Response.Item responseItem;
int slot = item.slot;
try {
responseItem = new Response.Item(slot, percolatorService.percolate(item.request));
} catch (Throwable t) {
if (TransportActions.isShardNotAvailableException(t)) {
throw (ElasticsearchException) t;
} else {
logger.debug("{} failed to multi percolate", t, request.shardId());
responseItem =
new Response.Item(slot, new StringText(ExceptionsHelper.detailedMessage(t)));
}
}
response.items.add(responseItem);
}
return response;
}
private void perform(@Nullable final Throwable currentFailure) {
Throwable lastFailure = this.lastFailure;
if (lastFailure == null || TransportActions.isReadOverrideException(currentFailure)) {
lastFailure = currentFailure;
this.lastFailure = currentFailure;
}
final ShardRouting shardRouting = shardIt.nextOrNull();
if (shardRouting == null) {
Throwable failure = lastFailure;
if (failure == null || isShardNotAvailableException(failure)) {
failure =
new NoShardAvailableActionException(
null,
LoggerMessageFormat.format(
"No shard available for [{}]", internalRequest.request()),
failure);
} else {
if (logger.isDebugEnabled()) {
logger.debug("{}: failed to execute [{}]", failure, null, internalRequest.request());
}
}
listener.onFailure(failure);
return;
}
DiscoveryNode node = nodes.get(shardRouting.currentNodeId());
if (node == null) {
onFailure(shardRouting, new NoShardAvailableActionException(shardRouting.shardId()));
} else {
internalRequest.request().internalShardId = shardRouting.shardId();
if (logger.isTraceEnabled()) {
logger.trace(
"sending request [{}] to shard [{}] on node [{}]",
internalRequest.request(),
internalRequest.request().internalShardId,
node);
}
transportService.sendRequest(
node,
transportShardAction,
internalRequest.request(),
new BaseTransportResponseHandler<Response>() {
@Override
public Response newInstance() {
return newResponse();
}
@Override
public String executor() {
return ThreadPool.Names.SAME;
}
@Override
public void handleResponse(final Response response) {
listener.onResponse(response);
}
@Override
public void handleException(TransportException exp) {
onFailure(shardRouting, exp);
}
});
}
}
private void onFirstPhaseResult(
final int shardIndex,
@Nullable ShardRouting shard,
@Nullable String nodeId,
final ShardIterator shardIt,
Exception e) {
// we always add the shard failure for a specific shard instance
// we do make sure to clean it on a successful response from a shard
SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId());
addShardFailure(shardIndex, shardTarget, e);
if (totalOps.incrementAndGet() == expectedTotalOps) {
if (logger.isDebugEnabled()) {
if (e != null && !TransportActions.isShardNotAvailableException(e)) {
logger.debug(
(Supplier<?>)
() ->
new ParameterizedMessage(
"{}: Failed to execute [{}]",
shard != null ? shard.shortSummary() : shardIt.shardId(),
request),
e);
} else if (logger.isTraceEnabled()) {
logger.trace(
(Supplier<?>)
() -> new ParameterizedMessage("{}: Failed to execute [{}]", shard, request),
e);
}
}
final ShardSearchFailure[] shardSearchFailures = buildShardFailures();
if (successfulOps.get() == 0) {
if (logger.isDebugEnabled()) {
logger.debug(
(Supplier<?>)
() ->
new ParameterizedMessage(
"All shards failed for phase: [{}]", firstPhaseName()),
e);
}
// no successful ops, raise an exception
raiseEarlyFailure(
new SearchPhaseExecutionException(
firstPhaseName(), "all shards failed", e, shardSearchFailures));
} else {
try {
innerMoveToSecondPhase();
} catch (Exception inner) {
inner.addSuppressed(e);
raiseEarlyFailure(
new ReduceSearchPhaseException(firstPhaseName(), "", inner, shardSearchFailures));
}
}
} else {
final ShardRouting nextShard = shardIt.nextOrNull();
final boolean lastShard = nextShard == null;
// trace log this exception
logger.trace(
(Supplier<?>)
() ->
new ParameterizedMessage(
"{}: Failed to execute [{}] lastShard [{}]",
shard != null ? shard.shortSummary() : shardIt.shardId(),
request,
lastShard),
e);
if (!lastShard) {
try {
performFirstPhase(shardIndex, shardIt, nextShard);
} catch (Exception inner) {
inner.addSuppressed(e);
onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, inner);
}
} else {
// no more shards active, add a failure
if (logger.isDebugEnabled()
&& !logger.isTraceEnabled()) { // do not double log this exception
if (e != null && !TransportActions.isShardNotAvailableException(e)) {
logger.debug(
(Supplier<?>)
() ->
new ParameterizedMessage(
"{}: Failed to execute [{}] lastShard [{}]",
shard != null ? shard.shortSummary() : shardIt.shardId(),
request,
lastShard),
e);
}
}
}
}
}
@Override
protected boolean retryOnFailure(Throwable e) {
return TransportActions.isShardNotAvailableException(e);
}
protected boolean retryPrimaryException(Throwable e) {
return e.getClass() == RetryOnPrimaryException.class
|| TransportActions.isShardNotAvailableException(e);
}