private void raiseEarlyFailure(Exception e) {
for (AtomicArray.Entry<FirstResult> entry : firstResults.asList()) {
try {
DiscoveryNode node = nodeIdToDiscoveryNode.apply(entry.value.shardTarget().nodeId());
sendReleaseSearchContext(entry.value.id(), node);
} catch (Exception inner) {
inner.addSuppressed(e);
logger.trace("failed to release context", inner);
}
}
listener.onFailure(e);
}
@Override
protected void doExecute(
final KnapsackPullRequest request, ActionListener<KnapsackPullResponse> listener) {
final KnapsackState state =
new KnapsackState().setMode("pull").setNodeName(nodeService.nodeName());
final KnapsackPullResponse response = new KnapsackPullResponse().setState(state);
try {
final BulkTransportClient transportClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.put(clientSettings(client, request))
.toBulkTransportClient();
final BulkNodeClient nodeClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.toBulkNodeClient(client);
state.setTimestamp(new DateTime());
response.setRunning(true);
knapsack.submit(
new Thread() {
public void run() {
performPull(request, state, transportClient, nodeClient);
}
});
listener.onResponse(response);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
listener.onFailure(e);
}
}
public void start() {
if (shardsIts.size() == 0) {
// no shards
try {
listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
} catch (Throwable e) {
listener.onFailure(e);
}
return;
}
request.beforeStart();
// count the local operations, and perform the non local ones
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.nextOrNull();
if (shard != null) {
performOperation(shardIt, shard, shardIndex);
} else {
// really, no shards active in this group
onOperation(
null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
}
}
}
void finishHim() {
try {
listener.onResponse(newResponse(request, shardsResponses, clusterState));
} catch (Throwable e) {
listener.onFailure(e);
}
}
void performOnPrimary(
int primaryShardId,
boolean fromDiscoveryListener,
final ShardRouting shard,
ClusterState clusterState) {
try {
PrimaryResponse<Response, ReplicaRequest> response =
shardOperationOnPrimary(
clusterState, new PrimaryOperationRequest(primaryShardId, request));
performReplicas(response);
} catch (Exception e) {
// shard has not been allocated yet, retry it here
if (retryPrimaryException(e)) {
primaryOperationStarted.set(false);
retry(fromDiscoveryListener, null);
return;
}
if (e instanceof ElasticSearchException
&& ((ElasticSearchException) e).status() == RestStatus.CONFLICT) {
if (logger.isTraceEnabled()) {
logger.trace(shard.shortSummary() + ": Failed to execute [" + request + "]", e);
}
} else {
if (logger.isDebugEnabled()) {
logger.debug(shard.shortSummary() + ": Failed to execute [" + request + "]", e);
}
}
listener.onFailure(e);
}
}
@SuppressWarnings("unchecked")
void onShardResponse(ShardId shardId, TransportShardMultiPercolateAction.Response response) {
logger.trace("{} Percolate shard response", shardId);
try {
for (TransportShardMultiPercolateAction.Response.Item item : response.items()) {
AtomicReferenceArray shardResults = responsesByItemAndShard.get(item.slot());
if (shardResults == null) {
assert false : "shardResults can't be null";
continue;
}
if (item.failed()) {
shardResults.set(
shardId.id(),
new BroadcastShardOperationFailedException(shardId, item.error().string()));
} else {
shardResults.set(shardId.id(), item.response());
}
assert expectedOperationsPerItem.get(item.slot()).get() >= 1
: "slot[" + item.slot() + "] can't be lower than one";
if (expectedOperationsPerItem.get(item.slot()).decrementAndGet() == 0) {
// Failure won't bubble up, since we fail the whole request now via the catch clause
// below,
// so expectedOperationsPerItem will not be decremented twice.
reduce(item.slot());
}
}
} catch (Throwable e) {
logger.error("{} Percolate original reduce error", e, shardId);
finalListener.onFailure(e);
}
}
@Override
public void login(final RestRequest request, final ActionListener<String[]> listener) {
String username = request.param(usernameKey);
String password = request.param(passwordKey);
final BytesReference content = request.content();
final XContentType xContentType = XContentFactory.xContentType(content);
XContentParser parser = null;
try {
parser = XContentFactory.xContent(xContentType).createParser(content);
final XContentParser.Token t = parser.nextToken();
if (t != null) {
final Map<String, Object> contentMap = parser.map();
username = MapUtil.getAsString(contentMap, usernameKey, username);
password = MapUtil.getAsString(contentMap, passwordKey, password);
}
} catch (final Exception e) {
listener.onFailure(e);
return;
} finally {
if (parser != null) {
parser.close();
}
}
if (username == null) {
listener.onResponse(new String[0]);
return;
}
processLogin(username, password, listener);
}
private void finishHim() {
try {
innerFinishHim();
} catch (Exception e) {
listener.onFailure(new ReduceSearchPhaseException("fetch", "", e, buildShardFailures()));
}
}
protected void onCompletion() {
Response response = null;
try {
response =
newResponse(request, responses, unavailableShardExceptions, nodeIds, clusterState);
} catch (Throwable t) {
logger.debug("failed to combine responses from nodes", t);
listener.onFailure(t);
}
if (response != null) {
try {
listener.onResponse(response);
} catch (Throwable t) {
listener.onFailure(t);
}
}
}
@Override
public void nodeOperation(
final NodeFetchRequest request, final ActionListener<NodeFetchResponse> fetchResponse) {
statsTables.operationStarted(request.executionNodeId(), request.jobId(), "fetch");
String ramAccountingContextId =
String.format(Locale.ENGLISH, "%s: %d", request.jobId(), request.executionNodeId());
final RamAccountingContext ramAccountingContext =
new RamAccountingContext(ramAccountingContextId, circuitBreaker);
NodeFetchOperation fetchOperation =
new NodeFetchOperation(
request.jobId(),
request.executionNodeId(),
request.jobSearchContextDocIds(),
request.toFetchReferences(),
request.closeContext(),
jobContextService,
threadPool,
functions,
ramAccountingContext);
Streamer<?>[] streamers = outputStreamers(request.toFetchReferences());
SingleBucketBuilder bucketBuilder = new SingleBucketBuilder(streamers);
final NodeFetchResponse response = new NodeFetchResponse(streamers);
Futures.addCallback(
bucketBuilder.result(),
new FutureCallback<Bucket>() {
@Override
public void onSuccess(@Nullable Bucket result) {
assert result != null;
response.rows(result);
fetchResponse.onResponse(response);
statsTables.operationFinished(
request.executionNodeId(), null, ramAccountingContext.totalBytes());
ramAccountingContext.close();
}
@Override
public void onFailure(@Nonnull Throwable t) {
fetchResponse.onFailure(t);
statsTables.operationFinished(
request.executionNodeId(),
Exceptions.messageOf(t),
ramAccountingContext.totalBytes());
ramAccountingContext.close();
}
});
try {
fetchOperation.fetch(bucketBuilder);
} catch (Throwable t) {
fetchResponse.onFailure(t);
statsTables.operationFinished(
request.executionNodeId(), Exceptions.messageOf(t), ramAccountingContext.totalBytes());
ramAccountingContext.close();
}
}
@Override
public void onFailure(final Throwable e) {
try {
close();
} catch (final Exception e1) {
// ignore
}
listener.onFailure(new DfContentException("Failed to write data.", e));
}
void finishAsFailed(Throwable failure) {
if (finished.compareAndSet(false, true)) {
Releasables.close(indexShardReference);
logger.trace("operation failed", failure);
listener.onFailure(failure);
} else {
assert false : "finishAsFailed called but operation is already finished";
}
}
void finishWithUnexpectedFailure(Throwable failure) {
logger.warn("unexpected error during the primary phase for action [{}]", failure, actionName);
if (finished.compareAndSet(false, true)) {
Releasables.close(indexShardReference);
listener.onFailure(failure);
} else {
assert false : "finishWithUnexpectedFailure called but operation is already finished";
}
}
@Override
public void write(
final File outputFile, final SearchResponse response, final ActionListener<Void> listener) {
try {
final OnLoadListener onLoadListener = new OnLoadListener(outputFile, listener);
onLoadListener.onResponse(response);
} catch (final Exception e) {
listener.onFailure(new DfContentException("Failed to write data.", e));
}
}
@Test
public void testNonEsRejectedExceptionDoesNotResultInRetryButAborts() throws Throwable {
expectedException.expect(RuntimeException.class);
expectedException.expectMessage("a random exception");
final AtomicReference<ActionListener<ShardUpsertResponse>> ref = new AtomicReference<>();
SymbolBasedTransportShardUpsertActionDelegate transportShardBulkActionDelegate =
new SymbolBasedTransportShardUpsertActionDelegate() {
@Override
public void execute(
SymbolBasedShardUpsertRequest request, ActionListener<ShardUpsertResponse> listener) {
ref.set(listener);
}
};
BulkRetryCoordinator bulkRetryCoordinator = new BulkRetryCoordinator(ImmutableSettings.EMPTY);
BulkRetryCoordinatorPool coordinatorPool = mock(BulkRetryCoordinatorPool.class);
when(coordinatorPool.coordinator(any(ShardId.class))).thenReturn(bulkRetryCoordinator);
SymbolBasedShardUpsertRequest.Builder builder =
new SymbolBasedShardUpsertRequest.Builder(
TimeValue.timeValueMillis(10),
false,
false,
null,
new Reference[] {fooRef},
UUID.randomUUID());
final SymbolBasedBulkShardProcessor<SymbolBasedShardUpsertRequest, ShardUpsertResponse>
bulkShardProcessor =
new SymbolBasedBulkShardProcessor<>(
clusterService,
mock(TransportBulkCreateIndicesAction.class),
ImmutableSettings.EMPTY,
coordinatorPool,
false,
1,
builder,
transportShardBulkActionDelegate,
UUID.randomUUID());
bulkShardProcessor.add("foo", "1", new Object[] {"bar1"}, null, null);
ActionListener<ShardUpsertResponse> listener = ref.get();
listener.onFailure(new RuntimeException("a random exception"));
assertFalse(bulkShardProcessor.add("foo", "2", new Object[] {"bar2"}, null, null));
try {
bulkShardProcessor.result().get();
} catch (ExecutionException e) {
throw e.getCause();
} finally {
bulkShardProcessor.close();
}
}
@Override
public void onFailure(Throwable e) {
if (ExceptionsHelper.unwrapCause(e) instanceof ConnectTransportException) {
int i = ++this.i;
if (i >= nodes.size()) {
listener.onFailure(
new NoNodeAvailableException(
"None of the configured nodes were available: " + nodes, e));
} else {
try {
callback.doWithNode(nodes.get((index + i) % nodes.size()), this);
} catch (Throwable t) {
// this exception can't come from the TransportService as it doesn't throw exceptions at
// all
listener.onFailure(t);
}
}
} else {
listener.onFailure(e);
}
}
public <Response> void execute(
NodeListenerCallback<Response> callback, ActionListener<Response> listener)
throws ElasticsearchException {
ImmutableList<DiscoveryNode> nodes = this.nodes;
ensureNodesAreAvailable(nodes);
int index = getNodeNumber();
RetryListener<Response> retryListener = new RetryListener<>(callback, listener, nodes, index);
DiscoveryNode node = nodes.get((index) % nodes.size());
try {
callback.doWithNode(node, retryListener);
} catch (Throwable t) {
// this exception can't come from the TransportService as it doesn't throw exception at all
listener.onFailure(t);
}
}
public void start() {
if (scrollId.getContext().length == 0) {
listener.onFailure(
new SearchPhaseExecutionException(
"query", "no nodes to search on", ShardSearchFailure.EMPTY_ARRAY));
return;
}
ScrollIdForNode[] context = scrollId.getContext();
for (int i = 0; i < context.length; i++) {
ScrollIdForNode target = context[i];
DiscoveryNode node = nodes.get(target.getNode());
if (node != null) {
executePhase(i, node, target.getScrollId());
} else {
if (logger.isDebugEnabled()) {
logger.debug(
"Node ["
+ target.getNode()
+ "] not available for scroll request ["
+ scrollId.getSource()
+ "]");
}
successfulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
}
for (ScrollIdForNode target : scrollId.getContext()) {
DiscoveryNode node = nodes.get(target.getNode());
if (node == null) {
if (logger.isDebugEnabled()) {
logger.debug(
"Node ["
+ target.getNode()
+ "] not available for scroll request ["
+ scrollId.getSource()
+ "]");
}
successfulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
}
}
void retry(@Nullable final Throwable failure) {
if (observer.isTimedOut()) {
// we running as a last attempt after a timeout has happened. don't retry
Throwable listenFailure = failure;
if (listenFailure == null) {
if (shardIt == null) {
listenFailure =
new UnavailableShardsException(
request.concreteIndex(),
-1,
"Timeout waiting for [{}], request: {}",
request.timeout(),
actionName);
} else {
listenFailure =
new UnavailableShardsException(
shardIt.shardId(),
"[{}] shardIt, [{}] active : Timeout waiting for [{}], request: {}",
shardIt.size(),
shardIt.sizeActive(),
request.timeout(),
actionName);
}
}
listener.onFailure(listenFailure);
return;
}
observer.waitForNextChange(
new ClusterStateObserver.Listener() {
@Override
public void onNewClusterState(ClusterState state) {
doStart();
}
@Override
public void onClusterServiceClose() {
listener.onFailure(new NodeClosedException(nodes.getLocalNode()));
}
@Override
public void onTimeout(TimeValue timeout) {
// just to be on the safe side, see if we can start it now?
doStart();
}
},
request.timeout());
}
private void onPhaseFailure(Throwable t, long searchId, int shardIndex) {
if (logger.isDebugEnabled()) {
logger.debug("[{}] Failed to execute query phase", t, searchId);
}
addShardFailure(shardIndex, new ShardSearchFailure(t));
successfulOps.decrementAndGet();
if (counter.decrementAndGet() == 0) {
if (successfulOps.get() == 0) {
listener.onFailure(
new SearchPhaseExecutionException(
"query_fetch", "all shards failed", t, buildShardFailures()));
} else {
finishHim();
}
}
}
public void start() {
if (nodeIds.size() == 0) {
try {
onCompletion();
} catch (Throwable e) {
listener.onFailure(e);
}
} else {
int nodeIndex = -1;
for (Map.Entry<String, List<ShardRouting>> entry : nodeIds.entrySet()) {
nodeIndex++;
DiscoveryNode node = nodes.get(entry.getKey());
sendNodeRequest(node, entry.getValue(), nodeIndex);
}
}
}
@Override
public void updateUser(
final String username,
final String password,
final String[] roles,
final ActionListener<Void> listener) {
try {
final XContentBuilder builder = jsonBuilder().startObject().field("doc").startObject();
if (password != null) {
builder.field("password", hashPassword(password));
}
if (roles != null) {
builder.field("roles", roles);
}
builder.endObject().endObject();
final String userId = getUserId(username);
client
.prepareUpdate(authIndex, userType, userId)
.setSource(builder)
.setRefresh(true)
.execute(
new ActionListener<UpdateResponse>() {
@Override
public void onResponse(final UpdateResponse response) {
if (!userId.equals(response.getId())) {
listener.onFailure(
new AuthException(RestStatus.BAD_REQUEST, "Could not update " + username));
} else {
listener.onResponse(null);
}
}
@Override
public void onFailure(final Throwable e) {
listener.onFailure(
new AuthException(
RestStatus.INTERNAL_SERVER_ERROR, "Could not update " + username, e));
}
});
} catch (final Exception e) {
listener.onFailure(
new AuthException(RestStatus.INTERNAL_SERVER_ERROR, "Could not update " + username, e));
}
}
@Override
public void performOn(
ShardRouting replicaRouting,
IndexRequest request,
ActionListener<TransportResponse.Empty> listener) {
try {
IndexShard replica =
replicationGroup
.replicas
.stream()
.filter(s -> replicaRouting.isSameAllocation(s.routingEntry()))
.findFirst()
.get();
TransportIndexAction.executeIndexRequestOnReplica(request, replica);
listener.onResponse(TransportResponse.Empty.INSTANCE);
} catch (Exception t) {
listener.onFailure(t);
}
}
@Override
protected void doExecute(
final KnapsackStateRequest request, ActionListener<KnapsackStateResponse> listener) {
final KnapsackStateResponse response = new KnapsackStateResponse();
try {
if (knapsack != null) {
for (KnapsackState state : knapsack.getExports()) {
response.addState(state);
}
for (KnapsackState state : knapsack.getImports()) {
response.addState(state);
}
}
listener.onResponse(response);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
listener.onFailure(e);
}
}
@Override
public void createUser(
final String username,
final String password,
final String[] roles,
final ActionListener<Void> listener) {
try {
final XContentBuilder builder =
jsonBuilder() //
.startObject() //
.field("username", username) //
.field("password", hashPassword(password)) //
.field("roles", roles) //
.endObject();
client
.prepareIndex(authIndex, userType, getUserId(username))
.setSource(builder)
.setRefresh(true)
.execute(
new ActionListener<IndexResponse>() {
@Override
public void onResponse(final IndexResponse response) {
listener.onResponse(null);
}
@Override
public void onFailure(final Throwable e) {
listener.onFailure(
new AuthException(
RestStatus.INTERNAL_SERVER_ERROR, "Could not create " + username, e));
}
});
} catch (final Exception e) {
listener.onFailure(
new AuthException(RestStatus.INTERNAL_SERVER_ERROR, "Could not create " + username, e));
}
}
@SuppressWarnings("unchecked")
void onShardFailure(ShardId shardId, Throwable e) {
logger.debug("{} Shard multi percolate failure", e, shardId);
try {
IntArrayList slots = shardToSlots.get(shardId);
for (int i = 0; i < slots.size(); i++) {
int slot = slots.get(i);
AtomicReferenceArray shardResults = responsesByItemAndShard.get(slot);
if (shardResults == null) {
continue;
}
shardResults.set(shardId.id(), new BroadcastShardOperationFailedException(shardId, e));
assert expectedOperationsPerItem.get(slot).get() >= 1
: "slot[" + slot + "] can't be lower than one. Caused by: " + e.getMessage();
if (expectedOperationsPerItem.get(slot).decrementAndGet() == 0) {
reduce(slot);
}
}
} catch (Throwable t) {
logger.error("{} Percolate original reduce error, original error {}", t, shardId, e);
finalListener.onFailure(t);
}
}
protected void doStart() {
final ClusterState clusterState = observer.observedState();
final Predicate<ClusterState> masterChangePredicate =
MasterNodeChangePredicate.build(clusterState);
final DiscoveryNodes nodes = clusterState.nodes();
if (nodes.isLocalNodeElectedMaster() || localExecute(request)) {
// check for block, if blocked, retry, else, execute locally
final ClusterBlockException blockException = checkBlock(request, clusterState);
if (blockException != null) {
if (!blockException.retryable()) {
listener.onFailure(blockException);
} else {
logger.trace("can't execute due to a cluster block, retrying", blockException);
retry(
blockException,
newState -> {
ClusterBlockException newException = checkBlock(request, newState);
return (newException == null || !newException.retryable());
});
}
} else {
ActionListener<Response> delegate =
new ActionListener<Response>() {
@Override
public void onResponse(Response response) {
listener.onResponse(response);
}
@Override
public void onFailure(Exception t) {
if (t instanceof Discovery.FailedToCommitClusterStateException
|| (t instanceof NotMasterException)) {
logger.debug(
(org.apache.logging.log4j.util.Supplier<?>)
() ->
new ParameterizedMessage(
"master could not publish cluster state or stepped down before publishing action [{}], scheduling a retry",
actionName),
t);
retry(t, masterChangePredicate);
} else {
listener.onFailure(t);
}
}
};
taskManager.registerChildTask(task, nodes.getLocalNodeId());
threadPool
.executor(executor)
.execute(
new ActionRunnable(delegate) {
@Override
protected void doRun() throws Exception {
masterOperation(task, request, clusterState, delegate);
}
});
}
} else {
if (nodes.getMasterNode() == null) {
logger.debug("no known master node, scheduling a retry");
retry(null, masterChangePredicate);
} else {
taskManager.registerChildTask(task, nodes.getMasterNode().getId());
transportService.sendRequest(
nodes.getMasterNode(),
actionName,
request,
new ActionListenerResponseHandler<Response>(
listener, TransportMasterNodeAction.this::newResponse) {
@Override
public void handleException(final TransportException exp) {
Throwable cause = exp.unwrapCause();
if (cause instanceof ConnectTransportException) {
// we want to retry here a bit to see if a new master is elected
logger.debug(
"connection exception while trying to forward request with action name [{}] to master node [{}], scheduling a retry. Error: [{}]",
actionName,
nodes.getMasterNode(),
exp.getDetailedMessage());
retry(cause, masterChangePredicate);
} else {
listener.onFailure(exp);
}
}
});
}
}
}
public void start() {
if (shardsIts.size() == 0) {
// no shards
try {
listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
} catch (Throwable e) {
listener.onFailure(e);
}
}
request.beforeStart();
// count the local operations, and perform the non local ones
int localOperations = 0;
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
localOperations++;
} else {
// do the remote operation here, the localAsync flag is not relevant
performOperation(shardIt, shardIndex, true);
}
} else {
// really, no shards active in this group
onOperation(
null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
}
}
// we have local operations, perform them now
if (localOperations > 0) {
if (request.operationThreading() == BroadcastOperationThreading.SINGLE_THREAD) {
request.beforeLocalFork();
threadPool
.executor(executor)
.execute(
new Runnable() {
@Override
public void run() {
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
performOperation(shardIt, shardIndex, false);
}
}
}
}
});
} else {
boolean localAsync =
request.operationThreading() == BroadcastOperationThreading.THREAD_PER_SHARD;
if (localAsync) {
request.beforeLocalFork();
}
shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
performOperation(shardIt, shardIndex, localAsync);
}
}
}
}
}
}
protected boolean doStart() {
nodes = observer.observedState().nodes();
try {
ClusterBlockException blockException = checkGlobalBlock(observer.observedState());
if (blockException != null) {
if (blockException.retryable()) {
retry(blockException);
return false;
} else {
throw blockException;
}
}
internalRequest.concreteIndex(
observer
.observedState()
.metaData()
.concreteSingleIndex(
internalRequest.request().index(), internalRequest.request().indicesOptions()));
// check if we need to execute, and if not, return
if (!resolveRequest(observer.observedState(), internalRequest, listener)) {
return true;
}
blockException = checkRequestBlock(observer.observedState(), internalRequest);
if (blockException != null) {
if (blockException.retryable()) {
retry(blockException);
return false;
} else {
throw blockException;
}
}
shardIt = shards(observer.observedState(), internalRequest);
} catch (Throwable e) {
listener.onFailure(e);
return true;
}
// no shardIt, might be in the case between index gateway recovery and shardIt initialization
if (shardIt.size() == 0) {
retry(null);
return false;
}
// this transport only make sense with an iterator that returns a single shard routing (like
// primary)
assert shardIt.size() == 1;
ShardRouting shard = shardIt.nextOrNull();
assert shard != null;
if (!shard.active()) {
retry(null);
return false;
}
if (!operationStarted.compareAndSet(false, true)) {
return true;
}
internalRequest.request().shardId = shardIt.shardId().id();
if (shard.currentNodeId().equals(nodes.localNodeId())) {
internalRequest.request().beforeLocalFork();
try {
threadPool
.executor(executor)
.execute(
new Runnable() {
@Override
public void run() {
try {
shardOperation(internalRequest, listener);
} catch (Throwable e) {
if (retryOnFailure(e)) {
operationStarted.set(false);
// we already marked it as started when we executed it (removed the
// listener) so pass false
// to re-add to the cluster listener
retry(null);
} else {
listener.onFailure(e);
}
}
}
});
} catch (Throwable e) {
if (retryOnFailure(e)) {
retry(null);
} else {
listener.onFailure(e);
}
}
} else {
DiscoveryNode node = nodes.get(shard.currentNodeId());
transportService.sendRequest(
node,
actionName,
internalRequest.request(),
transportOptions(),
new BaseTransportResponseHandler<Response>() {
@Override
public Response newInstance() {
return newResponse();
}
@Override
public String executor() {
return ThreadPool.Names.SAME;
}
@Override
public void handleResponse(Response response) {
listener.onResponse(response);
}
@Override
public void handleException(TransportException exp) {
// if we got disconnected from the node, or the node / shard is not in the right
// state (being closed)
if (exp.unwrapCause() instanceof ConnectTransportException
|| exp.unwrapCause() instanceof NodeClosedException
|| retryOnFailure(exp)) {
operationStarted.set(false);
// we already marked it as started when we executed it (removed the listener) so
// pass false
// to re-add to the cluster listener
retry(null);
} else {
listener.onFailure(exp);
}
}
});
}
return true;
}
@Override
protected void doExecute(final Request request, final ActionListener<Response> listener) {
ClusterState clusterState = clusterService.state();
ClusterBlockException blockException = checkGlobalBlock(clusterState, request);
if (blockException != null) {
throw blockException;
}
// update to concrete index
request.index(
clusterState.metaData().concreteSingleIndex(request.index(), request.indicesOptions()));
blockException = checkRequestBlock(clusterState, request);
if (blockException != null) {
throw blockException;
}
GroupShardsIterator groups;
try {
groups = shards(request);
} catch (Throwable e) {
listener.onFailure(e);
return;
}
final AtomicInteger indexCounter = new AtomicInteger();
final AtomicInteger failureCounter = new AtomicInteger();
final AtomicInteger completionCounter = new AtomicInteger(groups.size());
final AtomicReferenceArray<ShardActionResult> shardsResponses =
new AtomicReferenceArray<>(groups.size());
for (final ShardIterator shardIt : groups) {
ShardRequest shardRequest = newShardRequestInstance(request, shardIt.shardId().id());
// TODO for now, we fork operations on shardIt of the index
shardRequest.beforeLocalFork(); // optimize for local fork
shardRequest.operationThreaded(true);
// no need for threaded listener, we will fork when its done based on the index request
shardRequest.listenerThreaded(false);
shardAction.execute(
shardRequest,
new ActionListener<ShardResponse>() {
@Override
public void onResponse(ShardResponse result) {
shardsResponses.set(indexCounter.getAndIncrement(), new ShardActionResult(result));
returnIfNeeded();
}
@Override
public void onFailure(Throwable e) {
failureCounter.getAndIncrement();
int index = indexCounter.getAndIncrement();
if (accumulateExceptions()) {
shardsResponses.set(
index,
new ShardActionResult(
new DefaultShardOperationFailedException(
request.index, shardIt.shardId().id(), e)));
}
returnIfNeeded();
}
private void returnIfNeeded() {
if (completionCounter.decrementAndGet() == 0) {
List<ShardResponse> responses = Lists.newArrayList();
List<ShardOperationFailedException> failures = Lists.newArrayList();
for (int i = 0; i < shardsResponses.length(); i++) {
ShardActionResult shardActionResult = shardsResponses.get(i);
if (shardActionResult == null) {
assert !accumulateExceptions();
continue;
}
if (shardActionResult.isFailure()) {
assert accumulateExceptions() && shardActionResult.shardFailure != null;
failures.add(shardActionResult.shardFailure);
} else {
responses.add(shardActionResult.shardResponse);
}
}
assert failures.size() == 0 || failures.size() == failureCounter.get();
listener.onResponse(
newResponseInstance(request, responses, failureCounter.get(), failures));
}
}
});
}
}
