protected AbstractSearchAsyncAction(
Logger logger,
SearchTransportService searchTransportService,
Function<String, DiscoveryNode> nodeIdToDiscoveryNode,
Map<String, AliasFilter> aliasFilter,
Executor executor,
SearchRequest request,
ActionListener<SearchResponse> listener,
GroupShardsIterator shardsIts,
long startTime,
long clusterStateVersion,
SearchTask task) {
super(startTime);
this.logger = logger;
this.searchTransportService = searchTransportService;
this.executor = executor;
this.request = request;
this.task = task;
this.listener = listener;
this.nodeIdToDiscoveryNode = nodeIdToDiscoveryNode;
this.clusterStateVersion = clusterStateVersion;
this.shardsIts = shardsIts;
expectedSuccessfulOps = shardsIts.size();
// we need to add 1 for non active partition, since we count it in the total!
expectedTotalOps = shardsIts.totalSizeWith1ForEmpty();
firstResults = new AtomicArray<>(shardsIts.size());
this.aliasFilter = aliasFilter;
}
public void start() {
if (shardsIts.size() == 0) {
// no shards
try {
listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
} catch (Throwable e) {
listener.onFailure(e);
}
return;
}
request.beforeStart();
// count the local operations, and perform the non local ones
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.nextOrNull();
if (shard != null) {
performOperation(shardIt, shard, shardIndex);
} else {
// really, no shards active in this group
onOperation(
null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
}
}
}
protected AsyncBroadcastAction(Request request, ActionListener<Response> listener) {
this.request = request;
this.listener = listener;
clusterState = clusterService.state();
ClusterBlockException blockException = checkGlobalBlock(clusterState, request);
if (blockException != null) {
throw blockException;
}
// update to concrete indices
String[] concreteIndices =
clusterState.metaData().concreteIndices(request.indicesOptions(), request.indices());
blockException = checkRequestBlock(clusterState, request, concreteIndices);
if (blockException != null) {
throw blockException;
}
nodes = clusterState.nodes();
logger.trace("resolving shards based on cluster state version [{}]", clusterState.version());
shardsIts = shards(clusterState, request, concreteIndices);
expectedOps = shardsIts.size();
shardsResponses = new AtomicReferenceArray<Object>(expectedOps);
}
protected final void addShardFailure(
final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
// we don't aggregate shard failures on non active shards (but do keep the header counts right)
if (TransportActions.isShardNotAvailableException(e)) {
return;
}
// lazily create shard failures, so we can early build the empty shard failure list in most
// cases (no failures)
if (shardFailures == null) {
synchronized (shardFailuresMutex) {
if (shardFailures == null) {
shardFailures = new AtomicArray<>(shardsIts.size());
}
}
}
ShardSearchFailure failure = shardFailures.get(shardIndex);
if (failure == null) {
shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
} else {
// the failure is already present, try and not override it with an exception that is less
// meaningless
// for example, getting illegal shard state
if (TransportActions.isReadOverrideException(e)) {
shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
}
}
}
void performFirstPhase(
final int shardIndex, final ShardIterator shardIt, final ShardRouting shard) {
if (shard == null) {
// no more active shards... (we should not really get here, but just for safety)
onFirstPhaseResult(
shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
} else {
final DiscoveryNode node = nodeIdToDiscoveryNode.apply(shard.currentNodeId());
if (node == null) {
onFirstPhaseResult(
shardIndex,
shard,
null,
shardIt,
new NoShardAvailableActionException(shardIt.shardId()));
} else {
AliasFilter filter = this.aliasFilter.get(shard.index().getUUID());
assert filter != null;
ShardSearchTransportRequest transportRequest =
new ShardSearchTransportRequest(
request, shardIt.shardId(), shardsIts.size(), filter, startTime());
sendExecuteFirstPhase(
node,
transportRequest,
new ActionListener<FirstResult>() {
@Override
public void onResponse(FirstResult result) {
onFirstPhaseResult(shardIndex, shard.currentNodeId(), result, shardIt);
}
@Override
public void onFailure(Exception t) {
onFirstPhaseResult(shardIndex, shard, node.getId(), shardIt, t);
}
});
}
}
}
public void start() {
if (shardsIts.size() == 0) {
// no shards
try {
listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
} catch (Throwable e) {
listener.onFailure(e);
}
}
request.beforeStart();
// count the local operations, and perform the non local ones
int localOperations = 0;
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
localOperations++;
} else {
// do the remote operation here, the localAsync flag is not relevant
performOperation(shardIt, shardIndex, true);
}
} else {
// really, no shards active in this group
onOperation(
null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
}
}
// we have local operations, perform them now
if (localOperations > 0) {
if (request.operationThreading() == BroadcastOperationThreading.SINGLE_THREAD) {
request.beforeLocalFork();
threadPool
.executor(executor)
.execute(
new Runnable() {
@Override
public void run() {
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
performOperation(shardIt, shardIndex, false);
}
}
}
}
});
} else {
boolean localAsync =
request.operationThreading() == BroadcastOperationThreading.THREAD_PER_SHARD;
if (localAsync) {
request.beforeLocalFork();
}
shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
performOperation(shardIt, shardIndex, localAsync);
}
}
}
}
}
}
private int numShards(String... index) {
ClusterState state = client().admin().cluster().prepareState().get().getState();
GroupShardsIterator shardIterators =
state.getRoutingTable().activePrimaryShardsGrouped(index, false);
return shardIterators.size();
}
@Override
protected void doExecute(final Request request, final ActionListener<Response> listener) {
ClusterState clusterState = clusterService.state();
ClusterBlockException blockException = checkGlobalBlock(clusterState, request);
if (blockException != null) {
throw blockException;
}
// update to concrete index
request.index(
clusterState.metaData().concreteSingleIndex(request.index(), request.indicesOptions()));
blockException = checkRequestBlock(clusterState, request);
if (blockException != null) {
throw blockException;
}
GroupShardsIterator groups;
try {
groups = shards(request);
} catch (Throwable e) {
listener.onFailure(e);
return;
}
final AtomicInteger indexCounter = new AtomicInteger();
final AtomicInteger failureCounter = new AtomicInteger();
final AtomicInteger completionCounter = new AtomicInteger(groups.size());
final AtomicReferenceArray<ShardActionResult> shardsResponses =
new AtomicReferenceArray<>(groups.size());
for (final ShardIterator shardIt : groups) {
ShardRequest shardRequest = newShardRequestInstance(request, shardIt.shardId().id());
// TODO for now, we fork operations on shardIt of the index
shardRequest.beforeLocalFork(); // optimize for local fork
shardRequest.operationThreaded(true);
// no need for threaded listener, we will fork when its done based on the index request
shardRequest.listenerThreaded(false);
shardAction.execute(
shardRequest,
new ActionListener<ShardResponse>() {
@Override
public void onResponse(ShardResponse result) {
shardsResponses.set(indexCounter.getAndIncrement(), new ShardActionResult(result));
returnIfNeeded();
}
@Override
public void onFailure(Throwable e) {
failureCounter.getAndIncrement();
int index = indexCounter.getAndIncrement();
if (accumulateExceptions()) {
shardsResponses.set(
index,
new ShardActionResult(
new DefaultShardOperationFailedException(
request.index, shardIt.shardId().id(), e)));
}
returnIfNeeded();
}
private void returnIfNeeded() {
if (completionCounter.decrementAndGet() == 0) {
List<ShardResponse> responses = Lists.newArrayList();
List<ShardOperationFailedException> failures = Lists.newArrayList();
for (int i = 0; i < shardsResponses.length(); i++) {
ShardActionResult shardActionResult = shardsResponses.get(i);
if (shardActionResult == null) {
assert !accumulateExceptions();
continue;
}
if (shardActionResult.isFailure()) {
assert accumulateExceptions() && shardActionResult.shardFailure != null;
failures.add(shardActionResult.shardFailure);
} else {
responses.add(shardActionResult.shardResponse);
}
}
assert failures.size() == 0 || failures.size() == failureCounter.get();
listener.onResponse(
newResponseInstance(request, responses, failureCounter.get(), failures));
}
}
});
}
}
ASyncAction(
MultiPercolateRequest multiPercolateRequest,
List<Object> percolateRequests,
ActionListener<MultiPercolateResponse> finalListener,
ClusterState clusterState) {
this.finalListener = finalListener;
this.multiPercolateRequest = multiPercolateRequest;
this.percolateRequests = percolateRequests;
responsesByItemAndShard = new AtomicReferenceArray<>(percolateRequests.size());
expectedOperationsPerItem = new AtomicReferenceArray<>(percolateRequests.size());
reducedResponses = new AtomicArray<>(percolateRequests.size());
// Resolving concrete indices and routing and grouping the requests by shard
requestsByShard = new HashMap<>();
// Keep track what slots belong to what shard, in case a request to a shard fails on all
// copies
shardToSlots = new HashMap<>();
int expectedResults = 0;
for (int slot = 0; slot < percolateRequests.size(); slot++) {
Object element = percolateRequests.get(slot);
assert element != null;
if (element instanceof PercolateRequest) {
PercolateRequest percolateRequest = (PercolateRequest) element;
String[] concreteIndices;
try {
concreteIndices =
indexNameExpressionResolver.concreteIndices(clusterState, percolateRequest);
} catch (IndexNotFoundException e) {
reducedResponses.set(slot, e);
responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
expectedOperationsPerItem.set(slot, new AtomicInteger(0));
continue;
}
Map<String, Set<String>> routing =
indexNameExpressionResolver.resolveSearchRouting(
clusterState, percolateRequest.routing(), percolateRequest.indices());
// TODO: I only need shardIds, ShardIterator(ShardRouting) is only needed in
// TransportShardMultiPercolateAction
GroupShardsIterator shards =
clusterService
.operationRouting()
.searchShards(
clusterState, concreteIndices, routing, percolateRequest.preference());
if (shards.size() == 0) {
reducedResponses.set(slot, new UnavailableShardsException(null, "No shards available"));
responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
expectedOperationsPerItem.set(slot, new AtomicInteger(0));
continue;
}
// The shard id is used as index in the atomic ref array, so we need to find out how many
// shards there are regardless of routing:
int numShards =
clusterService
.operationRouting()
.searchShardsCount(clusterState, concreteIndices, null);
responsesByItemAndShard.set(slot, new AtomicReferenceArray(numShards));
expectedOperationsPerItem.set(slot, new AtomicInteger(shards.size()));
for (ShardIterator shard : shards) {
ShardId shardId = shard.shardId();
TransportShardMultiPercolateAction.Request requests = requestsByShard.get(shardId);
if (requests == null) {
requestsByShard.put(
shardId,
requests =
new TransportShardMultiPercolateAction.Request(
multiPercolateRequest,
shardId.getIndex(),
shardId.getId(),
percolateRequest.preference()));
}
logger.trace("Adding shard[{}] percolate request for item[{}]", shardId, slot);
requests.add(
new TransportShardMultiPercolateAction.Request.Item(
slot, new PercolateShardRequest(shardId, percolateRequest)));
IntArrayList items = shardToSlots.get(shardId);
if (items == null) {
shardToSlots.put(shardId, items = new IntArrayList());
}
items.add(slot);
}
expectedResults++;
} else if (element instanceof Throwable || element instanceof MultiGetResponse.Failure) {
logger.trace("item[{}] won't be executed, reason: {}", slot, element);
reducedResponses.set(slot, element);
responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
expectedOperationsPerItem.set(slot, new AtomicInteger(0));
}
}
expectedOperations = new AtomicInteger(expectedResults);
}