public void start() { if (shardsIts.size() == 0) { // no shards try { listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState)); } catch (Throwable e) { listener.onFailure(e); } return; } request.beforeStart(); // count the local operations, and perform the non local ones int shardIndex = -1; for (final ShardIterator shardIt : shardsIts) { shardIndex++; final ShardRouting shard = shardIt.nextOrNull(); if (shard != null) { performOperation(shardIt, shard, shardIndex); } else { // really, no shards active in this group onOperation( null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId())); } } }
@Override protected ShardIterator shards(ClusterState clusterState, UpdateRequest request) throws ElasticSearchException { if (request.shardId() != -1) { return clusterState .routingTable() .index(request.index()) .shard(request.shardId()) .primaryShardIt(); } ShardIterator shardIterator = clusterService .operationRouting() .indexShards( clusterService.state(), request.index(), request.type(), request.id(), request.routing()); ShardRouting shard; while ((shard = shardIterator.nextOrNull()) != null) { if (shard.primary()) { return new PlainShardIterator(shardIterator.shardId(), ImmutableList.of(shard)); } } return new PlainShardIterator(shardIterator.shardId(), ImmutableList.<ShardRouting>of()); }
public void start() { if (expectedSuccessfulOps == 0) { // no search shards to search on, bail with empty response // (it happens with search across _all with no indices around and consistent with broadcast // operations) listener.onResponse( new SearchResponse( InternalSearchResponse.empty(), null, 0, 0, buildTookInMillis(), ShardSearchFailure.EMPTY_ARRAY)); return; } int shardIndex = -1; for (final ShardIterator shardIt : shardsIts) { shardIndex++; final ShardRouting shard = shardIt.nextOrNull(); if (shard != null) { performFirstPhase(shardIndex, shardIt, shard); } else { // really, no shards active in this group onFirstPhaseResult( shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId())); } } }
/** start sending current requests to replicas */ @Override protected void doRun() { if (pending.get() == 0) { doFinish(); return; } ShardRouting shard; shardIt.reset(); // reset the iterator while ((shard = shardIt.nextOrNull()) != null) { // if its unassigned, nothing to do here... if (shard.unassigned()) { continue; } // we index on a replica that is initializing as well since we might not have got the event // yet that it was started. We will get an exception IllegalShardState exception if its not // started // and that's fine, we will ignore it if (shard.primary()) { if (originalPrimaryShard.currentNodeId().equals(shard.currentNodeId()) == false) { // there is a new primary, we'll have to replicate to it. performOnReplica(shard, shard.currentNodeId()); } if (shard.relocating()) { performOnReplica(shard, shard.relocatingNodeId()); } } else if (shouldExecuteReplication(indexMetaData.getSettings())) { performOnReplica(shard, shard.currentNodeId()); if (shard.relocating()) { performOnReplica(shard, shard.relocatingNodeId()); } } } }
protected ShardRouting resolvePrimary(ShardIterator shardIt) { // no shardIt, might be in the case between index gateway recovery and shardIt initialization ShardRouting shard; while ((shard = shardIt.nextOrNull()) != null) { // we only deal with primary shardIt here... if (shard.primary()) { return shard; } } return null; }
@SuppressWarnings({"unchecked"}) void onOperation( @Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) { // we set the shard failure always, even if its the first in the replication group, and the // next one // will work (it will just override it...) setFailure(shardIt, shardIndex, t); ShardRouting nextShard = shardIt.nextOrNull(); if (nextShard != null) { if (t != null) { if (logger.isTraceEnabled()) { if (!TransportActions.isShardNotAvailableException(t)) { if (shard != null) { logger.trace(shard.shortSummary() + ": Failed to execute [" + request + "]", t); } else { logger.trace(shardIt.shardId() + ": Failed to execute [" + request + "]", t); } } } } // we are not threaded here if we got here from the transport // or we possibly threaded if we got from a local threaded one, // in which case, the next shard in the partition will not be local one // so there is no meaning to this flag performOperation(shardIt, nextShard, shardIndex, true); } else { if (logger.isDebugEnabled()) { if (t != null) { if (!TransportActions.isShardNotAvailableException(t)) { if (shard != null) { logger.debug(shard.shortSummary() + ": Failed to execute [" + request + "]", t); } else { logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "]", t); } } } } if (expectedOps == counterOps.incrementAndGet()) { finishHim(); } } }
@Override public Routing getRouting(WhereClause whereClause, @Nullable String preference) { Map<String, Map<String, List<Integer>>> locations = new TreeMap<>(); GroupShardsIterator shardIterators = clusterService .operationRouting() .searchShards( clusterService.state(), Strings.EMPTY_ARRAY, new String[] {index}, null, preference); ShardRouting shardRouting; for (ShardIterator shardIterator : shardIterators) { shardRouting = shardIterator.nextOrNull(); processShardRouting(locations, shardRouting, shardIterator.shardId()); } return new Routing(locations); }
private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException { ClusterState state = client().admin().cluster().prepareState().get().getState(); Index test = state.metaData().index("test").getIndex(); GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] {"test"}, false); List<ShardIterator> iterators = iterableAsArrayList(shardIterators); ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators); ShardRouting shardRouting = shardIterator.nextOrNull(); assertNotNull(shardRouting); assertTrue(shardRouting.primary()); assertTrue(shardRouting.assignedToNode()); String nodeId = shardRouting.currentNodeId(); NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get(); Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) { String path = info.getPath(); Path file = PathUtils.get(path) .resolve("indices") .resolve(test.getUUID()) .resolve(Integer.toString(shardRouting.getId())) .resolve("index"); if (Files.exists(file)) { // multi data path might only have one path in use try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) { for (Path item : stream) { if (Files.isRegularFile(item) && "write.lock".equals(item.getFileName().toString()) == false) { if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) { files.add(item); } } } } } } pruneOldDeleteGenerations(files); CorruptionUtils.corruptFile(random(), files.toArray(new Path[0])); return shardRouting; }
@SuppressWarnings({"unchecked"}) void onOperation( @Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) { // we set the shard failure always, even if its the first in the replication group, and the // next one // will work (it will just override it...) setFailure(shardIt, shardIndex, t); ShardRouting nextShard = shardIt.nextOrNull(); if (nextShard != null) { if (t != null) { if (logger.isTraceEnabled()) { if (!TransportActions.isShardNotAvailableException(t)) { logger.trace( "{}: failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request); } } } performOperation(shardIt, nextShard, shardIndex); } else { if (logger.isDebugEnabled()) { if (t != null) { if (!TransportActions.isShardNotAvailableException(t)) { logger.debug( "{}: failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request); } } } if (expectedOps == counterOps.incrementAndGet()) { finishHim(); } } }
void performOperation(final ShardIterator shardIt, int shardIndex, boolean localAsync) { performOperation(shardIt, shardIt.nextOrNull(), shardIndex, localAsync); }
protected boolean doStart() { nodes = observer.observedState().nodes(); try { ClusterBlockException blockException = checkGlobalBlock(observer.observedState()); if (blockException != null) { if (blockException.retryable()) { retry(blockException); return false; } else { throw blockException; } } internalRequest.concreteIndex( observer .observedState() .metaData() .concreteSingleIndex( internalRequest.request().index(), internalRequest.request().indicesOptions())); // check if we need to execute, and if not, return if (!resolveRequest(observer.observedState(), internalRequest, listener)) { return true; } blockException = checkRequestBlock(observer.observedState(), internalRequest); if (blockException != null) { if (blockException.retryable()) { retry(blockException); return false; } else { throw blockException; } } shardIt = shards(observer.observedState(), internalRequest); } catch (Throwable e) { listener.onFailure(e); return true; } // no shardIt, might be in the case between index gateway recovery and shardIt initialization if (shardIt.size() == 0) { retry(null); return false; } // this transport only make sense with an iterator that returns a single shard routing (like // primary) assert shardIt.size() == 1; ShardRouting shard = shardIt.nextOrNull(); assert shard != null; if (!shard.active()) { retry(null); return false; } if (!operationStarted.compareAndSet(false, true)) { return true; } internalRequest.request().shardId = shardIt.shardId().id(); if (shard.currentNodeId().equals(nodes.localNodeId())) { internalRequest.request().beforeLocalFork(); try { threadPool .executor(executor) .execute( new Runnable() { @Override public void run() { try { shardOperation(internalRequest, listener); } catch (Throwable e) { if (retryOnFailure(e)) { operationStarted.set(false); // we already marked it as started when we executed it (removed the // listener) so pass false // to re-add to the cluster listener retry(null); } else { listener.onFailure(e); } } } }); } catch (Throwable e) { if (retryOnFailure(e)) { retry(null); } else { listener.onFailure(e); } } } else { DiscoveryNode node = nodes.get(shard.currentNodeId()); transportService.sendRequest( node, actionName, internalRequest.request(), transportOptions(), new BaseTransportResponseHandler<Response>() { @Override public Response newInstance() { return newResponse(); } @Override public String executor() { return ThreadPool.Names.SAME; } @Override public void handleResponse(Response response) { listener.onResponse(response); } @Override public void handleException(TransportException exp) { // if we got disconnected from the node, or the node / shard is not in the right // state (being closed) if (exp.unwrapCause() instanceof ConnectTransportException || exp.unwrapCause() instanceof NodeClosedException || retryOnFailure(exp)) { operationStarted.set(false); // we already marked it as started when we executed it (removed the listener) so // pass false // to re-add to the cluster listener retry(null); } else { listener.onFailure(exp); } } }); } return true; }
/** * Tests corruption that happens on a single shard when no replicas are present. We make sure that * the primary stays unassigned and all other replicas for the healthy shards happens */ public void testCorruptPrimaryNoReplica() throws ExecutionException, InterruptedException, IOException { int numDocs = scaledRandomIntBetween(100, 1000); internalCluster().ensureAtLeastNumDataNodes(2); assertAcked( prepareCreate("test") .setSettings( Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) .put( MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose .put( IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue( 1, ByteSizeUnit .PB)) // no translog based flush - it might change the .liv / // segments.N files )); ensureGreen(); IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; for (int i = 0; i < builders.length; i++) { builders[i] = client().prepareIndex("test", "type").setSource("field", "value"); } indexRandom(true, builders); ensureGreen(); assertAllSuccessful( client() .admin() .indices() .prepareFlush() .setForce(true) .setWaitIfOngoing(true) .execute() .actionGet()); // we have to flush at least once here since we don't corrupt the translog SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); ShardRouting shardRouting = corruptRandomPrimaryFile(); /* * we corrupted the primary shard - now lets make sure we never recover from it successfully */ Settings build = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1").build(); client().admin().indices().prepareUpdateSettings("test").setSettings(build).get(); client().admin().cluster().prepareReroute().get(); boolean didClusterTurnRed = awaitBusy( () -> { ClusterHealthStatus test = client() .admin() .cluster() .health(Requests.clusterHealthRequest("test")) .actionGet() .getStatus(); return test == ClusterHealthStatus.RED; }, 5, TimeUnit .MINUTES); // sometimes on slow nodes the replication / recovery is just dead slow final ClusterHealthResponse response = client().admin().cluster().health(Requests.clusterHealthRequest("test")).get(); if (response.getStatus() != ClusterHealthStatus.RED) { logger.info("Cluster turned red in busy loop: {}", didClusterTurnRed); logger.info( "cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint()); } assertThat(response.getStatus(), is(ClusterHealthStatus.RED)); ClusterState state = client().admin().cluster().prepareState().get().getState(); GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] {"test"}, false); for (ShardIterator iterator : shardIterators) { ShardRouting routing; while ((routing = iterator.nextOrNull()) != null) { if (routing.getId() == shardRouting.getId()) { assertThat(routing.state(), equalTo(ShardRoutingState.UNASSIGNED)); } else { assertThat( routing.state(), anyOf(equalTo(ShardRoutingState.RELOCATING), equalTo(ShardRoutingState.STARTED))); } } } final List<Path> files = listShardFiles(shardRouting); Path corruptedFile = null; for (Path file : files) { if (file.getFileName().toString().startsWith("corrupted_")) { corruptedFile = file; break; } } assertThat(corruptedFile, notNullValue()); }
protected void doStart() { nodes = observer.observedState().nodes(); try { ClusterBlockException blockException = checkGlobalBlock(observer.observedState()); if (blockException != null) { if (blockException.retryable()) { retry(blockException); return; } else { throw blockException; } } request.concreteIndex( indexNameExpressionResolver .concreteSingleIndex(observer.observedState(), request) .getName()); resolveRequest(observer.observedState(), request); blockException = checkRequestBlock(observer.observedState(), request); if (blockException != null) { if (blockException.retryable()) { retry(blockException); return; } else { throw blockException; } } shardIt = shards(observer.observedState(), request); } catch (Throwable e) { listener.onFailure(e); return; } // no shardIt, might be in the case between index gateway recovery and shardIt initialization if (shardIt.size() == 0) { retry(null); return; } // this transport only make sense with an iterator that returns a single shard routing (like // primary) assert shardIt.size() == 1; ShardRouting shard = shardIt.nextOrNull(); assert shard != null; if (!shard.active()) { retry(null); return; } request.shardId = shardIt.shardId(); DiscoveryNode node = nodes.get(shard.currentNodeId()); transportService.sendRequest( node, shardActionName, request, transportOptions(), new BaseTransportResponseHandler<Response>() { @Override public Response newInstance() { return newResponse(); } @Override public String executor() { return ThreadPool.Names.SAME; } @Override public void handleResponse(Response response) { listener.onResponse(response); } @Override public void handleException(TransportException exp) { Throwable cause = exp.unwrapCause(); // if we got disconnected from the node, or the node / shard is not in the right state // (being closed) if (cause instanceof ConnectTransportException || cause instanceof NodeClosedException || retryOnFailure(exp)) { retry(cause); } else { listener.onFailure(exp); } } }); }
/** Returns <tt>true</tt> if the action starting to be performed on the primary (or is done). */ public boolean start(final boolean fromClusterEvent) throws ElasticSearchException { final ClusterState clusterState = clusterService.state(); nodes = clusterState.nodes(); try { ClusterBlockException blockException = checkGlobalBlock(clusterState, request); if (blockException != null) { if (blockException.retryable()) { retry(fromClusterEvent, blockException); return false; } else { throw blockException; } } // check if we need to execute, and if not, return if (!resolveRequest(clusterState, request, listener)) { return true; } blockException = checkRequestBlock(clusterState, request); if (blockException != null) { if (blockException.retryable()) { retry(fromClusterEvent, blockException); return false; } else { throw blockException; } } shardIt = shards(clusterState, request); } catch (Exception e) { listener.onFailure(e); return true; } // no shardIt, might be in the case between index gateway recovery and shardIt initialization if (shardIt.size() == 0) { retry(fromClusterEvent, null); return false; } boolean foundPrimary = false; ShardRouting shardX; while ((shardX = shardIt.nextOrNull()) != null) { final ShardRouting shard = shardX; // we only deal with primary shardIt here... if (!shard.primary()) { continue; } if (!shard.active() || !nodes.nodeExists(shard.currentNodeId())) { retry(fromClusterEvent, null); return false; } // check here for consistency if (checkWriteConsistency) { WriteConsistencyLevel consistencyLevel = defaultWriteConsistencyLevel; if (request.consistencyLevel() != WriteConsistencyLevel.DEFAULT) { consistencyLevel = request.consistencyLevel(); } int requiredNumber = 1; if (consistencyLevel == WriteConsistencyLevel.QUORUM && shardIt.size() > 2) { // only for more than 2 in the number of shardIt it makes sense, otherwise its 1 shard // with 1 replica, quorum is 1 (which is what it is initialized to) requiredNumber = (shardIt.size() / 2) + 1; } else if (consistencyLevel == WriteConsistencyLevel.ALL) { requiredNumber = shardIt.size(); } if (shardIt.sizeActive() < requiredNumber) { retry(fromClusterEvent, null); return false; } } if (!primaryOperationStarted.compareAndSet(false, true)) { return true; } foundPrimary = true; if (shard.currentNodeId().equals(nodes.localNodeId())) { if (request.operationThreaded()) { request.beforeLocalFork(); threadPool .executor(executor) .execute( new Runnable() { @Override public void run() { performOnPrimary(shard.id(), fromClusterEvent, shard, clusterState); } }); } else { performOnPrimary(shard.id(), fromClusterEvent, shard, clusterState); } } else { DiscoveryNode node = nodes.get(shard.currentNodeId()); transportService.sendRequest( node, transportAction, request, transportOptions, new BaseTransportResponseHandler<Response>() { @Override public Response newInstance() { return newResponseInstance(); } @Override public String executor() { return ThreadPool.Names.SAME; } @Override public void handleResponse(Response response) { listener.onResponse(response); } @Override public void handleException(TransportException exp) { // if we got disconnected from the node, or the node / shard is not in the right // state (being closed) if (exp.unwrapCause() instanceof ConnectTransportException || exp.unwrapCause() instanceof NodeClosedException || retryPrimaryException(exp)) { primaryOperationStarted.set(false); // we already marked it as started when we executed it (removed the listener) so // pass false // to re-add to the cluster listener retry(false, null); } else { listener.onFailure(exp); } } }); } break; } // we should never get here, but here we go if (!foundPrimary) { retry(fromClusterEvent, null); return false; } return true; }
/** * the constructor doesn't take any action, just calculates state. Call {@link #run()} to start * replicating. */ public ReplicationPhase( ShardIterator originalShardIt, ReplicaRequest replicaRequest, Response finalResponse, ClusterStateObserver observer, ShardRouting originalPrimaryShard, InternalRequest internalRequest, ActionListener<Response> listener, Releasable indexShardReference, TimeValue shardFailedTimeout) { this.replicaRequest = replicaRequest; this.listener = listener; this.finalResponse = finalResponse; this.originalPrimaryShard = originalPrimaryShard; this.observer = observer; indexMetaData = observer.observedState().metaData().index(internalRequest.concreteIndex()); this.indexShardReference = indexShardReference; this.shardFailedTimeout = shardFailedTimeout; ShardRouting shard; // we double check on the state, if it got changed we need to make sure we take the latest one // cause // maybe a replica shard started its recovery process and we need to apply it there... // we also need to make sure if the new state has a new primary shard (that we indexed to // before) started // and assigned to another node (while the indexing happened). In that case, we want to apply // it on the // new primary shard as well... ClusterState newState = clusterService.state(); int numberOfUnassignedOrIgnoredReplicas = 0; int numberOfPendingShardInstances = 0; if (observer.observedState() != newState) { observer.reset(newState); shardIt = shards(newState, internalRequest); while ((shard = shardIt.nextOrNull()) != null) { if (shard.primary()) { if (originalPrimaryShard.currentNodeId().equals(shard.currentNodeId()) == false) { // there is a new primary, we'll have to replicate to it. numberOfPendingShardInstances++; } if (shard.relocating()) { numberOfPendingShardInstances++; } } else if (shouldExecuteReplication(indexMetaData.getSettings()) == false) { // If the replicas use shadow replicas, there is no reason to // perform the action on the replica, so skip it and // immediately return // this delays mapping updates on replicas because they have // to wait until they get the new mapping through the cluster // state, which is why we recommend pre-defined mappings for // indices using shadow replicas numberOfUnassignedOrIgnoredReplicas++; } else if (shard.unassigned()) { numberOfUnassignedOrIgnoredReplicas++; } else if (shard.relocating()) { // we need to send to two copies numberOfPendingShardInstances += 2; } else { numberOfPendingShardInstances++; } } } else { shardIt = originalShardIt; shardIt.reset(); while ((shard = shardIt.nextOrNull()) != null) { if (shard.unassigned()) { numberOfUnassignedOrIgnoredReplicas++; } else if (shard.primary()) { if (shard.relocating()) { // we have to replicate to the other copy numberOfPendingShardInstances += 1; } } else if (shouldExecuteReplication(indexMetaData.getSettings()) == false) { // If the replicas use shadow replicas, there is no reason to // perform the action on the replica, so skip it and // immediately return // this delays mapping updates on replicas because they have // to wait until they get the new mapping through the cluster // state, which is why we recommend pre-defined mappings for // indices using shadow replicas numberOfUnassignedOrIgnoredReplicas++; } else if (shard.relocating()) { // we need to send to two copies numberOfPendingShardInstances += 2; } else { numberOfPendingShardInstances++; } } } // one for the primary already done this.totalShards = 1 + numberOfPendingShardInstances + numberOfUnassignedOrIgnoredReplicas; this.pending = new AtomicInteger(numberOfPendingShardInstances); }
private void onFirstPhaseResult( final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId, final ShardIterator shardIt, Exception e) { // we always add the shard failure for a specific shard instance // we do make sure to clean it on a successful response from a shard SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId()); addShardFailure(shardIndex, shardTarget, e); if (totalOps.incrementAndGet() == expectedTotalOps) { if (logger.isDebugEnabled()) { if (e != null && !TransportActions.isShardNotAvailableException(e)) { logger.debug( (Supplier<?>) () -> new ParameterizedMessage( "{}: Failed to execute [{}]", shard != null ? shard.shortSummary() : shardIt.shardId(), request), e); } else if (logger.isTraceEnabled()) { logger.trace( (Supplier<?>) () -> new ParameterizedMessage("{}: Failed to execute [{}]", shard, request), e); } } final ShardSearchFailure[] shardSearchFailures = buildShardFailures(); if (successfulOps.get() == 0) { if (logger.isDebugEnabled()) { logger.debug( (Supplier<?>) () -> new ParameterizedMessage( "All shards failed for phase: [{}]", firstPhaseName()), e); } // no successful ops, raise an exception raiseEarlyFailure( new SearchPhaseExecutionException( firstPhaseName(), "all shards failed", e, shardSearchFailures)); } else { try { innerMoveToSecondPhase(); } catch (Exception inner) { inner.addSuppressed(e); raiseEarlyFailure( new ReduceSearchPhaseException(firstPhaseName(), "", inner, shardSearchFailures)); } } } else { final ShardRouting nextShard = shardIt.nextOrNull(); final boolean lastShard = nextShard == null; // trace log this exception logger.trace( (Supplier<?>) () -> new ParameterizedMessage( "{}: Failed to execute [{}] lastShard [{}]", shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard), e); if (!lastShard) { try { performFirstPhase(shardIndex, shardIt, nextShard); } catch (Exception inner) { inner.addSuppressed(e); onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, inner); } } else { // no more shards active, add a failure if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception if (e != null && !TransportActions.isShardNotAvailableException(e)) { logger.debug( (Supplier<?>) () -> new ParameterizedMessage( "{}: Failed to execute [{}] lastShard [{}]", shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard), e); } } } } }
void performReplicas(final PrimaryResponse<Response, ReplicaRequest> response) { if (ignoreReplicas() || shardIt.size() == 1 /* no replicas */) { postPrimaryOperation(request, response); listener.onResponse(response.response()); return; } // initialize the counter int replicaCounter = shardIt.assignedReplicasIncludingRelocating(); if (replicaCounter == 0) { postPrimaryOperation(request, response); listener.onResponse(response.response()); return; } if (replicationType == ReplicationType.ASYNC) { postPrimaryOperation(request, response); // async replication, notify the listener listener.onResponse(response.response()); // now, trick the counter so it won't decrease to 0 and notify the listeners replicaCounter = Integer.MIN_VALUE; } // we add one to the replica count to do the postPrimaryOperation replicaCounter++; AtomicInteger counter = new AtomicInteger(replicaCounter); shardIt.reset(); // reset the iterator ShardRouting shard; while ((shard = shardIt.nextOrNull()) != null) { // if its unassigned, nothing to do here... if (shard.unassigned()) { continue; } // if the shard is primary and relocating, add one to the counter since we perform it on the // replica as well // (and we already did it on the primary) boolean doOnlyOnRelocating = false; if (shard.primary()) { if (shard.relocating()) { doOnlyOnRelocating = true; } else { continue; } } // we index on a replica that is initializing as well since we might not have got the event // yet that it was started. We will get an exception IllegalShardState exception if its not // started // and that's fine, we will ignore it if (!doOnlyOnRelocating) { performOnReplica(response, counter, shard, shard.currentNodeId()); } if (shard.relocating()) { performOnReplica(response, counter, shard, shard.relocatingNodeId()); } } // now do the postPrimary operation, and check if the listener needs to be invoked postPrimaryOperation(request, response); // we also invoke here in case replicas finish before postPrimaryAction does if (counter.decrementAndGet() == 0) { listener.onResponse(response.response()); } }