@Override
public void messageReceived(JoinRequest request, TransportChannel channel) throws Exception {
ClusterState clusterState = listener.onJoin(request.node);
if (request.withClusterState) {
channel.sendResponse(new JoinResponse(clusterState));
} else {
channel.sendResponse(VoidStreamable.INSTANCE);
}
}
@Override
public void messageReceived(final NodeRequest request, TransportChannel channel)
throws Exception {
List<ShardRouting> shards = request.getShards();
final int totalShards = shards.size();
logger.trace("[{}] executing operation on [{}] shards", actionName, totalShards);
final Object[] shardResultOrExceptions = new Object[totalShards];
int shardIndex = -1;
for (final ShardRouting shardRouting : shards) {
shardIndex++;
onShardOperation(request, shardResultOrExceptions, shardIndex, shardRouting);
}
List<BroadcastShardOperationFailedException> accumulatedExceptions = new ArrayList<>();
List<ShardOperationResult> results = new ArrayList<>();
for (int i = 0; i < totalShards; i++) {
if (shardResultOrExceptions[i] instanceof BroadcastShardOperationFailedException) {
accumulatedExceptions.add(
(BroadcastShardOperationFailedException) shardResultOrExceptions[i]);
} else {
results.add((ShardOperationResult) shardResultOrExceptions[i]);
}
}
channel.sendResponse(
new NodeResponse(request.getNodeId(), totalShards, results, accumulatedExceptions));
}
@Override
public void messageReceived(
final ShardSingleOperationRequest request, final TransportChannel channel)
throws Exception {
Response response = shardOperation(request.request(), request.shardId());
channel.sendResponse(response);
}
@Override
public void messageReceived(final RecoveryFileChunkRequest request, TransportChannel channel)
throws Exception {
try (RecoveriesCollection.RecoveryRef recoveryRef =
onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
final RecoveryTarget status = recoveryRef.status();
final RecoveryState.Index indexState = status.state().getIndex();
if (request.sourceThrottleTimeInNanos() != RecoveryState.Index.UNKNOWN) {
indexState.addSourceThrottling(request.sourceThrottleTimeInNanos());
}
RateLimiter rateLimiter = recoverySettings.rateLimiter();
if (rateLimiter != null) {
long bytes = bytesSinceLastPause.addAndGet(request.content().length());
if (bytes > rateLimiter.getMinPauseCheckBytes()) {
// Time to pause
bytesSinceLastPause.addAndGet(-bytes);
long throttleTimeInNanos = rateLimiter.pause(bytes);
indexState.addTargetThrottling(throttleTimeInNanos);
status.indexShard().recoveryStats().addThrottleTime(throttleTimeInNanos);
}
}
status.writeFileChunk(
request.metadata(),
request.position(),
request.content(),
request.lastChunk(),
request.totalTranslogOps());
}
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
@Override
protected void doRun() throws Exception {
try (Releasable shardReference = getIndexShardOperationsCounter(request.internalShardId)) {
shardOperationOnReplica(request.internalShardId, request);
}
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
@Override
public void messageReceived(RecoveryCleanFilesRequest request, TransportChannel channel)
throws Exception {
try (RecoveriesCollection.StatusRef statusRef =
onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
final RecoveryStatus recoveryStatus = statusRef.status();
recoveryStatus.state().getTranslog().totalOperations(request.totalTranslogOps());
// first, we go and move files that were created with the recovery id suffix to
// the actual names, its ok if we have a corrupted index here, since we have replicas
// to recover from in case of a full cluster shutdown just when this code executes...
recoveryStatus
.indexShard()
.deleteShardState(); // we have to delete it first since even if we fail to rename the
// shard might be invalid
recoveryStatus.renameAllTempFiles();
final Store store = recoveryStatus.store();
// now write checksums
recoveryStatus.legacyChecksums().write(store);
Store.MetadataSnapshot sourceMetaData = request.sourceMetaSnapshot();
try {
store.cleanupAndVerify("recovery CleanFilesRequestHandler", sourceMetaData);
} catch (CorruptIndexException
| IndexFormatTooNewException
| IndexFormatTooOldException ex) {
// this is a fatal exception at this stage.
// this means we transferred files from the remote that have not be checksummed and they
// are
// broken. We have to clean up this shard entirely, remove all files and bubble it up to
// the
// source shard since this index might be broken there as well? The Source can handle this
// and checks
// its content on disk if possible.
try {
try {
store.removeCorruptionMarker();
} finally {
Lucene.cleanLuceneIndex(store.directory()); // clean up and delete all files
}
} catch (Throwable e) {
logger.debug("Failed to clean lucene index", e);
ex.addSuppressed(e);
}
RecoveryFailedException rfe =
new RecoveryFailedException(
recoveryStatus.state(), "failed to clean after recovery", ex);
recoveryStatus.fail(rfe, true);
throw rfe;
} catch (Exception ex) {
RecoveryFailedException rfe =
new RecoveryFailedException(
recoveryStatus.state(), "failed to clean after recovery", ex);
recoveryStatus.fail(rfe, true);
throw rfe;
}
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
}
@Override
public void messageReceived(RecoveryFinalizeRecoveryRequest request, TransportChannel channel)
throws Exception {
try (RecoveriesCollection.RecoveryRef recoveryRef =
onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
recoveryRef.status().finalizeRecovery();
}
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
@Override
public void messageReceived(
RecoveryWaitForClusterStateRequest request, TransportChannel channel) throws Exception {
try (RecoveriesCollection.RecoveryRef recoveryRef =
onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
recoveryRef.status().ensureClusterStateVersion(request.clusterStateVersion());
}
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
@Override
public void messageReceived(RecoveryCleanFilesRequest request, TransportChannel channel)
throws Exception {
try (RecoveriesCollection.RecoveryRef recoveryRef =
onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
recoveryRef.status().cleanFiles(request.totalTranslogOps(), request.sourceMetaSnapshot());
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
}
@Override
public void messageReceived(final Request request, final TransportChannel channel)
throws Exception {
if (logger.isTraceEnabled()) {
logger.trace("executing [{}] on shard [{}]", request, request.internalShardId);
}
Response response = shardOperation(request, request.internalShardId);
channel.sendResponse(response);
}
@Override
public void messageReceived(final RecoveryFileChunkRequest request, TransportChannel channel)
throws Exception {
try (RecoveriesCollection.StatusRef statusRef =
onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
final RecoveryStatus recoveryStatus = statusRef.status();
final Store store = recoveryStatus.store();
recoveryStatus.state().getTranslog().totalOperations(request.totalTranslogOps());
final RecoveryState.Index indexState = recoveryStatus.state().getIndex();
if (request.sourceThrottleTimeInNanos() != RecoveryState.Index.UNKNOWN) {
indexState.addSourceThrottling(request.sourceThrottleTimeInNanos());
}
IndexOutput indexOutput;
if (request.position() == 0) {
indexOutput =
recoveryStatus.openAndPutIndexOutput(request.name(), request.metadata(), store);
} else {
indexOutput = recoveryStatus.getOpenIndexOutput(request.name());
}
BytesReference content = request.content();
if (!content.hasArray()) {
content = content.toBytesArray();
}
RateLimiter rl = recoverySettings.rateLimiter();
if (rl != null) {
long bytes = bytesSinceLastPause.addAndGet(content.length());
if (bytes > rl.getMinPauseCheckBytes()) {
// Time to pause
bytesSinceLastPause.addAndGet(-bytes);
long throttleTimeInNanos = rl.pause(bytes);
indexState.addTargetThrottling(throttleTimeInNanos);
recoveryStatus.indexShard().recoveryStats().addThrottleTime(throttleTimeInNanos);
}
}
indexOutput.writeBytes(content.array(), content.arrayOffset(), content.length());
indexState.addRecoveredBytesToFile(request.name(), content.length());
if (indexOutput.getFilePointer() >= request.length() || request.lastChunk()) {
try {
Store.verify(indexOutput);
} finally {
// we are done
indexOutput.close();
}
// write the checksum
recoveryStatus.legacyChecksums().add(request.metadata());
final String temporaryFileName = recoveryStatus.getTempNameForFile(request.name());
assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName);
store.directory().sync(Collections.singleton(temporaryFileName));
IndexOutput remove = recoveryStatus.removeOpenIndexOutputs(request.name());
assert remove == null || remove == indexOutput; // remove maybe null if we got finished
}
}
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
@Override
public void messageReceived(VerifyNodeRepositoryRequest request, TransportChannel channel)
throws Exception {
try {
doVerify(request.repository, request.verificationToken);
} catch (Exception ex) {
logger.warn("[{}] failed to verify repository", ex, request.repository);
throw ex;
}
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
@Override
public void messageReceived(
RecoveryPrepareForTranslogOperationsRequest request, TransportChannel channel)
throws Exception {
try (RecoveriesCollection.StatusRef statusRef =
onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
final RecoveryStatus recoveryStatus = statusRef.status();
recoveryStatus.state().getTranslog().totalOperations(request.totalTranslogOps());
recoveryStatus.indexShard().skipTranslogRecovery();
}
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
protected void responseWithFailure(Throwable t) {
try {
channel.sendResponse(t);
} catch (IOException responseException) {
logger.warn(
"failed to send error message back to client for action ["
+ transportReplicaAction
+ "]",
responseException);
logger.warn("actual Exception", t);
}
}
@Override
public void messageReceived(
RecoveryPrepareForTranslogOperationsRequest request, TransportChannel channel)
throws Exception {
try (RecoveriesCollection.RecoveryRef recoveryRef =
onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
recoveryRef
.status()
.prepareForTranslogOperations(
request.totalTranslogOps(), request.getMaxUnsafeAutoIdTimestamp());
}
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
@Override
public void messageReceived(MasterPingRequest request, TransportChannel channel)
throws Exception {
DiscoveryNodes nodes = nodesProvider.nodes();
// check if we are really the same master as the one we seemed to be think we are
// this can happen if the master got "kill -9" and then another node started using the same
// port
if (!request.masterNodeId.equals(nodes.localNodeId())) {
throw new ElasticSearchIllegalStateException(
"Got ping as master with id [" + request.masterNodeId + "], but not master and no id");
}
// send a response, and note if we are connected to the master or not
channel.sendResponse(new MasterPingResponseResponse(nodes.nodeExists(request.nodeId)));
}
@Override
public void messageReceived(PingRequest request, TransportChannel channel) throws Exception {
// if we are not the node we are supposed to be pinged, send an exception
// this can happen when a kill -9 is sent, and another node is started using the same port
if (!latestNodes.localNodeId().equals(request.nodeId)) {
throw new ElasticSearchIllegalStateException(
"Got pinged as node ["
+ request.nodeId
+ "], but I am node ["
+ latestNodes.localNodeId()
+ "]");
}
channel.sendResponse(new PingResponse());
}
@Override
public void messageReceived(RecoveryFilesInfoRequest request, TransportChannel channel)
throws Exception {
try (RecoveriesCollection.RecoveryRef recoveryRef =
onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
recoveryRef
.status()
.receiveFileInfo(
request.phase1FileNames,
request.phase1FileSizes,
request.phase1ExistingFileNames,
request.phase1ExistingFileSizes,
request.totalTranslogOps);
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
}
@Override
public void messageReceived(MasterPingRequest request, TransportChannel channel)
throws Exception {
DiscoveryNodes nodes = nodesProvider.nodes();
// check if we are really the same master as the one we seemed to be think we are
// this can happen if the master got "kill -9" and then another node started using the same
// port
if (!request.masterNodeId.equals(nodes.localNodeId())) {
throw new NotMasterException();
}
// if we are no longer master, fail...
if (!nodes.localNodeMaster()) {
throw new NoLongerMasterException();
}
if (!nodes.nodeExists(request.nodeId)) {
throw new NodeDoesNotExistOnMasterException();
}
// send a response, and note if we are connected to the master or not
channel.sendResponse(new MasterPingResponseResponse(nodes.nodeExists(request.nodeId)));
}
@Override
public void messageReceived(RecoveryFilesInfoRequest request, TransportChannel channel)
throws Exception {
try (RecoveriesCollection.StatusRef statusRef =
onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
final RecoveryStatus recoveryStatus = statusRef.status();
final RecoveryState.Index index = recoveryStatus.state().getIndex();
for (int i = 0; i < request.phase1ExistingFileNames.size(); i++) {
index.addFileDetail(
request.phase1ExistingFileNames.get(i), request.phase1ExistingFileSizes.get(i), true);
}
for (int i = 0; i < request.phase1FileNames.size(); i++) {
index.addFileDetail(
request.phase1FileNames.get(i), request.phase1FileSizes.get(i), false);
}
recoveryStatus.state().getTranslog().totalOperations(request.totalTranslogOps);
recoveryStatus.state().getTranslog().totalOperationsOnStart(request.totalTranslogOps);
// recoveryBytesCount / recoveryFileCount will be set as we go...
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
}
@Override
public void messageReceived(ShardRoutingEntry request, TransportChannel channel)
throws Exception {
innerShardStarted(request.shardRouting, request.reason);
channel.sendResponse(VoidStreamable.INSTANCE);
}
@Override
public void messageReceived(
final ReplicaOperationRequest request, final TransportChannel channel) throws Exception {
shardOperationOnReplica(request);
channel.sendResponse(VoidStreamable.INSTANCE);
}
@Override
public void messageReceived(
final RecoveryTranslogOperationsRequest request, final TransportChannel channel)
throws IOException {
try (RecoveriesCollection.RecoveryRef recoveryRef =
onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
final ClusterStateObserver observer =
new ClusterStateObserver(clusterService, null, logger, threadPool.getThreadContext());
final RecoveryTarget recoveryTarget = recoveryRef.status();
try {
recoveryTarget.indexTranslogOperations(request.operations(), request.totalTranslogOps());
channel.sendResponse(TransportResponse.Empty.INSTANCE);
} catch (TranslogRecoveryPerformer.BatchOperationException exception) {
MapperException mapperException =
(MapperException) ExceptionsHelper.unwrap(exception, MapperException.class);
if (mapperException == null) {
throw exception;
}
// in very rare cases a translog replay from primary is processed before a mapping update
// on this node
// which causes local mapping changes since the mapping (clusterstate) might not have
// arrived on this node.
// we want to wait until these mappings are processed but also need to do some maintenance
// and roll back the
// number of processed (completed) operations in this batch to ensure accounting is
// correct.
logger.trace(
(Supplier<?>)
() ->
new ParameterizedMessage(
"delaying recovery due to missing mapping changes (rolling back stats for [{}] ops)",
exception.completedOperations()),
exception);
final RecoveryState.Translog translog = recoveryTarget.state().getTranslog();
translog.decrementRecoveredOperations(
exception.completedOperations()); // do the maintainance and rollback competed ops
// we do not need to use a timeout here since the entire recovery mechanism has an
// inactivity protection (it will be
// canceled)
observer.waitForNextChange(
new ClusterStateObserver.Listener() {
@Override
public void onNewClusterState(ClusterState state) {
try {
messageReceived(request, channel);
} catch (Exception e) {
onFailure(e);
}
}
protected void onFailure(Exception e) {
try {
channel.sendResponse(e);
} catch (IOException e1) {
logger.warn("failed to send error back to recovery source", e1);
}
}
@Override
public void onClusterServiceClose() {
onFailure(
new ElasticsearchException(
"cluster service was closed while waiting for mapping updates"));
}
@Override
public void onTimeout(TimeValue timeout) {
// note that we do not use a timeout (see comment above)
onFailure(
new ElasticsearchTimeoutException(
"timed out waiting for mapping updates (timeout [" + timeout + "])"));
}
});
}
}
}
@Override
public void messageReceived(LivenessRequest request, TransportChannel channel) throws Exception {
channel.sendResponse(
new LivenessResponse(clusterService.getClusterName(), clusterService.localNode()));
}
@Override
public void messageReceived(final ShardRequest request, final TransportChannel channel)
throws Exception {
channel.sendResponse(shardOperation(request));
}
@Override
public void messageReceived(final StartRecoveryRequest request, final TransportChannel channel)
throws Exception {
RecoveryResponse response = recover(request);
channel.sendResponse(response);
}
@Override
public void messageReceived(
final RecoveryTranslogOperationsRequest request, final TransportChannel channel)
throws Exception {
try (RecoveriesCollection.StatusRef statusRef =
onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
final ClusterStateObserver observer =
new ClusterStateObserver(clusterService, null, logger);
final RecoveryStatus recoveryStatus = statusRef.status();
final RecoveryState.Translog translog = recoveryStatus.state().getTranslog();
translog.totalOperations(request.totalTranslogOps());
assert recoveryStatus.indexShard().recoveryState() == recoveryStatus.state();
try {
recoveryStatus.indexShard().performBatchRecovery(request.operations());
channel.sendResponse(TransportResponse.Empty.INSTANCE);
} catch (TranslogRecoveryPerformer.BatchOperationException exception) {
MapperException mapperException =
(MapperException) ExceptionsHelper.unwrap(exception, MapperException.class);
if (mapperException == null) {
throw exception;
}
// in very rare cases a translog replay from primary is processed before a mapping update
// on this node
// which causes local mapping changes. we want to wait until these mappings are processed.
logger.trace(
"delaying recovery due to missing mapping changes (rolling back stats for [{}] ops)",
exception,
exception.completedOperations());
translog.decrementRecoveredOperations(exception.completedOperations());
// we do not need to use a timeout here since the entire recovery mechanism has an
// inactivity protection (it will be
// canceled)
observer.waitForNextChange(
new ClusterStateObserver.Listener() {
@Override
public void onNewClusterState(ClusterState state) {
try {
messageReceived(request, channel);
} catch (Exception e) {
onFailure(e);
}
}
protected void onFailure(Exception e) {
try {
channel.sendResponse(e);
} catch (IOException e1) {
logger.warn("failed to send error back to recovery source", e1);
}
}
@Override
public void onClusterServiceClose() {
onFailure(
new ElasticsearchException(
"cluster service was closed while waiting for mapping updates"));
}
@Override
public void onTimeout(TimeValue timeout) {
// note that we do not use a timeout (see comment above)
onFailure(
new ElasticsearchTimeoutException(
"timed out waiting for mapping updates (timeout [" + timeout + "])"));
}
});
}
}
}
@Override
public void messageReceived(LeaveRequest request, TransportChannel channel) throws Exception {
listener.onLeave(request.node);
channel.sendResponse(VoidStreamable.INSTANCE);
}
@Override
public void messageReceived(NodeIndexStoreDeletedMessage message, TransportChannel channel)
throws Exception {
innerNodeIndexStoreDeleted(message.index, message.nodeId);
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}