private void doRecovery(final RecoveryStatus recoveryStatus) {
assert recoveryStatus.sourceNode() != null : "can't do a recovery without a source node";
logger.trace("collecting local files for {}", recoveryStatus);
Store.MetadataSnapshot metadataSnapshot = null;
try {
metadataSnapshot = recoveryStatus.store().getMetadataOrEmpty();
} catch (IOException e) {
logger.warn("error while listing local files, recover as if there are none", e);
metadataSnapshot = Store.MetadataSnapshot.EMPTY;
} catch (Exception e) {
// this will be logged as warning later on...
logger.trace("unexpected error while listing local files, failing recovery", e);
onGoingRecoveries.failRecovery(
recoveryStatus.recoveryId(),
new RecoveryFailedException(recoveryStatus.state(), "failed to list local files", e),
true);
return;
}
final StartRecoveryRequest request =
new StartRecoveryRequest(
recoveryStatus.shardId(),
recoveryStatus.sourceNode(),
clusterService.localNode(),
false,
metadataSnapshot,
recoveryStatus.state().getType(),
recoveryStatus.recoveryId());
final AtomicReference<RecoveryResponse> responseHolder = new AtomicReference<>();
try {
logger.trace(
"[{}][{}] starting recovery from {}",
request.shardId().index().name(),
request.shardId().id(),
request.sourceNode());
recoveryStatus.indexShard().prepareForIndexRecovery();
recoveryStatus
.CancellableThreads()
.execute(
new CancellableThreads.Interruptable() {
@Override
public void run() throws InterruptedException {
responseHolder.set(
transportService
.submitRequest(
request.sourceNode(),
RecoverySource.Actions.START_RECOVERY,
request,
new FutureTransportResponseHandler<RecoveryResponse>() {
@Override
public RecoveryResponse newInstance() {
return new RecoveryResponse();
}
})
.txGet());
}
});
final RecoveryResponse recoveryResponse = responseHolder.get();
assert responseHolder != null;
final TimeValue recoveryTime = new TimeValue(recoveryStatus.state().getTimer().time());
// do this through ongoing recoveries to remove it from the collection
onGoingRecoveries.markRecoveryAsDone(recoveryStatus.recoveryId());
if (logger.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append('[')
.append(request.shardId().index().name())
.append(']')
.append('[')
.append(request.shardId().id())
.append("] ");
sb.append("recovery completed from ")
.append(request.sourceNode())
.append(", took[")
.append(recoveryTime)
.append("]\n");
sb.append(" phase1: recovered_files [")
.append(recoveryResponse.phase1FileNames.size())
.append("]")
.append(" with total_size of [")
.append(new ByteSizeValue(recoveryResponse.phase1TotalSize))
.append("]")
.append(", took [")
.append(timeValueMillis(recoveryResponse.phase1Time))
.append("], throttling_wait [")
.append(timeValueMillis(recoveryResponse.phase1ThrottlingWaitTime))
.append(']')
.append("\n");
sb.append(" : reusing_files [")
.append(recoveryResponse.phase1ExistingFileNames.size())
.append("] with total_size of [")
.append(new ByteSizeValue(recoveryResponse.phase1ExistingTotalSize))
.append("]\n");
sb.append(" phase2: start took [")
.append(timeValueMillis(recoveryResponse.startTime))
.append("]\n");
sb.append(" : recovered [")
.append(recoveryResponse.phase2Operations)
.append("]")
.append(" transaction log operations")
.append(", took [")
.append(timeValueMillis(recoveryResponse.phase2Time))
.append("]")
.append("\n");
logger.trace(sb.toString());
} else {
logger.debug(
"{} recovery done from [{}], took [{}]",
request.shardId(),
recoveryStatus.sourceNode(),
recoveryTime);
}
} catch (CancellableThreads.ExecutionCancelledException e) {
logger.trace("recovery cancelled", e);
} catch (Throwable e) {
if (logger.isTraceEnabled()) {
logger.trace(
"[{}][{}] Got exception on recovery",
e,
request.shardId().index().name(),
request.shardId().id());
}
Throwable cause = ExceptionsHelper.unwrapCause(e);
if (cause instanceof RecoveryEngineException) {
// unwrap an exception that was thrown as part of the recovery
cause = cause.getCause();
}
// do it twice, in case we have double transport exception
cause = ExceptionsHelper.unwrapCause(cause);
if (cause instanceof RecoveryEngineException) {
// unwrap an exception that was thrown as part of the recovery
cause = cause.getCause();
}
// here, we would add checks against exception that need to be retried (and not removeAndClean
// in this case)
if (cause instanceof IllegalIndexShardStateException
|| cause instanceof IndexNotFoundException
|| cause instanceof ShardNotFoundException) {
// if the target is not ready yet, retry
retryRecovery(
recoveryStatus,
"remote shard not ready",
recoverySettings.retryDelayStateSync(),
request);
return;
}
if (cause instanceof DelayRecoveryException) {
retryRecovery(recoveryStatus, cause, recoverySettings.retryDelayStateSync(), request);
return;
}
if (cause instanceof ConnectTransportException) {
logger.debug(
"delaying recovery of {} for [{}] due to networking error [{}]",
recoveryStatus.shardId(),
recoverySettings.retryDelayNetwork(),
cause.getMessage());
retryRecovery(
recoveryStatus, cause.getMessage(), recoverySettings.retryDelayNetwork(), request);
return;
}
if (cause instanceof IndexShardClosedException) {
onGoingRecoveries.failRecovery(
recoveryStatus.recoveryId(),
new RecoveryFailedException(request, "source shard is closed", cause),
false);
return;
}
if (cause instanceof AlreadyClosedException) {
onGoingRecoveries.failRecovery(
recoveryStatus.recoveryId(),
new RecoveryFailedException(request, "source shard is closed", cause),
false);
return;
}
onGoingRecoveries.failRecovery(
recoveryStatus.recoveryId(), new RecoveryFailedException(request, e), true);
}
}
/** Snapshots the given shard into the gateway. */
public synchronized void snapshot(final String reason)
throws IndexShardGatewaySnapshotFailedException {
if (!indexShard.routingEntry().primary()) {
return;
// throw new IndexShardGatewaySnapshotNotAllowedException(shardId, "Snapshot not
// allowed on non primary shard");
}
if (indexShard.routingEntry().relocating()) {
// do not snapshot when in the process of relocation of primaries so we won't get conflicts
return;
}
if (indexShard.state() == IndexShardState.CREATED) {
// shard has just been created, ignore it and return
return;
}
if (indexShard.state() == IndexShardState.RECOVERING) {
// shard is recovering, don't snapshot
return;
}
if (snapshotLock == null) {
try {
snapshotLock = shardGateway.obtainSnapshotLock();
} catch (Exception e) {
logger.warn("failed to obtain snapshot lock, ignoring snapshot", e);
return;
}
}
try {
SnapshotStatus snapshotStatus =
indexShard.snapshot(
new Engine.SnapshotHandler<SnapshotStatus>() {
@Override
public SnapshotStatus snapshot(
SnapshotIndexCommit snapshotIndexCommit, Translog.Snapshot translogSnapshot)
throws EngineException {
if (lastIndexVersion != snapshotIndexCommit.getGeneration()
|| lastTranslogId != translogSnapshot.translogId()
|| lastTranslogLength < translogSnapshot.length()) {
logger.debug("snapshot ({}) to {} ...", reason, shardGateway);
SnapshotStatus snapshotStatus =
shardGateway.snapshot(
new IndexShardGateway.Snapshot(
snapshotIndexCommit,
translogSnapshot,
lastIndexVersion,
lastTranslogId,
lastTranslogLength,
lastTotalTranslogOperations));
lastIndexVersion = snapshotIndexCommit.getGeneration();
lastTranslogId = translogSnapshot.translogId();
lastTranslogLength = translogSnapshot.length();
lastTotalTranslogOperations = translogSnapshot.estimatedTotalOperations();
return snapshotStatus;
}
return null;
}
});
if (snapshotStatus != null) {
if (logger.isDebugEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append("snapshot (")
.append(reason)
.append(") completed to ")
.append(shardGateway)
.append(", took [")
.append(TimeValue.timeValueMillis(snapshotStatus.time()))
.append("]\n");
sb.append(" index : version [")
.append(lastIndexVersion)
.append("], number_of_files [")
.append(snapshotStatus.index().numberOfFiles())
.append("] with total_size [")
.append(new ByteSizeValue(snapshotStatus.index().totalSize()))
.append("], took [")
.append(TimeValue.timeValueMillis(snapshotStatus.index().time()))
.append("]\n");
sb.append(" translog : id [")
.append(lastTranslogId)
.append("], number_of_operations [")
.append(snapshotStatus.translog().expectedNumberOfOperations())
.append("], took [")
.append(TimeValue.timeValueMillis(snapshotStatus.translog().time()))
.append("]");
logger.debug(sb.toString());
}
}
} catch (SnapshotFailedEngineException e) {
if (e.getCause() instanceof IllegalStateException) {
// ignore, that's fine, snapshot has not started yet
} else {
throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to snapshot", e);
}
} catch (IllegalIndexShardStateException e) {
// ignore, that's fine, snapshot has not started yet
} catch (IndexShardGatewaySnapshotFailedException e) {
throw e;
} catch (Exception e) {
throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to snapshot", e);
}
}