@Override
public void recover(IntStream stream, RecognitionException ex) {
if (recoveryListener != null) recoveryListener.beginErrorRecovery();
removeUnexpectedElements();
if (ex instanceof FailedPredicateException && ex.token.getType() == Token.EOF) {
failedPredicateAtEOF = true;
}
super.recover(stream, ex);
if (recoveryListener != null) recoveryListener.endErrorRecovery();
}
/** Recovers the state of the shard from the gateway. */
public void recover(final boolean indexShouldExists, final RecoveryListener listener)
throws IndexShardGatewayRecoveryException, IgnoreGatewayRecoveryException {
if (indexShard.state() == IndexShardState.CLOSED) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
if (!indexShard.routingEntry().primary()) {
listener.onRecoveryFailed(
new IndexShardGatewayRecoveryException(
shardId, "Trying to recover when the shard is in backup state", null));
return;
}
try {
if (indexShard.routingEntry().restoreSource() != null) {
indexShard.recovering("from snapshot");
} else {
indexShard.recovering("from gateway");
}
} catch (IllegalIndexShardStateException e) {
// that's fine, since we might be called concurrently, just ignore this, we are already
// recovering
listener.onIgnoreRecovery("already in recovering process, " + e.getMessage());
return;
}
threadPool
.generic()
.execute(
new Runnable() {
@Override
public void run() {
recoveryStatus = new RecoveryStatus();
recoveryStatus.updateStage(RecoveryStatus.Stage.INIT);
try {
if (indexShard.routingEntry().restoreSource() != null) {
logger.debug(
"restoring from {} ...", indexShard.routingEntry().restoreSource());
snapshotService.restore(recoveryStatus);
} else {
logger.debug("starting recovery from {} ...", shardGateway);
shardGateway.recover(indexShouldExists, recoveryStatus);
}
lastIndexVersion = recoveryStatus.index().version();
lastTranslogId = -1;
lastTranslogLength = 0;
lastTotalTranslogOperations =
recoveryStatus.translog().currentTranslogOperations();
// start the shard if the gateway has not started it already. Note that if the
// gateway
// moved shard to POST_RECOVERY, it may have been started as well if:
// 1) master sent a new cluster state indicating shard is initializing
// 2) IndicesClusterStateService#applyInitializingShard will send a shard started
// event
// 3) Master will mark shard as started and this will be processed locally.
IndexShardState shardState = indexShard.state();
if (shardState != IndexShardState.POST_RECOVERY
&& shardState != IndexShardState.STARTED) {
indexShard.postRecovery("post recovery from gateway");
}
// refresh the shard
indexShard.refresh(new Engine.Refresh("post_gateway").force(true));
recoveryStatus.time(System.currentTimeMillis() - recoveryStatus.startTime());
recoveryStatus.updateStage(RecoveryStatus.Stage.DONE);
if (logger.isDebugEnabled()) {
logger.debug(
"recovery completed from [{}], took [{}]",
shardGateway,
timeValueMillis(recoveryStatus.time()));
} else if (logger.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append("recovery completed from ")
.append(shardGateway)
.append(", took [")
.append(timeValueMillis(recoveryStatus.time()))
.append("]\n");
sb.append(" index : files [")
.append(recoveryStatus.index().numberOfFiles())
.append("] with total_size [")
.append(new ByteSizeValue(recoveryStatus.index().totalSize()))
.append("], took[")
.append(TimeValue.timeValueMillis(recoveryStatus.index().time()))
.append("]\n");
sb.append(" : recovered_files [")
.append(recoveryStatus.index().numberOfRecoveredFiles())
.append("] with total_size [")
.append(new ByteSizeValue(recoveryStatus.index().recoveredTotalSize()))
.append("]\n");
sb.append(" : reusing_files [")
.append(recoveryStatus.index().numberOfReusedFiles())
.append("] with total_size [")
.append(new ByteSizeValue(recoveryStatus.index().reusedTotalSize()))
.append("]\n");
sb.append(" start : took [")
.append(TimeValue.timeValueMillis(recoveryStatus.start().time()))
.append("], check_index [")
.append(timeValueMillis(recoveryStatus.start().checkIndexTime()))
.append("]\n");
sb.append(" translog : number_of_operations [")
.append(recoveryStatus.translog().currentTranslogOperations())
.append("], took [")
.append(TimeValue.timeValueMillis(recoveryStatus.translog().time()))
.append("]");
logger.trace(sb.toString());
}
listener.onRecoveryDone();
scheduleSnapshotIfNeeded();
} catch (IndexShardGatewayRecoveryException e) {
if (indexShard.state() == IndexShardState.CLOSED) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
if ((e.getCause() instanceof IndexShardClosedException)
|| (e.getCause() instanceof IndexShardNotStartedException)) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
listener.onRecoveryFailed(e);
} catch (IndexShardClosedException e) {
listener.onIgnoreRecovery("shard closed");
} catch (IndexShardNotStartedException e) {
listener.onIgnoreRecovery("shard closed");
} catch (Exception e) {
if (indexShard.state() == IndexShardState.CLOSED) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
listener.onRecoveryFailed(
new IndexShardGatewayRecoveryException(shardId, "failed recovery", e));
}
}
});
}
