synchronized void cancel(IndexShard shard, String reason) {
final ShardRecoveryContext shardRecoveryContext = ongoingRecoveries.get(shard);
if (shardRecoveryContext != null) {
final List<Exception> failures = new ArrayList<>();
for (RecoverySourceHandler handlers : shardRecoveryContext.recoveryHandlers) {
try {
handlers.cancel(reason);
} catch (Exception ex) {
failures.add(ex);
} finally {
shard.recoveryStats().decCurrentAsSource();
}
}
ExceptionsHelper.maybeThrowRuntimeAndSuppress(failures);
}
}
public void recoverReplica(
IndexShard replica,
BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
boolean markAsRecovering)
throws IOException {
final DiscoveryNode pNode = getPrimaryNode();
final DiscoveryNode rNode = getDiscoveryNode(replica.routingEntry().currentNodeId());
if (markAsRecovering) {
replica.markAsRecovering(
"remote",
new RecoveryState(replica.shardId(), false, RecoveryState.Type.REPLICA, pNode, rNode));
} else {
assertEquals(replica.state(), IndexShardState.RECOVERING);
}
replica.prepareForIndexRecovery();
RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
StartRecoveryRequest request =
new StartRecoveryRequest(
replica.shardId(),
pNode,
rNode,
getMetadataSnapshotOrEmpty(replica),
RecoveryState.Type.REPLICA,
0);
RecoverySourceHandler recovery =
new RecoverySourceHandler(
primary,
recoveryTarget,
request,
() -> 0L,
e -> () -> {},
(int) ByteSizeUnit.MB.toKB(1),
logger);
recovery.recoverToTarget();
recoveryTarget.markAsDone();
replica.updateRoutingEntry(ShardRoutingHelper.moveToStarted(replica.routingEntry()));
}
private RecoveryResponse recover(final StartRecoveryRequest request) throws IOException {
final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
final IndexShard shard = indexService.getShard(request.shardId().id());
// starting recovery from that our (the source) shard state is marking the shard to be in
// recovery mode as well, otherwise
// the index operations will not be routed to it properly
RoutingNode node = clusterService.state().getRoutingNodes().node(request.targetNode().getId());
if (node == null) {
logger.debug(
"delaying recovery of {} as source node {} is unknown",
request.shardId(),
request.targetNode());
throw new DelayRecoveryException(
"source node does not have the node [" + request.targetNode() + "] in its state yet..");
}
ShardRouting routingEntry = shard.routingEntry();
if (request.isPrimaryRelocation()
&& (routingEntry.relocating() == false
|| routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) {
logger.debug(
"delaying recovery of {} as source shard is not marked yet as relocating to {}",
request.shardId(),
request.targetNode());
throw new DelayRecoveryException(
"source shard is not marked yet as relocating to [" + request.targetNode() + "]");
}
ShardRouting targetShardRouting = node.getByShardId(request.shardId());
if (targetShardRouting == null) {
logger.debug(
"delaying recovery of {} as it is not listed as assigned to target node {}",
request.shardId(),
request.targetNode());
throw new DelayRecoveryException(
"source node does not have the shard listed in its state as allocated on the node");
}
if (!targetShardRouting.initializing()) {
logger.debug(
"delaying recovery of {} as it is not listed as initializing on the target node {}. known shards state is [{}]",
request.shardId(),
request.targetNode(),
targetShardRouting.state());
throw new DelayRecoveryException(
"source node has the state of the target shard to be ["
+ targetShardRouting.state()
+ "], expecting to be [initializing]");
}
RecoverySourceHandler handler =
ongoingRecoveries.addNewRecovery(request, targetShardRouting.allocationId().getId(), shard);
logger.trace(
"[{}][{}] starting recovery to {}",
request.shardId().getIndex().getName(),
request.shardId().id(),
request.targetNode());
try {
return handler.recoverToTarget();
} finally {
ongoingRecoveries.remove(shard, handler);
}
}
private RecoveryResponse recover(final StartRecoveryRequest request) {
final IndexService indexService =
indicesService.indexServiceSafe(request.shardId().index().name());
final IndexShard shard = indexService.getShard(request.shardId().id());
// starting recovery from that our (the source) shard state is marking the shard to be in
// recovery mode as well, otherwise
// the index operations will not be routed to it properly
RoutingNode node = clusterService.state().getRoutingNodes().node(request.targetNode().id());
if (node == null) {
logger.debug(
"delaying recovery of {} as source node {} is unknown",
request.shardId(),
request.targetNode());
throw new DelayRecoveryException(
"source node does not have the node [" + request.targetNode() + "] in its state yet..");
}
ShardRouting targetShardRouting = null;
for (ShardRouting shardRouting : node) {
if (shardRouting.shardId().equals(request.shardId())) {
targetShardRouting = shardRouting;
break;
}
}
if (targetShardRouting == null) {
logger.debug(
"delaying recovery of {} as it is not listed as assigned to target node {}",
request.shardId(),
request.targetNode());
throw new DelayRecoveryException(
"source node does not have the shard listed in its state as allocated on the node");
}
if (!targetShardRouting.initializing()) {
logger.debug(
"delaying recovery of {} as it is not listed as initializing on the target node {}. known shards state is [{}]",
request.shardId(),
request.targetNode(),
targetShardRouting.state());
throw new DelayRecoveryException(
"source node has the state of the target shard to be ["
+ targetShardRouting.state()
+ "], expecting to be [initializing]");
}
logger.trace(
"[{}][{}] starting recovery to {}, mark_as_relocated {}",
request.shardId().index().name(),
request.shardId().id(),
request.targetNode(),
request.markAsRelocated());
final RecoverySourceHandler handler;
if (IndexMetaData.isOnSharedFilesystem(shard.indexSettings())) {
handler =
new SharedFSRecoverySourceHandler(
shard, request, recoverySettings, transportService, logger);
} else {
handler =
new RecoverySourceHandler(shard, request, recoverySettings, transportService, logger);
}
ongoingRecoveries.add(shard, handler);
try {
return handler.recoverToTarget();
} finally {
ongoingRecoveries.remove(shard, handler);
}
}
public void testSendFiles() throws Throwable {
Settings settings =
Settings.builder()
.put("indices.recovery.concurrent_streams", 1)
.put("indices.recovery.concurrent_small_file_streams", 1)
.build();
final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
StartRecoveryRequest request =
new StartRecoveryRequest(
shardId,
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
null,
RecoveryState.Type.STORE,
randomLong());
Store store = newStore(createTempDir());
RecoverySourceHandler handler =
new RecoverySourceHandler(null, request, recoverySettings, null, logger);
Directory dir = store.directory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
int numDocs = randomIntBetween(10, 100);
for (int i = 0; i < numDocs; i++) {
Document document = new Document();
document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
document.add(
newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
writer.addDocument(document);
}
writer.commit();
Store.MetadataSnapshot metadata = store.getMetadata();
List<StoreFileMetaData> metas = new ArrayList<>();
for (StoreFileMetaData md : metadata) {
metas.add(md);
}
Store targetStore = newStore(createTempDir());
handler.sendFiles(
store,
metas.toArray(new StoreFileMetaData[0]),
(md) -> {
try {
return new IndexOutputOutputStream(
targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) {
@Override
public void close() throws IOException {
super.close();
store
.directory()
.sync(Collections.singleton(md.name())); // sync otherwise MDW will mess with it
}
};
} catch (IOException e) {
throw new RuntimeException(e);
}
});
Store.MetadataSnapshot targetStoreMetadata = targetStore.getMetadata();
Store.RecoveryDiff recoveryDiff = targetStoreMetadata.recoveryDiff(metadata);
assertEquals(metas.size(), recoveryDiff.identical.size());
assertEquals(0, recoveryDiff.different.size());
assertEquals(0, recoveryDiff.missing.size());
IndexReader reader = DirectoryReader.open(targetStore.directory());
assertEquals(numDocs, reader.maxDoc());
IOUtils.close(reader, writer, store, targetStore);
}
public void testHandleExceptinoOnSendSendFiles() throws Throwable {
Settings settings =
Settings.builder()
.put("indices.recovery.concurrent_streams", 1)
.put("indices.recovery.concurrent_small_file_streams", 1)
.build();
final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
StartRecoveryRequest request =
new StartRecoveryRequest(
shardId,
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
null,
RecoveryState.Type.STORE,
randomLong());
Path tempDir = createTempDir();
Store store = newStore(tempDir, false);
AtomicBoolean failedEngine = new AtomicBoolean(false);
RecoverySourceHandler handler =
new RecoverySourceHandler(null, request, recoverySettings, null, logger) {
@Override
protected void failEngine(IOException cause) {
assertFalse(failedEngine.get());
failedEngine.set(true);
}
};
Directory dir = store.directory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
int numDocs = randomIntBetween(10, 100);
for (int i = 0; i < numDocs; i++) {
Document document = new Document();
document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
document.add(
newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
writer.addDocument(document);
}
writer.commit();
writer.close();
Store.MetadataSnapshot metadata = store.getMetadata();
List<StoreFileMetaData> metas = new ArrayList<>();
for (StoreFileMetaData md : metadata) {
metas.add(md);
}
final boolean throwCorruptedIndexException = randomBoolean();
Store targetStore = newStore(createTempDir(), false);
try {
handler.sendFiles(
store,
metas.toArray(new StoreFileMetaData[0]),
(md) -> {
if (throwCorruptedIndexException) {
throw new RuntimeException(new CorruptIndexException("foo", "bar"));
} else {
throw new RuntimeException("boom");
}
});
fail("exception index");
} catch (RuntimeException ex) {
assertNull(ExceptionsHelper.unwrapCorruption(ex));
if (throwCorruptedIndexException) {
assertEquals(
ex.getMessage(), "[File corruption occurred on recovery but checksums are ok]");
} else {
assertEquals(ex.getMessage(), "boom");
}
} catch (CorruptIndexException ex) {
fail("not expected here");
}
assertFalse(failedEngine.get());
IOUtils.close(store, targetStore);
}
public void testHandleCorruptedIndexOnSendSendFiles() throws Throwable {
Settings settings =
Settings.builder()
.put("indices.recovery.concurrent_streams", 1)
.put("indices.recovery.concurrent_small_file_streams", 1)
.build();
final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
StartRecoveryRequest request =
new StartRecoveryRequest(
shardId,
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
null,
RecoveryState.Type.STORE,
randomLong());
Path tempDir = createTempDir();
Store store = newStore(tempDir, false);
AtomicBoolean failedEngine = new AtomicBoolean(false);
RecoverySourceHandler handler =
new RecoverySourceHandler(null, request, recoverySettings, null, logger) {
@Override
protected void failEngine(IOException cause) {
assertFalse(failedEngine.get());
failedEngine.set(true);
}
};
Directory dir = store.directory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
int numDocs = randomIntBetween(10, 100);
for (int i = 0; i < numDocs; i++) {
Document document = new Document();
document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
document.add(
newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
writer.addDocument(document);
}
writer.commit();
writer.close();
Store.MetadataSnapshot metadata = store.getMetadata();
List<StoreFileMetaData> metas = new ArrayList<>();
for (StoreFileMetaData md : metadata) {
metas.add(md);
}
CorruptionUtils.corruptFile(
getRandom(),
FileSystemUtils.files(
tempDir,
(p) ->
(p.getFileName().toString().equals("write.lock")
|| p.getFileName().toString().startsWith("extra"))
== false));
Store targetStore = newStore(createTempDir(), false);
try {
handler.sendFiles(
store,
metas.toArray(new StoreFileMetaData[0]),
(md) -> {
try {
return new IndexOutputOutputStream(
targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) {
@Override
public void close() throws IOException {
super.close();
store
.directory()
.sync(
Collections.singleton(md.name())); // sync otherwise MDW will mess with it
}
};
} catch (IOException e) {
throw new RuntimeException(e);
}
});
fail("corrupted index");
} catch (IOException ex) {
assertNotNull(ExceptionsHelper.unwrapCorruption(ex));
}
assertTrue(failedEngine.get());
IOUtils.close(store, targetStore);
}