public void testSendFiles() throws Throwable {
Settings settings =
Settings.builder()
.put("indices.recovery.concurrent_streams", 1)
.put("indices.recovery.concurrent_small_file_streams", 1)
.build();
final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
StartRecoveryRequest request =
new StartRecoveryRequest(
shardId,
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
null,
RecoveryState.Type.STORE,
randomLong());
Store store = newStore(createTempDir());
RecoverySourceHandler handler =
new RecoverySourceHandler(null, request, recoverySettings, null, logger);
Directory dir = store.directory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
int numDocs = randomIntBetween(10, 100);
for (int i = 0; i < numDocs; i++) {
Document document = new Document();
document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
document.add(
newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
writer.addDocument(document);
}
writer.commit();
Store.MetadataSnapshot metadata = store.getMetadata();
List<StoreFileMetaData> metas = new ArrayList<>();
for (StoreFileMetaData md : metadata) {
metas.add(md);
}
Store targetStore = newStore(createTempDir());
handler.sendFiles(
store,
metas.toArray(new StoreFileMetaData[0]),
(md) -> {
try {
return new IndexOutputOutputStream(
targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) {
@Override
public void close() throws IOException {
super.close();
store
.directory()
.sync(Collections.singleton(md.name())); // sync otherwise MDW will mess with it
}
};
} catch (IOException e) {
throw new RuntimeException(e);
}
});
Store.MetadataSnapshot targetStoreMetadata = targetStore.getMetadata();
Store.RecoveryDiff recoveryDiff = targetStoreMetadata.recoveryDiff(metadata);
assertEquals(metas.size(), recoveryDiff.identical.size());
assertEquals(0, recoveryDiff.different.size());
assertEquals(0, recoveryDiff.missing.size());
IndexReader reader = DirectoryReader.open(targetStore.directory());
assertEquals(numDocs, reader.maxDoc());
IOUtils.close(reader, writer, store, targetStore);
}
public void testHandleCorruptedIndexOnSendSendFiles() throws Throwable {
Settings settings =
Settings.builder()
.put("indices.recovery.concurrent_streams", 1)
.put("indices.recovery.concurrent_small_file_streams", 1)
.build();
final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
StartRecoveryRequest request =
new StartRecoveryRequest(
shardId,
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
null,
RecoveryState.Type.STORE,
randomLong());
Path tempDir = createTempDir();
Store store = newStore(tempDir, false);
AtomicBoolean failedEngine = new AtomicBoolean(false);
RecoverySourceHandler handler =
new RecoverySourceHandler(null, request, recoverySettings, null, logger) {
@Override
protected void failEngine(IOException cause) {
assertFalse(failedEngine.get());
failedEngine.set(true);
}
};
Directory dir = store.directory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
int numDocs = randomIntBetween(10, 100);
for (int i = 0; i < numDocs; i++) {
Document document = new Document();
document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
document.add(
newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
writer.addDocument(document);
}
writer.commit();
writer.close();
Store.MetadataSnapshot metadata = store.getMetadata();
List<StoreFileMetaData> metas = new ArrayList<>();
for (StoreFileMetaData md : metadata) {
metas.add(md);
}
CorruptionUtils.corruptFile(
getRandom(),
FileSystemUtils.files(
tempDir,
(p) ->
(p.getFileName().toString().equals("write.lock")
|| p.getFileName().toString().startsWith("extra"))
== false));
Store targetStore = newStore(createTempDir(), false);
try {
handler.sendFiles(
store,
metas.toArray(new StoreFileMetaData[0]),
(md) -> {
try {
return new IndexOutputOutputStream(
targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) {
@Override
public void close() throws IOException {
super.close();
store
.directory()
.sync(
Collections.singleton(md.name())); // sync otherwise MDW will mess with it
}
};
} catch (IOException e) {
throw new RuntimeException(e);
}
});
fail("corrupted index");
} catch (IOException ex) {
assertNotNull(ExceptionsHelper.unwrapCorruption(ex));
}
assertTrue(failedEngine.get());
IOUtils.close(store, targetStore);
}