private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles)
throws IOException {
ClusterState state = client().admin().cluster().prepareState().get().getState();
Index test = state.metaData().index("test").getIndex();
GroupShardsIterator shardIterators =
state.getRoutingTable().activePrimaryShardsGrouped(new String[] {"test"}, false);
List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
ShardRouting shardRouting = shardIterator.nextOrNull();
assertNotNull(shardRouting);
assertTrue(shardRouting.primary());
assertTrue(shardRouting.assignedToNode());
String nodeId = shardRouting.currentNodeId();
NodesStatsResponse nodeStatses =
client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
String path = info.getPath();
Path file =
PathUtils.get(path)
.resolve("indices")
.resolve(test.getUUID())
.resolve(Integer.toString(shardRouting.getId()))
.resolve("index");
if (Files.exists(file)) { // multi data path might only have one path in use
try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
for (Path item : stream) {
if (Files.isRegularFile(item)
&& "write.lock".equals(item.getFileName().toString()) == false) {
if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) {
files.add(item);
}
}
}
}
}
}
pruneOldDeleteGenerations(files);
CorruptionUtils.corruptFile(random(), files.toArray(new Path[0]));
return shardRouting;
}
public void testHandleCorruptedIndexOnSendSendFiles() throws Throwable {
Settings settings =
Settings.builder()
.put("indices.recovery.concurrent_streams", 1)
.put("indices.recovery.concurrent_small_file_streams", 1)
.build();
final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
StartRecoveryRequest request =
new StartRecoveryRequest(
shardId,
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
new DiscoveryNode("b", DummyTransportAddress.INSTANCE, Version.CURRENT),
null,
RecoveryState.Type.STORE,
randomLong());
Path tempDir = createTempDir();
Store store = newStore(tempDir, false);
AtomicBoolean failedEngine = new AtomicBoolean(false);
RecoverySourceHandler handler =
new RecoverySourceHandler(null, request, recoverySettings, null, logger) {
@Override
protected void failEngine(IOException cause) {
assertFalse(failedEngine.get());
failedEngine.set(true);
}
};
Directory dir = store.directory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
int numDocs = randomIntBetween(10, 100);
for (int i = 0; i < numDocs; i++) {
Document document = new Document();
document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
document.add(
newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
writer.addDocument(document);
}
writer.commit();
writer.close();
Store.MetadataSnapshot metadata = store.getMetadata();
List<StoreFileMetaData> metas = new ArrayList<>();
for (StoreFileMetaData md : metadata) {
metas.add(md);
}
CorruptionUtils.corruptFile(
getRandom(),
FileSystemUtils.files(
tempDir,
(p) ->
(p.getFileName().toString().equals("write.lock")
|| p.getFileName().toString().startsWith("extra"))
== false));
Store targetStore = newStore(createTempDir(), false);
try {
handler.sendFiles(
store,
metas.toArray(new StoreFileMetaData[0]),
(md) -> {
try {
return new IndexOutputOutputStream(
targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) {
@Override
public void close() throws IOException {
super.close();
store
.directory()
.sync(
Collections.singleton(md.name())); // sync otherwise MDW will mess with it
}
};
} catch (IOException e) {
throw new RuntimeException(e);
}
});
fail("corrupted index");
} catch (IOException ex) {
assertNotNull(ExceptionsHelper.unwrapCorruption(ex));
}
assertTrue(failedEngine.get());
IOUtils.close(store, targetStore);
}