private void corruptFiles(
Path dirPath,
long[] crcs,
Integer[] corruptBlockIdxs,
DistributedFileSystem dfs,
Path[] files,
int[] numCorruptBlocksInFiles)
throws IOException {
int totalCorruptFiles = DFSUtil.getCorruptFiles(dfs).length;
TestDirectoryRaidDfs.corruptBlocksInDirectory(
conf, dirPath, crcs, corruptBlockIdxs, fileSys, dfsCluster, false, true);
String[] corruptFiles = DFSUtil.getCorruptFiles(dfs);
for (int i = 0; i < numCorruptBlocksInFiles.length; i++) {
if (numCorruptBlocksInFiles[i] > 0) totalCorruptFiles++;
}
assertEquals("files not corrupted", totalCorruptFiles, corruptFiles.length);
for (int i = 0; i < fileSizes.length; i++) {
assertEquals(
"wrong number of corrupt blocks for file " + files[i],
numCorruptBlocksInFiles[i],
RaidDFSUtil.corruptBlocksInFile(dfs, files[i].toUri().getPath(), 0, fileSizes[i]).size());
}
}
public void testMultiplePriorities() throws Exception {
long[] crcs = new long[3];
int[] seeds = new int[3];
Path dirPath = new Path("/home/test");
int stripeLength = 3;
short repl = 1;
mySetup(stripeLength);
Codec codec = Codec.getCodec("rs");
LOG.info("Starting testMultiplePriorities");
try {
// Create test file and raid it.
Path[] files =
TestRaidDfs.createTestFiles(
dirPath, fileSizes, blockSizes, crcs, seeds, fileSys, (short) 1);
FileStatus stat = fileSys.getFileStatus(dirPath);
RaidNode.doRaid(
conf,
stat,
new Path(codec.parityDirectory),
codec,
new RaidNode.Statistics(),
RaidUtils.NULL_PROGRESSABLE,
false,
repl,
repl);
Integer[] corruptBlockIdxs = new Integer[] {0, 2};
LOG.info("Corrupt block " + corruptBlockIdxs + " of directory " + dirPath);
TestDirectoryRaidDfs.corruptBlocksInDirectory(
conf, dirPath, crcs, corruptBlockIdxs, fileSys, dfsCluster, false, true);
// Create Block Fixer and fix.
FakeDistBlockIntegrityMonitor distBlockFixer = new FakeDistBlockIntegrityMonitor(conf);
assertEquals(0, distBlockFixer.submittedJobs.size());
// waiting for one job to submit
long startTime = System.currentTimeMillis();
while (System.currentTimeMillis() - startTime < 120000
&& distBlockFixer.submittedJobs.size() == 0) {
distBlockFixer.getCorruptionMonitor().checkAndReconstructBlocks();
LOG.info("Waiting for jobs to submit");
Thread.sleep(10000);
}
int submittedJob = distBlockFixer.submittedJobs.size();
LOG.info("Already Submitted " + submittedJob + " jobs");
assertTrue("Should submit more than 1 jobs", submittedJob >= 1);
// Corrupt two more blocks
corruptBlockIdxs = new Integer[] {4, 5};
LOG.info("Corrupt block " + corruptBlockIdxs + " of directory " + dirPath);
TestDirectoryRaidDfs.corruptBlocksInDirectory(
conf, dirPath, crcs, corruptBlockIdxs, fileSys, dfsCluster, false, true);
// A new job should be submitted since two blocks are corrupt.
startTime = System.currentTimeMillis();
while (System.currentTimeMillis() - startTime < 120000
&& distBlockFixer.submittedJobs.size() == submittedJob) {
distBlockFixer.getCorruptionMonitor().checkAndReconstructBlocks();
LOG.info("Waiting for more jobs to submit");
Thread.sleep(10000);
}
LOG.info("Already Submitted " + distBlockFixer.submittedJobs.size() + " jobs");
assertTrue(
"should submit more than 1 jobs",
distBlockFixer.submittedJobs.size() - submittedJob >= 1);
} finally {
myTearDown();
}
}
/**
* Tests integrity of generated block. Create a file and delete a block entirely. Wait for the
* block to be regenerated. Now stop RaidNode and corrupt the generated block. Test that
* corruption in the generated block can be detected by clients.
*/
private void generatedBlockTestCommon(String testName, int blockToCorrupt, boolean local)
throws Exception {
LOG.info("Test " + testName + " started.");
int stripeLength = 3;
mySetup(stripeLength);
long[] crcs = new long[3];
int[] seeds = new int[3];
Path dirPath = new Path("/user/dhruba/raidtest");
Path[] files =
TestRaidDfs.createTestFiles(
dirPath, fileSizes, blockSizes, crcs, seeds, fileSys, (short) 1);
Path destPath = new Path("/destraid/user/dhruba");
LOG.info("Test " + testName + " created test files");
Configuration localConf = this.getRaidNodeConfig(conf, local);
try {
cnode = RaidNode.createRaidNode(null, localConf);
TestRaidDfs.waitForDirRaided(LOG, fileSys, dirPath, destPath);
cnode.stop();
cnode.join();
DistributedFileSystem dfs = (DistributedFileSystem) fileSys;
String[] corruptFiles = DFSUtil.getCorruptFiles(dfs);
assertEquals("no corrupt files expected", 0, corruptFiles.length);
assertEquals(
"filesFixed() should return 0 before fixing files",
0,
cnode.blockIntegrityMonitor.getNumFilesFixed());
Integer[] corruptBlockIdxs = new Integer[] {blockToCorrupt};
TestDirectoryRaidDfs.corruptBlocksInDirectory(
conf, dirPath, crcs, corruptBlockIdxs, fileSys, dfsCluster, false, true);
corruptFiles = DFSUtil.getCorruptFiles(dfs);
assertEquals("files not corrupted", corruptBlockIdxs.length, corruptFiles.length);
int corruptFileIdx = -1;
for (int i = 0; i < files.length; i++) {
if (files[i].toUri().getPath().equals(corruptFiles[0])) {
corruptFileIdx = i;
break;
}
}
assertNotSame("Wrong corrupt file", -1, corruptFileIdx);
cnode = RaidNode.createRaidNode(null, localConf);
long start = System.currentTimeMillis();
while (cnode.blockIntegrityMonitor.getNumFilesFixed() < 1
&& System.currentTimeMillis() - start < 120000) {
LOG.info("Test testDirBlockFix waiting for files to be fixed.");
Thread.sleep(1000);
}
TestBlockFixer.verifyMetrics(fileSys, cnode, local, 1L, corruptBlockIdxs.length);
// Stop RaidNode
cnode.stop();
cnode.join();
cnode = null;
// The block has successfully been reconstructed.
dfs = getDFS(conf, dfs);
assertTrue(
"file not fixed",
TestRaidDfs.validateFile(
dfs, files[corruptFileIdx], fileSizes[corruptFileIdx], crcs[corruptFileIdx]));
// Now corrupt the generated block.
TestDirectoryRaidDfs.corruptBlocksInDirectory(
conf, dirPath, crcs, corruptBlockIdxs, dfs, dfsCluster, false, false);
try {
TestRaidDfs.validateFile(
dfs, files[corruptFileIdx], fileSizes[corruptFileIdx], crcs[corruptFileIdx]);
fail("Expected exception not thrown");
} catch (org.apache.hadoop.fs.ChecksumException ce) {
} catch (org.apache.hadoop.fs.BlockMissingException bme) {
}
} catch (Exception e) {
LOG.info("Test " + testName + " Exception " + e, e);
throw e;
} finally {
myTearDown();
}
LOG.info("Test " + testName + " completed.");
}
