@Test
public void testGetRemoteEditLog() throws IOException {
StorageDirectory sd =
FSImageTestUtil.mockStorageDirectory(
NameNodeDirType.EDITS,
false,
NNStorage.getFinalizedEditsFileName(1, 100),
NNStorage.getFinalizedEditsFileName(101, 200),
NNStorage.getInProgressEditsFileName(201),
NNStorage.getFinalizedEditsFileName(1001, 1100));
FileJournalManager fjm = new FileJournalManager(sd);
assertEquals("[1,100],[101,200],[1001,1100]", getLogsAsString(fjm, 1));
assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 101));
assertEquals("[1001,1100]", getLogsAsString(fjm, 201));
try {
assertEquals("[]", getLogsAsString(fjm, 150));
fail("Did not throw when asking for a txn in the middle of a log");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains("150 which is in the middle", ioe);
}
assertEquals(
"Asking for a newer log than exists should return empty list",
"",
getLogsAsString(fjm, 9999));
}
/**
* Test that we can load an edits directory with a corrupt inprogress file. The corrupt inprogress
* file should be moved to the side.
*/
@Test
public void testManyLogsWithCorruptInprogress() throws IOException {
File f = new File(TestEditLog.TEST_DIR + "/filejournaltest5");
NNStorage storage =
setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0));
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
File[] files =
new File(f, "current")
.listFiles(
new FilenameFilter() {
public boolean accept(File dir, String name) {
if (name.startsWith("edits_inprogress")) {
return true;
}
return false;
}
});
assertEquals(files.length, 1);
corruptAfterStartSegment(files[0]);
FileJournalManager jm = new FileJournalManager(sd);
assertEquals(10 * TXNS_PER_ROLL + 1, jm.getNumberOfTransactions(1));
}
/**
* Test that we can read from a stream created by FileJournalManager. Create a single edits
* directory, failing it on the final roll. Then try loading from the point of the 3rd roll.
* Verify that we read the correct number of transactions from this point.
*/
@Test
public void testReadFromStream() throws IOException {
File f = new File(TestEditLog.TEST_DIR + "/filejournaltest1");
// abort after 10th roll
NNStorage storage =
setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0));
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
FileJournalManager jm = new FileJournalManager(sd);
long expectedTotalTxnCount = TXNS_PER_ROLL * 10 + TXNS_PER_FAIL;
assertEquals(expectedTotalTxnCount, jm.getNumberOfTransactions(1));
long skippedTxns = (3 * TXNS_PER_ROLL); // skip first 3 files
long startingTxId = skippedTxns + 1;
long numTransactionsToLoad = jm.getNumberOfTransactions(startingTxId);
long numLoaded = 0;
while (numLoaded < numTransactionsToLoad) {
EditLogInputStream editIn = jm.getInputStream(startingTxId);
FSEditLogLoader.EditLogValidation val = FSEditLogLoader.validateEditLog(editIn);
long count = val.getNumTransactions();
editIn.close();
startingTxId += count;
numLoaded += count;
}
assertEquals(expectedTotalTxnCount - skippedTxns, numLoaded);
}
/** Checks the JournalManager.isSegmentInProgress() */
@Test
public void testInprogressSegment() throws IOException {
File f = new File(TestEditLog.TEST_DIR + "/testInprogressSegment");
// abort after the 5th roll
NNStorage storage =
setupEdits(Collections.<URI>singletonList(f.toURI()), 5, new AbortSpec(5, 0));
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
FileJournalManager jm = new FileJournalManager(sd);
assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(0));
boolean isOneInProgress = false;
boolean isOneNotInProgress = false;
for (RemoteEditLog rel : jm.getRemoteEditLogs(0)) {
if (rel.inProgress()) {
isOneInProgress = true;
assertTrue(jm.isSegmentInProgress(rel.getStartTxId()));
} else {
isOneNotInProgress = true;
assertFalse(jm.isSegmentInProgress(rel.getStartTxId()));
}
}
assertTrue(isOneInProgress);
assertTrue(isOneNotInProgress);
}
/**
* Try to make a request with a start transaction id which doesn't match the start ID of some log
* segment. This should fail as edit logs must currently be treated as indevisable units.
*/
@Test(expected = IOException.class)
public void testAskForTransactionsMidfile() throws IOException {
File f = new File(TestEditLog.TEST_DIR + "/filejournaltest2");
NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10);
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
FileJournalManager jm = new FileJournalManager(sd);
jm.getNumberOfTransactions(2);
}
/**
* Test that inprogress files are handled correct. Set up a single edits directory. Fail on after
* the last roll. Then verify that the logs have the expected number of transactions.
*/
@Test
public void testInprogressRecovery() throws IOException {
File f = new File(TestEditLog.TEST_DIR + "/filejournaltest0");
// abort after the 5th roll
NNStorage storage =
setupEdits(Collections.<URI>singletonList(f.toURI()), 5, new AbortSpec(5, 0));
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
FileJournalManager jm = new FileJournalManager(sd);
assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1));
}
/**
* Test the normal operation of loading transactions from file journal manager. 3 edits
* directories are setup without any failures. Test that we read in the expected number of
* transactions.
*/
@Test
public void testNormalOperation() throws IOException {
File f1 = new File(TestEditLog.TEST_DIR + "/normtest0");
File f2 = new File(TestEditLog.TEST_DIR + "/normtest1");
File f3 = new File(TestEditLog.TEST_DIR + "/normtest2");
List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());
NNStorage storage = setupEdits(editUris, 5);
long numJournals = 0;
for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
FileJournalManager jm = new FileJournalManager(sd);
assertEquals(6 * TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
numJournals++;
}
assertEquals(3, numJournals);
}
@Override
public void inspectDirectory(StorageDirectory sd) throws IOException {
// Was the directory just formatted?
if (!sd.getVersionFile().exists()) {
LOG.info("No version file in " + sd.getRoot());
needToSave |= true;
return;
}
maxSeenTxId = Math.max(maxSeenTxId, NNStorage.readTransactionIdFile(sd));
File currentDir = sd.getCurrentDir();
File filesInStorage[];
try {
filesInStorage = FileUtil.listFiles(currentDir);
} catch (IOException ioe) {
LOG.warn("Unable to inspect storage directory " + currentDir, ioe);
return;
}
for (File f : filesInStorage) {
LOG.info("Inspecting images: Checking file " + f);
String name = f.getName();
// Check for fsimage_*
Matcher imageMatch = IMAGE_REGEX.matcher(name);
addImageIfMatching(imageMatch, foundImages, sd, f);
// Check for fsimage_*
Matcher imageCkptMatch = IMAGE_CKPT_REGEX.matcher(name);
addImageIfMatching(imageCkptMatch, foundCkptImages, sd, f);
}
// Check for a seen_txid file, which marks a minimum transaction ID that
// must be included in our load plan.
try {
maxSeenTxId = Math.max(maxSeenTxId, NNStorage.readTransactionIdFile(sd));
} catch (IOException ioe) {
LOG.warn("Unable to determine the max transaction ID seen by " + sd, ioe);
}
// set finalized flag
isUpgradeFinalized = isUpgradeFinalized && !sd.getPreviousDir().exists();
}
/**
* Test that we receive the correct number of transactions when we count the number of
* transactions around gaps. Set up a single edits directory, with no failures. Delete the 4th
* logfile. Test that getNumberOfTransactions returns the correct number of transactions before
* this gap and after this gap. Also verify that if you try to count on the gap that an exception
* is thrown.
*/
@Test
public void testManyLogsWithGaps() throws IOException {
File f = new File(TestEditLog.TEST_DIR + "/filejournaltest3");
NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10);
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
final long startGapTxId = 3 * TXNS_PER_ROLL; // 30
final long endGapTxId = 4 * TXNS_PER_ROLL - 1; // 39
File[] files =
new File(f, "current")
.listFiles(
new FilenameFilter() {
public boolean accept(File dir, String name) {
if (name.startsWith(
NNStorage.getFinalizedEditsFileName(startGapTxId, endGapTxId))) {
return true;
}
return false;
}
});
assertEquals(1, files.length);
assertTrue(files[0].delete());
FileJournalManager jm = new FileJournalManager(sd);
assertEquals(startGapTxId, jm.getNumberOfTransactions(0));
try {
jm.getNumberOfTransactions(startGapTxId);
fail("Should have thrown an exception by now");
} catch (IOException ioe) {
assertTrue(true);
}
// rolled 10 times so there should be 11 files.
assertEquals(
11 * TXNS_PER_ROLL - endGapTxId - 1, // 110 - 39 - 1
jm.getNumberOfTransactions(endGapTxId + 1));
}
/**
* Test a mixture of inprogress files and finalised. Set up 3 edits directories and fail the
* second on the last roll. Verify that reading the transactions, reads from the finalised
* directories.
*/
@Test
public void testInprogressRecoveryMixed() throws IOException {
File f1 = new File(TestEditLog.TEST_DIR + "/mixtest0");
File f2 = new File(TestEditLog.TEST_DIR + "/mixtest1");
File f3 = new File(TestEditLog.TEST_DIR + "/mixtest2");
List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());
// abort after the 5th roll
NNStorage storage = setupEdits(editUris, 5, new AbortSpec(5, 1));
Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS);
StorageDirectory sd = dirs.next();
FileJournalManager jm = new FileJournalManager(sd);
assertEquals(6 * TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
sd = dirs.next();
jm = new FileJournalManager(sd);
assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1));
sd = dirs.next();
jm = new FileJournalManager(sd);
assertEquals(6 * TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
}