// stops the core, removes the transaction logs, restarts the core.
void deleteLogs() throws Exception {
String logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
h.close();
try {
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
for (String file : files) {
// new File(logDir, file).delete();
fs.delete(new Path(logDir, file), false);
}
assertEquals(0, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
} finally {
// make sure we create the core again, even if the assert fails so it won't mess
// up the next test.
createCore();
assertJQ(req("q", "*:*"), "/response/numFound=="); // ensure it works
}
}
@Test
public void testReplicationFactor() throws Exception {
clearIndex();
HdfsUpdateLog ulog = (HdfsUpdateLog) h.getCore().getUpdateHandler().getUpdateLog();
assertU(commit());
addAndGetVersion(sdoc("id", "REP1"), null);
assertU(commit());
String[] logList = ulog.getLogList(new Path(ulog.getLogDir()));
boolean foundRep2 = false;
for (String tl : logList) {
FileStatus status = fs.getFileStatus(new Path(ulog.getLogDir(), tl));
if (status.getReplication() == 2) {
foundRep2 = true;
break;
}
}
assertTrue("Expected to find tlogs with a replication factor of 2", foundRep2);
}
@Test
public void testTruncatedLog() throws Exception {
try {
DirectUpdateHandler2.commitOnClose = false;
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
UpdateLog.testing_logReplayHook =
new Runnable() {
@Override
public void run() {
try {
assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
UpdateLog.testing_logReplayFinishHook =
new Runnable() {
@Override
public void run() {
logReplayFinish.release();
}
};
String logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
clearIndex();
assertU(commit());
assertU(adoc("id", "F1"));
assertU(adoc("id", "F2"));
assertU(adoc("id", "F3"));
h.close();
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
Arrays.sort(files);
FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length - 1]));
dos.writeLong(0xffffffffffffffffL);
dos.writeChars(
"This should be appended to a good log file, representing a bad partially written record.");
dos.close();
logReplay.release(1000);
logReplayFinish.drainPermits();
ignoreException(
"OutOfBoundsException"); // this is what the corrupted log currently produces... subject
// to change.
createCore();
assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
resetExceptionIgnores();
assertJQ(req("q", "*:*"), "/response/numFound==3");
//
// Now test that the bad log file doesn't mess up retrieving latest versions
//
updateJ(
jsonAdd(sdoc("id", "F4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "F5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "F6", "_version_", "106")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
// This currently skips the bad log file and also returns the version of the clearIndex (del
// *:*)
// assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]");
assertJQ(req("qt", "/get", "getVersions", "3"), "/versions==[106,105,104]");
} finally {
DirectUpdateHandler2.commitOnClose = true;
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
}
}
@Test
public void testRemoveOldLogs() throws Exception {
try {
DirectUpdateHandler2.commitOnClose = false;
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
UpdateLog.testing_logReplayHook =
new Runnable() {
@Override
public void run() {
try {
assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
UpdateLog.testing_logReplayFinishHook =
new Runnable() {
@Override
public void run() {
logReplayFinish.release();
}
};
clearIndex();
assertU(commit());
String logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
h.close();
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
for (String file : files) {
fs.delete(new Path(logDir, file), false);
}
assertEquals(0, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
createCore();
int start = 0;
int maxReq = 50;
LinkedList<Long> versions = new LinkedList<>();
addDocs(10, start, versions);
start += 10;
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertU(commit());
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
addDocs(10, start, versions);
start += 10;
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertU(commit());
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertEquals(2, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
addDocs(105, start, versions);
start += 105;
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertU(commit());
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
// previous two logs should be gone now
assertEquals(1, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
addDocs(1, start, versions);
start += 1;
h.close();
createCore(); // trigger recovery, make sure that tlog reference handling is correct
// test we can get versions while replay is happening
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
logReplay.release(1000);
assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
addDocs(105, start, versions);
start += 105;
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertU(commit());
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
// previous logs should be gone now
assertEquals(1, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
//
// test that a corrupt tlog file doesn't stop us from coming up, or seeing versions before
// that tlog file.
//
addDocs(
1,
start,
new LinkedList<
Long>()); // don't add this to the versions list because we are going to lose it...
h.close();
files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
;
Arrays.sort(files);
FSDataOutputStream dos =
fs.create(new Path(new Path(logDir), files[files.length - 1]), (short) 1);
dos.writeUTF("This is a trashed log file that really shouldn't work at all, but we'll see..");
dos.close();
ignoreException("Failure to open existing");
createCore();
// we should still be able to get the list of versions (not including the trashed log file)
assertJQ(
req("qt", "/get", "getVersions", "" + maxReq),
"/versions==" + versions.subList(0, Math.min(maxReq, start)));
resetExceptionIgnores();
} finally {
DirectUpdateHandler2.commitOnClose = true;
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
}
}
// in rare circumstances, two logs can be left uncapped (lacking a commit at the end signifying
// that all the content in the log was committed)
@Test
public void testRecoveryMultipleLogs() throws Exception {
try {
DirectUpdateHandler2.commitOnClose = false;
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
UpdateLog.testing_logReplayHook =
new Runnable() {
@Override
public void run() {
try {
assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
UpdateLog.testing_logReplayFinishHook =
new Runnable() {
@Override
public void run() {
logReplayFinish.release();
}
};
String logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
clearIndex();
assertU(commit());
assertU(adoc("id", "AAAAAA"));
assertU(adoc("id", "BBBBBB"));
assertU(adoc("id", "CCCCCC"));
h.close();
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
Arrays.sort(files);
String fname = files[files.length - 1];
FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length - 1]));
dos.writeLong(0xffffffffffffffffL);
dos.writeChars(
"This should be appended to a good log file, representing a bad partially written record.");
dos.close();
FSDataInputStream dis = fs.open(new Path(logDir, files[files.length - 1]));
byte[] content = new byte[(int) dis.available()];
dis.readFully(content);
dis.close();
// Now make a newer log file with just the IDs changed. NOTE: this may not work if log format
// changes too much!
findReplace(
"AAAAAA".getBytes(StandardCharsets.UTF_8),
"aaaaaa".getBytes(StandardCharsets.UTF_8),
content);
findReplace(
"BBBBBB".getBytes(StandardCharsets.UTF_8),
"bbbbbb".getBytes(StandardCharsets.UTF_8),
content);
findReplace(
"CCCCCC".getBytes(StandardCharsets.UTF_8),
"cccccc".getBytes(StandardCharsets.UTF_8),
content);
// WARNING... assumes format of .00000n where n is less than 9
long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1));
String fname2 =
String.format(
Locale.ROOT, UpdateLog.LOG_FILENAME_PATTERN, UpdateLog.TLOG_NAME, logNumber + 1);
dos = fs.create(new Path(logDir, fname2), (short) 1);
dos.write(content);
dos.close();
logReplay.release(1000);
logReplayFinish.drainPermits();
ignoreException(
"OutOfBoundsException"); // this is what the corrupted log currently produces... subject
// to change.
createCore();
assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
resetExceptionIgnores();
assertJQ(req("q", "*:*"), "/response/numFound==6");
} finally {
DirectUpdateHandler2.commitOnClose = true;
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
}
}
//
// test that a corrupt tlog doesn't stop us from coming up
//
@Test
public void testCorruptLog() throws Exception {
try {
DirectUpdateHandler2.commitOnClose = false;
String logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
clearIndex();
assertU(commit());
assertU(adoc("id", "G1"));
assertU(adoc("id", "G2"));
assertU(adoc("id", "G3"));
h.close();
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
Arrays.sort(files);
FSDataOutputStream dos = fs.create(new Path(logDir, files[files.length - 1]), (short) 1);
dos.write(new byte[(int) 800]); // zero out file
dos.close();
ignoreException(
"Failure to open existing log file"); // this is what the corrupted log currently
// produces... subject to change.
createCore();
resetExceptionIgnores();
// just make sure it responds
assertJQ(req("q", "*:*"), "/response/numFound==0");
//
// Now test that the bad log file doesn't mess up retrieving latest versions
//
updateJ(
jsonAdd(sdoc("id", "G4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "G5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "G6", "_version_", "106")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
// This currently skips the bad log file and also returns the version of the clearIndex (del
// *:*)
// assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]");
assertJQ(req("qt", "/get", "getVersions", "3"), "/versions==[106,105,104]");
assertU(commit());
assertJQ(req("q", "*:*"), "/response/numFound==3");
// This messes up some other tests (on windows) if we don't remove the bad log.
// This *should* hopefully just be because the tests are too fragile and not because of real
// bugs - but it should be investigated further.
deleteLogs();
} finally {
DirectUpdateHandler2.commitOnClose = true;
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
}
}
