@Test(timeout = 60000)
public void testForceGarbageCollection() throws Exception {
baseConf.setGcWaitTime(60000);
baseConf.setMinorCompactionInterval(120000);
baseConf.setMajorCompactionInterval(240000);
LedgerDirsManager dirManager =
new LedgerDirsManager(baseConf, tmpDirs.toArray(new File[tmpDirs.size()]));
CheckpointSource cp =
new CheckpointSource() {
@Override
public Checkpoint newCheckpoint() {
// Do nothing.
return null;
}
@Override
public void checkpointComplete(Checkpoint checkpoint, boolean compact)
throws IOException {
// Do nothing
}
};
InterleavedLedgerStorage storage =
new InterleavedLedgerStorage(
baseConf,
new FlatLedgerManager(baseConf, zkc),
dirManager,
dirManager,
cp,
NullStatsLogger.INSTANCE);
storage.start();
long startTime = MathUtils.now();
Thread.sleep(2000);
storage.gcThread.enableForceGC(false, false);
// wait until current gc cycle is done
while (storage.gcThread.forceGarbageCollection.get()) {
Thread.sleep(100);
}
// Minor and Major compaction times should be larger than when we started
// this test.
assertTrue(
"Minor or major compaction did not trigger even on forcing.",
storage.gcThread.lastMajorCompactionTime > startTime
&& storage.gcThread.lastMinorCompactionTime > startTime);
// disable compaction
long lastMajorCompactionTime = storage.gcThread.lastMajorCompactionTime;
long lastMinorCompactionTime = storage.gcThread.lastMinorCompactionTime;
storage.gcThread.enableForceGC(true, true);
// wait until current gc cycle is done
while (storage.gcThread.forceGarbageCollection.get()) {
Thread.sleep(100);
}
assertEquals(
"Major compaction shouldn't be triggered",
lastMajorCompactionTime,
storage.gcThread.lastMajorCompactionTime);
assertEquals(
"Minor compaction shouldn't be triggered",
lastMinorCompactionTime,
storage.gcThread.lastMinorCompactionTime);
}
/**
* Test that compaction doesnt add to index without having persisted entrylog first. This is
* needed because compaction doesn't go through the journal. {@see
* https://issues.apache.org/jira/browse/BOOKKEEPER-530} {@see
* https://issues.apache.org/jira/browse/BOOKKEEPER-664}
*/
@Test(timeout = 60000)
public void testCompactionSafety() throws Exception {
tearDown(); // I dont want the test infrastructure
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
final Set<Long> ledgers = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
ActiveLedgerManager manager = getActiveLedgerManager(ledgers);
File tmpDir = createTempDir("compaction", "compactionSafety");
File curDir = Bookie.getCurrentDirectory(tmpDir);
Bookie.checkDirectoryStructure(curDir);
conf.setLedgerDirNames(new String[] {tmpDir.toString()});
conf.setEntryLogSizeLimit(EntryLogger.LOGFILE_HEADER_LENGTH + 3 * (4 + ENTRY_SIZE));
conf.setGcWaitTime(100);
conf.setMinorCompactionThreshold(0.7f);
conf.setMajorCompactionThreshold(0.0f);
conf.setMinorCompactionInterval(1);
conf.setMajorCompactionInterval(10);
conf.setPageLimit(1);
CheckpointSource checkpointProgress =
new CheckpointSource() {
AtomicInteger idGen = new AtomicInteger(0);
class MyCheckpoint implements Checkpoint {
int id = idGen.incrementAndGet();
@Override
public int compareTo(Checkpoint o) {
if (o == Checkpoint.MAX) {
return -1;
} else if (o == Checkpoint.MIN) {
return 1;
}
return id - ((MyCheckpoint) o).id;
}
}
@Override
public Checkpoint newCheckpoint() {
return new MyCheckpoint();
}
@Override
public void checkpointComplete(Checkpoint checkpoint, boolean compact)
throws IOException {}
};
final byte[] KEY = "foobar".getBytes();
File log0 = new File(curDir, "0.log");
File log1 = new File(curDir, "1.log");
LedgerDirsManager dirs = new LedgerDirsManager(conf, conf.getLedgerDirs());
assertFalse("Log 0 shouldnt exist", log0.exists());
assertFalse("Log 1 shouldnt exist", log1.exists());
InterleavedLedgerStorage storage =
new InterleavedLedgerStorage(
conf, manager, dirs, dirs, checkpointProgress, NullStatsLogger.INSTANCE);
ledgers.add(1l);
ledgers.add(2l);
ledgers.add(3l);
storage.setMasterKey(1, KEY);
storage.setMasterKey(2, KEY);
storage.setMasterKey(3, KEY);
LOG.info("Write Ledger 1");
storage.addEntry(genEntry(1, 1, ENTRY_SIZE));
LOG.info("Write Ledger 2");
storage.addEntry(genEntry(2, 1, ENTRY_SIZE));
storage.addEntry(genEntry(2, 2, ENTRY_SIZE));
LOG.info("Write ledger 3");
storage.addEntry(genEntry(3, 2, ENTRY_SIZE));
storage.flush();
storage.shutdown();
assertTrue("Log 0 should exist", log0.exists());
assertTrue("Log 1 should exist", log1.exists());
ledgers.remove(2l);
ledgers.remove(3l);
storage =
new InterleavedLedgerStorage(
conf, manager, dirs, dirs, checkpointProgress, NullStatsLogger.INSTANCE);
storage.start();
for (int i = 0; i < 10; i++) {
if (!log0.exists() && !log1.exists()) {
break;
}
Thread.sleep(1000);
storage.entryLogger.flush(); // simulate sync thread
}
assertFalse("Log shouldnt exist", log0.exists());
assertFalse("Log shouldnt exist", log1.exists());
LOG.info("Write ledger 4");
ledgers.add(4l);
storage.setMasterKey(4, KEY);
storage.addEntry(genEntry(4, 1, ENTRY_SIZE)); // force ledger 1 page to flush
storage =
new InterleavedLedgerStorage(
conf, manager, dirs, dirs, checkpointProgress, NullStatsLogger.INSTANCE);
storage.getEntry(1, 1); // entry should exist
}
