@Test(timeout = 60000) public void testForceGarbageCollection() throws Exception { baseConf.setGcWaitTime(60000); baseConf.setMinorCompactionInterval(120000); baseConf.setMajorCompactionInterval(240000); LedgerDirsManager dirManager = new LedgerDirsManager(baseConf, tmpDirs.toArray(new File[tmpDirs.size()])); CheckpointSource cp = new CheckpointSource() { @Override public Checkpoint newCheckpoint() { // Do nothing. return null; } @Override public void checkpointComplete(Checkpoint checkpoint, boolean compact) throws IOException { // Do nothing } }; InterleavedLedgerStorage storage = new InterleavedLedgerStorage( baseConf, new FlatLedgerManager(baseConf, zkc), dirManager, dirManager, cp, NullStatsLogger.INSTANCE); storage.start(); long startTime = MathUtils.now(); Thread.sleep(2000); storage.gcThread.enableForceGC(false, false); // wait until current gc cycle is done while (storage.gcThread.forceGarbageCollection.get()) { Thread.sleep(100); } // Minor and Major compaction times should be larger than when we started // this test. assertTrue( "Minor or major compaction did not trigger even on forcing.", storage.gcThread.lastMajorCompactionTime > startTime && storage.gcThread.lastMinorCompactionTime > startTime); // disable compaction long lastMajorCompactionTime = storage.gcThread.lastMajorCompactionTime; long lastMinorCompactionTime = storage.gcThread.lastMinorCompactionTime; storage.gcThread.enableForceGC(true, true); // wait until current gc cycle is done while (storage.gcThread.forceGarbageCollection.get()) { Thread.sleep(100); } assertEquals( "Major compaction shouldn't be triggered", lastMajorCompactionTime, storage.gcThread.lastMajorCompactionTime); assertEquals( "Minor compaction shouldn't be triggered", lastMinorCompactionTime, storage.gcThread.lastMinorCompactionTime); }
/** * Test that compaction doesnt add to index without having persisted entrylog first. This is * needed because compaction doesn't go through the journal. {@see * https://issues.apache.org/jira/browse/BOOKKEEPER-530} {@see * https://issues.apache.org/jira/browse/BOOKKEEPER-664} */ @Test(timeout = 60000) public void testCompactionSafety() throws Exception { tearDown(); // I dont want the test infrastructure ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); final Set<Long> ledgers = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>()); ActiveLedgerManager manager = getActiveLedgerManager(ledgers); File tmpDir = createTempDir("compaction", "compactionSafety"); File curDir = Bookie.getCurrentDirectory(tmpDir); Bookie.checkDirectoryStructure(curDir); conf.setLedgerDirNames(new String[] {tmpDir.toString()}); conf.setEntryLogSizeLimit(EntryLogger.LOGFILE_HEADER_LENGTH + 3 * (4 + ENTRY_SIZE)); conf.setGcWaitTime(100); conf.setMinorCompactionThreshold(0.7f); conf.setMajorCompactionThreshold(0.0f); conf.setMinorCompactionInterval(1); conf.setMajorCompactionInterval(10); conf.setPageLimit(1); CheckpointSource checkpointProgress = new CheckpointSource() { AtomicInteger idGen = new AtomicInteger(0); class MyCheckpoint implements Checkpoint { int id = idGen.incrementAndGet(); @Override public int compareTo(Checkpoint o) { if (o == Checkpoint.MAX) { return -1; } else if (o == Checkpoint.MIN) { return 1; } return id - ((MyCheckpoint) o).id; } } @Override public Checkpoint newCheckpoint() { return new MyCheckpoint(); } @Override public void checkpointComplete(Checkpoint checkpoint, boolean compact) throws IOException {} }; final byte[] KEY = "foobar".getBytes(); File log0 = new File(curDir, "0.log"); File log1 = new File(curDir, "1.log"); LedgerDirsManager dirs = new LedgerDirsManager(conf, conf.getLedgerDirs()); assertFalse("Log 0 shouldnt exist", log0.exists()); assertFalse("Log 1 shouldnt exist", log1.exists()); InterleavedLedgerStorage storage = new InterleavedLedgerStorage( conf, manager, dirs, dirs, checkpointProgress, NullStatsLogger.INSTANCE); ledgers.add(1l); ledgers.add(2l); ledgers.add(3l); storage.setMasterKey(1, KEY); storage.setMasterKey(2, KEY); storage.setMasterKey(3, KEY); LOG.info("Write Ledger 1"); storage.addEntry(genEntry(1, 1, ENTRY_SIZE)); LOG.info("Write Ledger 2"); storage.addEntry(genEntry(2, 1, ENTRY_SIZE)); storage.addEntry(genEntry(2, 2, ENTRY_SIZE)); LOG.info("Write ledger 3"); storage.addEntry(genEntry(3, 2, ENTRY_SIZE)); storage.flush(); storage.shutdown(); assertTrue("Log 0 should exist", log0.exists()); assertTrue("Log 1 should exist", log1.exists()); ledgers.remove(2l); ledgers.remove(3l); storage = new InterleavedLedgerStorage( conf, manager, dirs, dirs, checkpointProgress, NullStatsLogger.INSTANCE); storage.start(); for (int i = 0; i < 10; i++) { if (!log0.exists() && !log1.exists()) { break; } Thread.sleep(1000); storage.entryLogger.flush(); // simulate sync thread } assertFalse("Log shouldnt exist", log0.exists()); assertFalse("Log shouldnt exist", log1.exists()); LOG.info("Write ledger 4"); ledgers.add(4l); storage.setMasterKey(4, KEY); storage.addEntry(genEntry(4, 1, ENTRY_SIZE)); // force ledger 1 page to flush storage = new InterleavedLedgerStorage( conf, manager, dirs, dirs, checkpointProgress, NullStatsLogger.INSTANCE); storage.getEntry(1, 1); // entry should exist }