예제 #1
0
  @Test(timeout = 60000)
  public void testForceGarbageCollection() throws Exception {
    baseConf.setGcWaitTime(60000);
    baseConf.setMinorCompactionInterval(120000);
    baseConf.setMajorCompactionInterval(240000);
    LedgerDirsManager dirManager =
        new LedgerDirsManager(baseConf, tmpDirs.toArray(new File[tmpDirs.size()]));
    CheckpointSource cp =
        new CheckpointSource() {
          @Override
          public Checkpoint newCheckpoint() {
            // Do nothing.
            return null;
          }

          @Override
          public void checkpointComplete(Checkpoint checkpoint, boolean compact)
              throws IOException {
            // Do nothing
          }
        };
    InterleavedLedgerStorage storage =
        new InterleavedLedgerStorage(
            baseConf,
            new FlatLedgerManager(baseConf, zkc),
            dirManager,
            dirManager,
            cp,
            NullStatsLogger.INSTANCE);
    storage.start();
    long startTime = MathUtils.now();
    Thread.sleep(2000);
    storage.gcThread.enableForceGC(false, false);
    // wait until current gc cycle is done
    while (storage.gcThread.forceGarbageCollection.get()) {
      Thread.sleep(100);
    }
    // Minor and Major compaction times should be larger than when we started
    // this test.
    assertTrue(
        "Minor or major compaction did not trigger even on forcing.",
        storage.gcThread.lastMajorCompactionTime > startTime
            && storage.gcThread.lastMinorCompactionTime > startTime);
    // disable compaction
    long lastMajorCompactionTime = storage.gcThread.lastMajorCompactionTime;
    long lastMinorCompactionTime = storage.gcThread.lastMinorCompactionTime;
    storage.gcThread.enableForceGC(true, true);
    // wait until current gc cycle is done
    while (storage.gcThread.forceGarbageCollection.get()) {
      Thread.sleep(100);
    }
    assertEquals(
        "Major compaction shouldn't be triggered",
        lastMajorCompactionTime,
        storage.gcThread.lastMajorCompactionTime);
    assertEquals(
        "Minor compaction shouldn't be triggered",
        lastMinorCompactionTime,
        storage.gcThread.lastMinorCompactionTime);
  }
예제 #2
0
  /**
   * Test that compaction doesnt add to index without having persisted entrylog first. This is
   * needed because compaction doesn't go through the journal. {@see
   * https://issues.apache.org/jira/browse/BOOKKEEPER-530} {@see
   * https://issues.apache.org/jira/browse/BOOKKEEPER-664}
   */
  @Test(timeout = 60000)
  public void testCompactionSafety() throws Exception {
    tearDown(); // I dont want the test infrastructure
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    final Set<Long> ledgers = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
    ActiveLedgerManager manager = getActiveLedgerManager(ledgers);

    File tmpDir = createTempDir("compaction", "compactionSafety");
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    conf.setLedgerDirNames(new String[] {tmpDir.toString()});

    conf.setEntryLogSizeLimit(EntryLogger.LOGFILE_HEADER_LENGTH + 3 * (4 + ENTRY_SIZE));
    conf.setGcWaitTime(100);
    conf.setMinorCompactionThreshold(0.7f);
    conf.setMajorCompactionThreshold(0.0f);
    conf.setMinorCompactionInterval(1);
    conf.setMajorCompactionInterval(10);
    conf.setPageLimit(1);

    CheckpointSource checkpointProgress =
        new CheckpointSource() {
          AtomicInteger idGen = new AtomicInteger(0);

          class MyCheckpoint implements Checkpoint {
            int id = idGen.incrementAndGet();

            @Override
            public int compareTo(Checkpoint o) {
              if (o == Checkpoint.MAX) {
                return -1;
              } else if (o == Checkpoint.MIN) {
                return 1;
              }
              return id - ((MyCheckpoint) o).id;
            }
          }

          @Override
          public Checkpoint newCheckpoint() {
            return new MyCheckpoint();
          }

          @Override
          public void checkpointComplete(Checkpoint checkpoint, boolean compact)
              throws IOException {}
        };

    final byte[] KEY = "foobar".getBytes();
    File log0 = new File(curDir, "0.log");
    File log1 = new File(curDir, "1.log");
    LedgerDirsManager dirs = new LedgerDirsManager(conf, conf.getLedgerDirs());
    assertFalse("Log 0 shouldnt exist", log0.exists());
    assertFalse("Log 1 shouldnt exist", log1.exists());
    InterleavedLedgerStorage storage =
        new InterleavedLedgerStorage(
            conf, manager, dirs, dirs, checkpointProgress, NullStatsLogger.INSTANCE);
    ledgers.add(1l);
    ledgers.add(2l);
    ledgers.add(3l);
    storage.setMasterKey(1, KEY);
    storage.setMasterKey(2, KEY);
    storage.setMasterKey(3, KEY);
    LOG.info("Write Ledger 1");
    storage.addEntry(genEntry(1, 1, ENTRY_SIZE));
    LOG.info("Write Ledger 2");
    storage.addEntry(genEntry(2, 1, ENTRY_SIZE));
    storage.addEntry(genEntry(2, 2, ENTRY_SIZE));
    LOG.info("Write ledger 3");
    storage.addEntry(genEntry(3, 2, ENTRY_SIZE));
    storage.flush();
    storage.shutdown();

    assertTrue("Log 0 should exist", log0.exists());
    assertTrue("Log 1 should exist", log1.exists());
    ledgers.remove(2l);
    ledgers.remove(3l);

    storage =
        new InterleavedLedgerStorage(
            conf, manager, dirs, dirs, checkpointProgress, NullStatsLogger.INSTANCE);
    storage.start();
    for (int i = 0; i < 10; i++) {
      if (!log0.exists() && !log1.exists()) {
        break;
      }
      Thread.sleep(1000);
      storage.entryLogger.flush(); // simulate sync thread
    }
    assertFalse("Log shouldnt exist", log0.exists());
    assertFalse("Log shouldnt exist", log1.exists());

    LOG.info("Write ledger 4");
    ledgers.add(4l);
    storage.setMasterKey(4, KEY);
    storage.addEntry(genEntry(4, 1, ENTRY_SIZE)); // force ledger 1 page to flush

    storage =
        new InterleavedLedgerStorage(
            conf, manager, dirs, dirs, checkpointProgress, NullStatsLogger.INSTANCE);
    storage.getEntry(1, 1); // entry should exist
  }