Esempio n. 1
0
  private void compactMemTableInternal() throws IOException {
    Preconditions.checkState(mutex.isHeldByCurrentThread());
    if (immutableMemTable == null) {
      return;
    }

    try {
      // Save the contents of the memtable as a new Table
      VersionEdit edit = new VersionEdit();
      Version base = versions.getCurrent();
      writeLevel0Table(immutableMemTable, edit, base);

      if (shuttingDown.get()) {
        throw new DatabaseShutdownException("Database shutdown during memtable compaction");
      }

      // Replace immutable memtable with the generated Table
      edit.setPreviousLogNumber(0);
      edit.setLogNumber(log.getFileNumber()); // Earlier logs no longer needed
      versions.logAndApply(edit);

      immutableMemTable = null;

      deleteObsoleteFiles();
    } finally {
      backgroundCondition.signalAll();
    }
  }
Esempio n. 2
0
  private void writeLevel0Table(MemTable mem, VersionEdit edit, Version base) throws IOException {
    Preconditions.checkState(mutex.isHeldByCurrentThread());

    // skip empty mem table
    if (mem.isEmpty()) {
      return;
    }

    // write the memtable to a new sstable
    long fileNumber = versions.getNextFileNumber();
    pendingOutputs.add(fileNumber);
    mutex.unlock();
    FileMetaData meta;
    try {
      meta = buildTable(mem, fileNumber);
    } finally {
      mutex.lock();
    }
    pendingOutputs.remove(fileNumber);

    // Note that if file size is zero, the file has been deleted and
    // should not be added to the manifest.
    int level = 0;
    if (meta != null && meta.getFileSize() > 0) {
      Slice minUserKey = meta.getSmallest().getUserKey();
      Slice maxUserKey = meta.getLargest().getUserKey();
      if (base != null) {
        level = base.pickLevelForMemTableOutput(minUserKey, maxUserKey);
      }
      edit.addFile(level, meta);
    }
  }
Esempio n. 3
0
  public DbImpl(Options options, File databaseDir) throws IOException {
    Preconditions.checkNotNull(options, "options is null");
    Preconditions.checkNotNull(databaseDir, "databaseDir is null");
    this.options = options;

    if (this.options.compressionType() == CompressionType.ZLIB && !Zlib.available()) {
      // There's little hope to continue.
      this.options.compressionType(CompressionType.NONE);
    }
    if (this.options.compressionType() == CompressionType.SNAPPY && !Snappy.available()) {
      // Disable snappy if it's not available.
      this.options.compressionType(CompressionType.NONE);
    }

    this.databaseDir = databaseDir;

    // use custom comparator if set
    DBComparator comparator = options.comparator();
    UserComparator userComparator;
    if (comparator != null) {
      userComparator = new CustomUserComparator(comparator);
    } else {
      userComparator = new BytewiseComparator();
    }
    internalKeyComparator = new InternalKeyComparator(userComparator);
    memTable = new MemTable(internalKeyComparator);
    immutableMemTable = null;

    ThreadFactory compactionThreadFactory =
        new ThreadFactoryBuilder()
            .setNameFormat("leveldb-compaction-%s")
            .setUncaughtExceptionHandler(
                new UncaughtExceptionHandler() {
                  @Override
                  public void uncaughtException(Thread t, Throwable e) {
                    // todo need a real UncaughtExceptionHandler
                    System.out.printf("%s%n", t);
                    e.printStackTrace();
                  }
                })
            .build();
    compactionExecutor = Executors.newSingleThreadExecutor(compactionThreadFactory);

    // Reserve ten files or so for other uses and give the rest to TableCache.
    int tableCacheSize = options.maxOpenFiles() - 10;
    tableCache =
        new TableCache(
            databaseDir,
            tableCacheSize,
            new InternalUserComparator(internalKeyComparator),
            options.verifyChecksums());

    // create the version set

    // create the database dir if it does not already exist
    databaseDir.mkdirs();
    Preconditions.checkArgument(
        databaseDir.exists(),
        "Database directory '%s' does not exist and could not be created",
        databaseDir);
    Preconditions.checkArgument(
        databaseDir.isDirectory(), "Database directory '%s' is not a directory", databaseDir);

    mutex.lock();
    try {
      // lock the database dir
      dbLock = new DbLock(new File(databaseDir, Filename.lockFileName()));

      // verify the "current" file
      File currentFile = new File(databaseDir, Filename.currentFileName());
      if (!currentFile.canRead()) {
        Preconditions.checkArgument(
            options.createIfMissing(),
            "Database '%s' does not exist and the create if missing option is disabled",
            databaseDir);
      } else {
        Preconditions.checkArgument(
            !options.errorIfExists(),
            "Database '%s' exists and the error if exists option is enabled",
            databaseDir);
      }

      versions = new VersionSet(databaseDir, tableCache, internalKeyComparator);

      // load  (and recover) current version
      versions.recover();

      // Recover from all newer log files than the ones named in the
      // descriptor (new log files may have been added by the previous
      // incarnation without registering them in the descriptor).
      //
      // Note that PrevLogNumber() is no longer used, but we pay
      // attention to it in case we are recovering a database
      // produced by an older version of leveldb.
      long minLogNumber = versions.getLogNumber();
      long previousLogNumber = versions.getPrevLogNumber();
      List<File> filenames = Filename.listFiles(databaseDir);

      List<Long> logs = Lists.newArrayList();
      for (File filename : filenames) {
        FileInfo fileInfo = Filename.parseFileName(filename);

        if (fileInfo != null
            && fileInfo.getFileType() == FileType.LOG
            && ((fileInfo.getFileNumber() >= minLogNumber)
                || (fileInfo.getFileNumber() == previousLogNumber))) {
          logs.add(fileInfo.getFileNumber());
        }
      }

      // Recover in the order in which the logs were generated
      VersionEdit edit = new VersionEdit();
      Collections.sort(logs);
      for (Long fileNumber : logs) {
        long maxSequence = recoverLogFile(fileNumber, edit);
        if (versions.getLastSequence() < maxSequence) {
          versions.setLastSequence(maxSequence);
        }
      }

      // open transaction log
      long logFileNumber = versions.getNextFileNumber();
      this.log =
          Logs.createLogWriter(
              new File(databaseDir, Filename.logFileName(logFileNumber)), logFileNumber);
      edit.setLogNumber(log.getFileNumber());

      // apply recovered edits
      versions.logAndApply(edit);

      // cleanup unused files
      deleteObsoleteFiles();

      // schedule compactions
      maybeScheduleCompaction();
    } finally {
      mutex.unlock();
    }
  }