Exemplo n.º 1
0
 @Override
 protected void startInternal() throws Exception {
   Path storeRoot = createStorageDir();
   Options options = new Options();
   options.createIfMissing(false);
   options.logger(new LeveldbLogger());
   LOG.info("Using state database at " + storeRoot + " for recovery");
   File dbfile = new File(storeRoot.toString());
   try {
     db = JniDBFactory.factory.open(dbfile, options);
   } catch (NativeDB.DBException e) {
     if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
       LOG.info("Creating state database at " + dbfile);
       options.createIfMissing(true);
       try {
         db = JniDBFactory.factory.open(dbfile, options);
         // store version
         storeVersion();
       } catch (DBException dbErr) {
         throw new IOException(dbErr.getMessage(), dbErr);
       }
     } else {
       throw e;
     }
   }
 }
 // Allows tests to have more control over when directories are cleaned up.
 @VisibleForTesting
 ExternalShuffleBlockResolver(
     TransportConf conf, File registeredExecutorFile, Executor directoryCleaner)
     throws IOException {
   this.conf = conf;
   this.registeredExecutorFile = registeredExecutorFile;
   if (registeredExecutorFile != null) {
     Options options = new Options();
     options.createIfMissing(false);
     options.logger(new LevelDBLogger());
     DB tmpDb;
     try {
       tmpDb = JniDBFactory.factory.open(registeredExecutorFile, options);
     } catch (NativeDB.DBException e) {
       if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
         logger.info("Creating state database at " + registeredExecutorFile);
         options.createIfMissing(true);
         try {
           tmpDb = JniDBFactory.factory.open(registeredExecutorFile, options);
         } catch (NativeDB.DBException dbExc) {
           throw new IOException("Unable to create state store", dbExc);
         }
       } else {
         // the leveldb file seems to be corrupt somehow.  Lets just blow it away and create a new
         // one, so we can keep processing new apps
         logger.error(
             "error opening leveldb file {}.  Creating new file, will not be able to "
                 + "recover state for existing applications",
             registeredExecutorFile,
             e);
         if (registeredExecutorFile.isDirectory()) {
           for (File f : registeredExecutorFile.listFiles()) {
             if (!f.delete()) {
               logger.warn("error deleting {}", f.getPath());
             }
           }
         }
         if (!registeredExecutorFile.delete()) {
           logger.warn("error deleting {}", registeredExecutorFile.getPath());
         }
         options.createIfMissing(true);
         try {
           tmpDb = JniDBFactory.factory.open(registeredExecutorFile, options);
         } catch (NativeDB.DBException dbExc) {
           throw new IOException("Unable to create state store", dbExc);
         }
       }
     }
     // if there is a version mismatch, we throw an exception, which means the service is unusable
     checkVersion(tmpDb);
     executors = reloadRegisteredExecutors(tmpDb);
     db = tmpDb;
   } else {
     db = null;
     executors = Maps.newConcurrentMap();
   }
   this.directoryCleaner = directoryCleaner;
 }
Exemplo n.º 3
0
 public DBase getDb(String name) throws IOException {
   if (!name.matches("[a-zA-Z]{1,32}")) { // $NON-NLS-1$
     throw new IllegalArgumentException("Invalid database name.");
   }
   Options options = new Options();
   options.createIfMissing(true);
   return new DBase(factory.open(new File(this.baseDir, "sc_" + name + ".db"), options), name);
 }
Exemplo n.º 4
0
    public LevelDBPersistence(String root, Map options, boolean readOnly, boolean allowCreate)
        throws IOException {
      this.readOnly = readOnly;
      new File(root).mkdirs();

      dboptions = new Options();
      dboptions.createIfMissing(allowCreate);
      dboptions.paranoidChecks(true);
      // 20mb TODO: set this using the options map
      dboptions.cacheSize(20 * 1024 * 1024);
      dboptions.compressionType(CompressionType.SNAPPY);

      db = factory.open(new File(root), dboptions);
    }
Exemplo n.º 5
0
  public DbImpl(Options options, File databaseDir) throws IOException {
    Preconditions.checkNotNull(options, "options is null");
    Preconditions.checkNotNull(databaseDir, "databaseDir is null");
    this.options = options;

    if (this.options.compressionType() == CompressionType.ZLIB && !Zlib.available()) {
      // There's little hope to continue.
      this.options.compressionType(CompressionType.NONE);
    }
    if (this.options.compressionType() == CompressionType.SNAPPY && !Snappy.available()) {
      // Disable snappy if it's not available.
      this.options.compressionType(CompressionType.NONE);
    }

    this.databaseDir = databaseDir;

    // use custom comparator if set
    DBComparator comparator = options.comparator();
    UserComparator userComparator;
    if (comparator != null) {
      userComparator = new CustomUserComparator(comparator);
    } else {
      userComparator = new BytewiseComparator();
    }
    internalKeyComparator = new InternalKeyComparator(userComparator);
    memTable = new MemTable(internalKeyComparator);
    immutableMemTable = null;

    ThreadFactory compactionThreadFactory =
        new ThreadFactoryBuilder()
            .setNameFormat("leveldb-compaction-%s")
            .setUncaughtExceptionHandler(
                new UncaughtExceptionHandler() {
                  @Override
                  public void uncaughtException(Thread t, Throwable e) {
                    // todo need a real UncaughtExceptionHandler
                    System.out.printf("%s%n", t);
                    e.printStackTrace();
                  }
                })
            .build();
    compactionExecutor = Executors.newSingleThreadExecutor(compactionThreadFactory);

    // Reserve ten files or so for other uses and give the rest to TableCache.
    int tableCacheSize = options.maxOpenFiles() - 10;
    tableCache =
        new TableCache(
            databaseDir,
            tableCacheSize,
            new InternalUserComparator(internalKeyComparator),
            options.verifyChecksums());

    // create the version set

    // create the database dir if it does not already exist
    databaseDir.mkdirs();
    Preconditions.checkArgument(
        databaseDir.exists(),
        "Database directory '%s' does not exist and could not be created",
        databaseDir);
    Preconditions.checkArgument(
        databaseDir.isDirectory(), "Database directory '%s' is not a directory", databaseDir);

    mutex.lock();
    try {
      // lock the database dir
      dbLock = new DbLock(new File(databaseDir, Filename.lockFileName()));

      // verify the "current" file
      File currentFile = new File(databaseDir, Filename.currentFileName());
      if (!currentFile.canRead()) {
        Preconditions.checkArgument(
            options.createIfMissing(),
            "Database '%s' does not exist and the create if missing option is disabled",
            databaseDir);
      } else {
        Preconditions.checkArgument(
            !options.errorIfExists(),
            "Database '%s' exists and the error if exists option is enabled",
            databaseDir);
      }

      versions = new VersionSet(databaseDir, tableCache, internalKeyComparator);

      // load  (and recover) current version
      versions.recover();

      // Recover from all newer log files than the ones named in the
      // descriptor (new log files may have been added by the previous
      // incarnation without registering them in the descriptor).
      //
      // Note that PrevLogNumber() is no longer used, but we pay
      // attention to it in case we are recovering a database
      // produced by an older version of leveldb.
      long minLogNumber = versions.getLogNumber();
      long previousLogNumber = versions.getPrevLogNumber();
      List<File> filenames = Filename.listFiles(databaseDir);

      List<Long> logs = Lists.newArrayList();
      for (File filename : filenames) {
        FileInfo fileInfo = Filename.parseFileName(filename);

        if (fileInfo != null
            && fileInfo.getFileType() == FileType.LOG
            && ((fileInfo.getFileNumber() >= minLogNumber)
                || (fileInfo.getFileNumber() == previousLogNumber))) {
          logs.add(fileInfo.getFileNumber());
        }
      }

      // Recover in the order in which the logs were generated
      VersionEdit edit = new VersionEdit();
      Collections.sort(logs);
      for (Long fileNumber : logs) {
        long maxSequence = recoverLogFile(fileNumber, edit);
        if (versions.getLastSequence() < maxSequence) {
          versions.setLastSequence(maxSequence);
        }
      }

      // open transaction log
      long logFileNumber = versions.getNextFileNumber();
      this.log =
          Logs.createLogWriter(
              new File(databaseDir, Filename.logFileName(logFileNumber)), logFileNumber);
      edit.setLogNumber(log.getFileNumber());

      // apply recovered edits
      versions.logAndApply(edit);

      // cleanup unused files
      deleteObsoleteFiles();

      // schedule compactions
      maybeScheduleCompaction();
    } finally {
      mutex.unlock();
    }
  }