@Override
protected void startInternal() throws Exception {
Path storeRoot = createStorageDir();
Options options = new Options();
options.createIfMissing(false);
options.logger(new LeveldbLogger());
LOG.info("Using state database at " + storeRoot + " for recovery");
File dbfile = new File(storeRoot.toString());
try {
db = JniDBFactory.factory.open(dbfile, options);
} catch (NativeDB.DBException e) {
if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
LOG.info("Creating state database at " + dbfile);
options.createIfMissing(true);
try {
db = JniDBFactory.factory.open(dbfile, options);
// store version
storeVersion();
} catch (DBException dbErr) {
throw new IOException(dbErr.getMessage(), dbErr);
}
} else {
throw e;
}
}
}
// Allows tests to have more control over when directories are cleaned up.
@VisibleForTesting
ExternalShuffleBlockResolver(
TransportConf conf, File registeredExecutorFile, Executor directoryCleaner)
throws IOException {
this.conf = conf;
this.registeredExecutorFile = registeredExecutorFile;
if (registeredExecutorFile != null) {
Options options = new Options();
options.createIfMissing(false);
options.logger(new LevelDBLogger());
DB tmpDb;
try {
tmpDb = JniDBFactory.factory.open(registeredExecutorFile, options);
} catch (NativeDB.DBException e) {
if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
logger.info("Creating state database at " + registeredExecutorFile);
options.createIfMissing(true);
try {
tmpDb = JniDBFactory.factory.open(registeredExecutorFile, options);
} catch (NativeDB.DBException dbExc) {
throw new IOException("Unable to create state store", dbExc);
}
} else {
// the leveldb file seems to be corrupt somehow. Lets just blow it away and create a new
// one, so we can keep processing new apps
logger.error(
"error opening leveldb file {}. Creating new file, will not be able to "
+ "recover state for existing applications",
registeredExecutorFile,
e);
if (registeredExecutorFile.isDirectory()) {
for (File f : registeredExecutorFile.listFiles()) {
if (!f.delete()) {
logger.warn("error deleting {}", f.getPath());
}
}
}
if (!registeredExecutorFile.delete()) {
logger.warn("error deleting {}", registeredExecutorFile.getPath());
}
options.createIfMissing(true);
try {
tmpDb = JniDBFactory.factory.open(registeredExecutorFile, options);
} catch (NativeDB.DBException dbExc) {
throw new IOException("Unable to create state store", dbExc);
}
}
}
// if there is a version mismatch, we throw an exception, which means the service is unusable
checkVersion(tmpDb);
executors = reloadRegisteredExecutors(tmpDb);
db = tmpDb;
} else {
db = null;
executors = Maps.newConcurrentMap();
}
this.directoryCleaner = directoryCleaner;
}
public DBase getDb(String name) throws IOException {
if (!name.matches("[a-zA-Z]{1,32}")) { // $NON-NLS-1$
throw new IllegalArgumentException("Invalid database name.");
}
Options options = new Options();
options.createIfMissing(true);
return new DBase(factory.open(new File(this.baseDir, "sc_" + name + ".db"), options), name);
}
public LevelDBPersistence(String root, Map options, boolean readOnly, boolean allowCreate)
throws IOException {
this.readOnly = readOnly;
new File(root).mkdirs();
dboptions = new Options();
dboptions.createIfMissing(allowCreate);
dboptions.paranoidChecks(true);
// 20mb TODO: set this using the options map
dboptions.cacheSize(20 * 1024 * 1024);
dboptions.compressionType(CompressionType.SNAPPY);
db = factory.open(new File(root), dboptions);
}
public DbImpl(Options options, File databaseDir) throws IOException {
Preconditions.checkNotNull(options, "options is null");
Preconditions.checkNotNull(databaseDir, "databaseDir is null");
this.options = options;
if (this.options.compressionType() == CompressionType.ZLIB && !Zlib.available()) {
// There's little hope to continue.
this.options.compressionType(CompressionType.NONE);
}
if (this.options.compressionType() == CompressionType.SNAPPY && !Snappy.available()) {
// Disable snappy if it's not available.
this.options.compressionType(CompressionType.NONE);
}
this.databaseDir = databaseDir;
// use custom comparator if set
DBComparator comparator = options.comparator();
UserComparator userComparator;
if (comparator != null) {
userComparator = new CustomUserComparator(comparator);
} else {
userComparator = new BytewiseComparator();
}
internalKeyComparator = new InternalKeyComparator(userComparator);
memTable = new MemTable(internalKeyComparator);
immutableMemTable = null;
ThreadFactory compactionThreadFactory =
new ThreadFactoryBuilder()
.setNameFormat("leveldb-compaction-%s")
.setUncaughtExceptionHandler(
new UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread t, Throwable e) {
// todo need a real UncaughtExceptionHandler
System.out.printf("%s%n", t);
e.printStackTrace();
}
})
.build();
compactionExecutor = Executors.newSingleThreadExecutor(compactionThreadFactory);
// Reserve ten files or so for other uses and give the rest to TableCache.
int tableCacheSize = options.maxOpenFiles() - 10;
tableCache =
new TableCache(
databaseDir,
tableCacheSize,
new InternalUserComparator(internalKeyComparator),
options.verifyChecksums());
// create the version set
// create the database dir if it does not already exist
databaseDir.mkdirs();
Preconditions.checkArgument(
databaseDir.exists(),
"Database directory '%s' does not exist and could not be created",
databaseDir);
Preconditions.checkArgument(
databaseDir.isDirectory(), "Database directory '%s' is not a directory", databaseDir);
mutex.lock();
try {
// lock the database dir
dbLock = new DbLock(new File(databaseDir, Filename.lockFileName()));
// verify the "current" file
File currentFile = new File(databaseDir, Filename.currentFileName());
if (!currentFile.canRead()) {
Preconditions.checkArgument(
options.createIfMissing(),
"Database '%s' does not exist and the create if missing option is disabled",
databaseDir);
} else {
Preconditions.checkArgument(
!options.errorIfExists(),
"Database '%s' exists and the error if exists option is enabled",
databaseDir);
}
versions = new VersionSet(databaseDir, tableCache, internalKeyComparator);
// load (and recover) current version
versions.recover();
// Recover from all newer log files than the ones named in the
// descriptor (new log files may have been added by the previous
// incarnation without registering them in the descriptor).
//
// Note that PrevLogNumber() is no longer used, but we pay
// attention to it in case we are recovering a database
// produced by an older version of leveldb.
long minLogNumber = versions.getLogNumber();
long previousLogNumber = versions.getPrevLogNumber();
List<File> filenames = Filename.listFiles(databaseDir);
List<Long> logs = Lists.newArrayList();
for (File filename : filenames) {
FileInfo fileInfo = Filename.parseFileName(filename);
if (fileInfo != null
&& fileInfo.getFileType() == FileType.LOG
&& ((fileInfo.getFileNumber() >= minLogNumber)
|| (fileInfo.getFileNumber() == previousLogNumber))) {
logs.add(fileInfo.getFileNumber());
}
}
// Recover in the order in which the logs were generated
VersionEdit edit = new VersionEdit();
Collections.sort(logs);
for (Long fileNumber : logs) {
long maxSequence = recoverLogFile(fileNumber, edit);
if (versions.getLastSequence() < maxSequence) {
versions.setLastSequence(maxSequence);
}
}
// open transaction log
long logFileNumber = versions.getNextFileNumber();
this.log =
Logs.createLogWriter(
new File(databaseDir, Filename.logFileName(logFileNumber)), logFileNumber);
edit.setLogNumber(log.getFileNumber());
// apply recovered edits
versions.logAndApply(edit);
// cleanup unused files
deleteObsoleteFiles();
// schedule compactions
maybeScheduleCompaction();
} finally {
mutex.unlock();
}
}
