private void openEnvironment(File file) throws DatabaseInitException {
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setCachePercent(30);
envConfig.setAllowCreate(true);
envConfig.setTransactional(true);
envConfig.setTxnTimeout(10, TimeUnit.SECONDS);
envConfig.setLockTimeout(2000, TimeUnit.MILLISECONDS);
try {
environment = new Environment(file, envConfig);
transactionConfig = new TransactionConfig();
transactionConfig.setReadCommitted(true);
cursorConfig = new CursorConfig();
cursorConfig.setReadCommitted(true);
} catch (EnvironmentLockedException e) {
String message =
"Environment locked exception. Another process is using the same database, or the current user has no write access (database location: \""
+ file.getAbsolutePath()
+ "\")";
throw new DatabaseInitException(message);
} catch (DatabaseException e) {
String message = "A database initialisation error has occured (" + e.getMessage() + ")";
throw new DatabaseInitException(message);
}
}
static {
envConfig.setConfigParam(EnvironmentParams.ENV_CHECK_LEAKS.getName(), "false");
envConfig.setConfigParam(EnvironmentParams.NODE_MAX.getName(), "6");
envConfig.setTxnNoSync(Boolean.getBoolean(TestUtils.NO_SYNC));
envConfig.setLockTimeout(1); // to speed up intentional deadlocks
envConfig.setAllowCreate(true);
}
public BdbStorageConfiguration(VoldemortConfig config) {
this.voldemortConfig = config;
environmentConfig = new EnvironmentConfig();
environmentConfig.setTransactional(true);
environmentConfig.setCacheSize(config.getBdbCacheSize());
if (config.isBdbWriteTransactionsEnabled() && config.isBdbFlushTransactionsEnabled()) {
environmentConfig.setTxnNoSync(false);
environmentConfig.setTxnWriteNoSync(false);
} else if (config.isBdbWriteTransactionsEnabled() && !config.isBdbFlushTransactionsEnabled()) {
environmentConfig.setTxnNoSync(false);
environmentConfig.setTxnWriteNoSync(true);
} else {
environmentConfig.setTxnNoSync(true);
}
environmentConfig.setAllowCreate(true);
environmentConfig.setConfigParam(
EnvironmentConfig.LOG_FILE_MAX, Long.toString(config.getBdbMaxLogFileSize()));
environmentConfig.setConfigParam(
EnvironmentConfig.CHECKPOINTER_BYTES_INTERVAL,
Long.toString(config.getBdbCheckpointBytes()));
environmentConfig.setConfigParam(
EnvironmentConfig.CHECKPOINTER_WAKEUP_INTERVAL,
Long.toString(config.getBdbCheckpointMs() * Time.US_PER_MS));
environmentConfig.setConfigParam(
EnvironmentConfig.CLEANER_MIN_FILE_UTILIZATION,
Integer.toString(config.getBdbCleanerMinFileUtilization()));
environmentConfig.setConfigParam(
EnvironmentConfig.CLEANER_MIN_UTILIZATION,
Integer.toString(config.getBdbCleanerMinUtilization()));
environmentConfig.setConfigParam(
EnvironmentConfig.CLEANER_THREADS, Integer.toString(config.getBdbCleanerThreads()));
environmentConfig.setConfigParam(
EnvironmentConfig.CLEANER_LOOK_AHEAD_CACHE_SIZE,
Integer.toString(config.getBdbCleanerLookAheadCacheSize()));
environmentConfig.setConfigParam(
EnvironmentConfig.LOCK_N_LOCK_TABLES, Integer.toString(config.getBdbLockNLockTables()));
environmentConfig.setConfigParam(
EnvironmentConfig.ENV_FAIR_LATCHES, Boolean.toString(config.getBdbFairLatches()));
environmentConfig.setConfigParam(
EnvironmentConfig.CHECKPOINTER_HIGH_PRIORITY,
Boolean.toString(config.getBdbCheckpointerHighPriority()));
environmentConfig.setConfigParam(
EnvironmentConfig.CLEANER_MAX_BATCH_FILES,
Integer.toString(config.getBdbCleanerMaxBatchFiles()));
environmentConfig.setLockTimeout(config.getBdbLockTimeoutMs(), TimeUnit.MILLISECONDS);
databaseConfig = new DatabaseConfig();
databaseConfig.setAllowCreate(true);
databaseConfig.setSortedDuplicates(config.isBdbSortedDuplicatesEnabled());
databaseConfig.setNodeMaxEntries(config.getBdbBtreeFanout());
databaseConfig.setTransactional(true);
bdbMasterDir = config.getBdbDataDirectory();
useOneEnvPerStore = config.isBdbOneEnvPerStore();
if (useOneEnvPerStore) environmentConfig.setSharedCache(true);
}
/**
* Create a JE environment configuration with default values.
*
* @return A JE environment config containing default values.
*/
public static EnvironmentConfig defaultConfig() {
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
// "je.env.sharedLatches" is "true" by default since JE #12136 (3.3.62?)
// This parameter was set to false while diagnosing a Berkeley DB JE bug.
// Normally cleansed log files are deleted, but if this is set false
// they are instead renamed from .jdb to .del.
envConfig.setConfigParam(CLEANER_EXPUNGE, "true");
// Under heavy write load the check point can fall behind causing
// uncontrolled DB growth over time. This parameter makes the out of
// the box configuration more robust at the cost of a slight
// reduction in maximum write throughput. Experiments have shown
// that response time predictability is not impacted negatively.
envConfig.setConfigParam(CHECKPOINTER_HIGH_PRIORITY, "true");
// If the JVM is reasonably large then we can safely default to
// bigger read buffers. This will result in more scalable checkpointer
// and cleaner performance.
if (Runtime.getRuntime().maxMemory() > 256 * 1024 * 1024) {
envConfig.setConfigParam(CLEANER_LOOK_AHEAD_CACHE_SIZE, String.valueOf(2 * 1024 * 1024));
envConfig.setConfigParam(LOG_ITERATOR_READ_SIZE, String.valueOf(2 * 1024 * 1024));
envConfig.setConfigParam(LOG_FAULT_READ_SIZE, String.valueOf(4 * 1024));
}
// Disable lock timeouts, meaning that no lock wait
// timelimit is enforced and a deadlocked operation
// will block indefinitely.
envConfig.setLockTimeout(0, TimeUnit.MICROSECONDS);
return envConfig;
}
/**
* Rebuild index(es) in the backend instance. Note that the server will not explicitly initialize
* this backend before calling this method.
*
* @param rebuildConfig The rebuild configuration.
* @throws ConfigException If an unrecoverable problem arises during initialization.
* @throws InitializationException If a problem occurs during initialization that is not related
* to the server configuration.
* @throws DirectoryException If a Directory Server error occurs.
*/
public void rebuildBackend(RebuildConfig rebuildConfig)
throws InitializationException, ConfigException, DirectoryException {
// If the backend already has the root container open, we must use the same
// underlying root container
boolean openRootContainer = rootContainer == null;
/*
* If the rootContainer is open, the backend is initialized by something
* else. We can't do any rebuild of system indexes while others are using
* this backend.
*/
if (!openRootContainer && rebuildConfig.includesSystemIndex()) {
Message message = ERR_JEB_REBUILD_BACKEND_ONLINE.get();
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
}
try {
EnvironmentConfig envConfig;
if (openRootContainer) {
envConfig = new EnvironmentConfig();
envConfig.setAllowCreate(true);
envConfig.setTransactional(false);
envConfig.setDurability(Durability.COMMIT_NO_SYNC);
envConfig.setLockTimeout(0, TimeUnit.SECONDS);
envConfig.setTxnTimeout(0, TimeUnit.SECONDS);
envConfig.setConfigParam(
EnvironmentConfig.CLEANER_MIN_FILE_UTILIZATION,
String.valueOf(cfg.getDBCleanerMinUtilization()));
envConfig.setConfigParam(
EnvironmentConfig.LOG_FILE_MAX, String.valueOf(cfg.getDBLogFileMax()));
Importer importer = new Importer(rebuildConfig, cfg, envConfig);
rootContainer = initializeRootContainer(envConfig);
importer.rebuildIndexes(rootContainer);
} else {
envConfig = ConfigurableEnvironment.parseConfigEntry(cfg);
Importer importer = new Importer(rebuildConfig, cfg, envConfig);
importer.rebuildIndexes(rootContainer);
}
} catch (ExecutionException execEx) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, execEx);
}
Message message = ERR_EXECUTION_ERROR.get(execEx.getMessage());
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
} catch (InterruptedException intEx) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, intEx);
}
Message message = ERR_INTERRUPTED_ERROR.get(intEx.getMessage());
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
} catch (ConfigException ce) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, ce);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), ce.getMessageObject());
} catch (JebException e) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, e);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), e.getMessageObject());
} catch (InitializationException e) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, e);
}
throw new InitializationException(e.getMessageObject());
} finally {
// If a root container was opened in this method as read only, close it
// to leave the backend in the same state.
if (openRootContainer && rootContainer != null) {
try {
rootContainer.close();
rootContainer = null;
} catch (DatabaseException e) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, e);
}
}
}
}
}
/** {@inheritDoc} */
@Override()
public LDIFImportResult importLDIF(LDIFImportConfig importConfig) throws DirectoryException {
RuntimeInformation.logInfo();
// If the backend already has the root container open, we must use the same
// underlying root container
boolean openRootContainer = rootContainer == null;
// If the rootContainer is open, the backend is initialized by something
// else.
// We can't do import while the backend is online.
if (!openRootContainer) {
Message message = ERR_JEB_IMPORT_BACKEND_ONLINE.get();
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
}
try {
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setAllowCreate(true);
envConfig.setTransactional(false);
envConfig.setDurability(Durability.COMMIT_NO_SYNC);
envConfig.setLockTimeout(0, TimeUnit.SECONDS);
envConfig.setTxnTimeout(0, TimeUnit.SECONDS);
envConfig.setConfigParam(
EnvironmentConfig.CLEANER_MIN_FILE_UTILIZATION,
String.valueOf(cfg.getDBCleanerMinUtilization()));
envConfig.setConfigParam(
EnvironmentConfig.LOG_FILE_MAX, String.valueOf(cfg.getDBLogFileMax()));
if (!importConfig.appendToExistingData()) {
if (importConfig.clearBackend() || cfg.getBaseDN().size() <= 1) {
// We have the writer lock on the environment, now delete the
// environment and re-open it. Only do this when we are
// importing to all the base DNs in the backend or if the backend only
// have one base DN.
File parentDirectory = getFileForPath(cfg.getDBDirectory());
File backendDirectory = new File(parentDirectory, cfg.getBackendId());
// If the backend does not exist the import will create it.
if (backendDirectory.exists()) {
EnvManager.removeFiles(backendDirectory.getPath());
}
}
}
Importer importer = new Importer(importConfig, cfg, envConfig);
rootContainer = initializeRootContainer(envConfig);
return importer.processImport(rootContainer);
} catch (ExecutionException execEx) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, execEx);
}
if (execEx.getCause() instanceof DirectoryException) {
throw ((DirectoryException) execEx.getCause());
} else {
Message message = ERR_EXECUTION_ERROR.get(execEx.getMessage());
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
}
} catch (InterruptedException intEx) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, intEx);
}
Message message = ERR_INTERRUPTED_ERROR.get(intEx.getMessage());
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
} catch (JebException je) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, je);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), je.getMessageObject());
} catch (InitializationException ie) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, ie);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), ie.getMessageObject());
} catch (ConfigException ce) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, ce);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), ce.getMessageObject());
} finally {
// leave the backend in the same state.
try {
if (rootContainer != null) {
long startTime = System.currentTimeMillis();
rootContainer.close();
long finishTime = System.currentTimeMillis();
long closeTime = (finishTime - startTime) / 1000;
Message msg = NOTE_JEB_IMPORT_LDIF_ROOTCONTAINER_CLOSE.get(closeTime);
logError(msg);
rootContainer = null;
}
// Sync the environment to disk.
if (debugEnabled()) {
Message message = NOTE_JEB_IMPORT_CLOSING_DATABASE.get();
TRACER.debugInfo(message.toString());
}
} catch (DatabaseException de) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, de);
}
}
}
}
