public static VoldemortConfig loadFromVoldemortHome(
String voldemortHome, String voldemortConfigDir) {
if (!Utils.isReadableDir(voldemortHome))
throw new ConfigurationException(
"Attempt to load configuration from VOLDEMORT_HOME, "
+ voldemortHome
+ " failed. That is not a readable directory.");
if (voldemortConfigDir == null) {
voldemortConfigDir = voldemortHome + File.separator + "config";
}
String propertiesFile = voldemortConfigDir + File.separator + "server.properties";
if (!Utils.isReadableFile(propertiesFile))
throw new ConfigurationException(propertiesFile + " is not a readable configuration file.");
Props properties = null;
try {
properties = new Props(new File(propertiesFile));
properties.put("voldemort.home", voldemortHome);
properties.put("metadata.directory", voldemortConfigDir);
} catch (IOException e) {
throw new ConfigurationException(e);
}
return new VoldemortConfig(properties);
}
/**
* Tests that any reservation that will not violate minimum shared cache will fail, during server
* startup and dynamic updation
*/
public void testMinimumSharedCache() {
int totalCache = 20 * ByteUtils.BYTES_PER_MB; // total cache size
int shareA = 10 * ByteUtils.BYTES_PER_MB; // A reserves 10MB
// lets use all the default values.
Props props = new Props();
props.put("node.id", 1);
props.put("voldemort.home", "test/common/voldemort/config");
VoldemortConfig voldemortConfig = new VoldemortConfig(props);
voldemortConfig.setBdbCacheSize(totalCache);
voldemortConfig.setBdbOneEnvPerStore(true);
voldemortConfig.setBdbDataDirectory(bdbMasterDir.toURI().getPath());
voldemortConfig.setBdbMinimumSharedCache(15 * ByteUtils.BYTES_PER_MB);
BdbStorageEngine storeA = null;
bdbStorage = new BdbStorageConfiguration(voldemortConfig);
assertEquals(0, bdbStorage.getReservedCacheSize());
try {
StoreDefinition defA =
TestUtils.makeStoreDefinition("storeA", shareA / ByteUtils.BYTES_PER_MB);
storeA = (BdbStorageEngine) bdbStorage.getStore(defA);
fail("Should have thrown exception since minSharedCache will be violated");
} catch (StorageInitializationException sie) {
// should come here.
}
// failing operations should not alter reserved cache size
assertEquals(0, bdbStorage.getReservedCacheSize());
voldemortConfig.setBdbMinimumSharedCache(10 * ByteUtils.BYTES_PER_MB);
bdbStorage = new BdbStorageConfiguration(voldemortConfig);
try {
StoreDefinition defA =
TestUtils.makeStoreDefinition("storeA", shareA / ByteUtils.BYTES_PER_MB);
storeA = (BdbStorageEngine) bdbStorage.getStore(defA);
} catch (StorageInitializationException sie) {
// should not come here.
fail("minSharedCache should n't have been violated");
}
assertEquals(shareA, bdbStorage.getReservedCacheSize());
long reserveCacheSize = bdbStorage.getReservedCacheSize();
// now, try increasing the reservation dynamically and it should fail
try {
StoreDefinition defA = TestUtils.makeStoreDefinition("storeA", 15);
bdbStorage.update(defA);
fail("Should have thrown exception since minSharedCache will be violated");
} catch (StorageInitializationException sie) {
// should come here.
}
// this failure cannot alter the reservedCacheSize
assertEquals(reserveCacheSize, bdbStorage.getReservedCacheSize());
if (storeA != null) storeA.close();
}
private void setProperties(Properties properties) {
Props props = new Props(properties);
if (props.containsKey(MAX_CONNECTIONS_PER_NODE_PROPERTY))
this.setMaxConnectionsPerNode(props.getInt(MAX_CONNECTIONS_PER_NODE_PROPERTY));
if (props.containsKey(MAX_TOTAL_CONNECTIONS_PROPERTY))
this.setMaxTotalConnections(props.getInt(MAX_TOTAL_CONNECTIONS_PROPERTY));
if (props.containsKey(MAX_THREADS_PROPERTY))
this.setMaxThreads(props.getInt(MAX_THREADS_PROPERTY));
if (props.containsKey(MAX_QUEUED_REQUESTS_PROPERTY))
this.setMaxQueuedRequests(props.getInt(MAX_QUEUED_REQUESTS_PROPERTY));
if (props.containsKey(THREAD_IDLE_MS_PROPERTY))
this.setThreadIdleTime(props.getLong(THREAD_IDLE_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(CONNECTION_TIMEOUT_MS_PROPERTY))
this.setConnectionTimeout(
props.getInt(CONNECTION_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(SOCKET_TIMEOUT_MS_PROPERTY))
this.setSocketTimeout(props.getInt(SOCKET_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(SOCKET_KEEPALIVE_PROPERTY))
this.setSocketKeepAlive(props.getBoolean(SOCKET_KEEPALIVE_PROPERTY));
if (props.containsKey(SELECTORS_PROPERTY)) this.setSelectors(props.getInt(SELECTORS_PROPERTY));
if (props.containsKey(ROUTING_TIMEOUT_MS_PROPERTY))
this.setRoutingTimeout(props.getInt(ROUTING_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
// By default, make all the timeouts equal to routing timeout
timeoutConfig = new TimeoutConfig(routingTimeoutMs, false);
if (props.containsKey(GETALL_ROUTING_TIMEOUT_MS_PROPERTY))
timeoutConfig.setOperationTimeout(
VoldemortOpCode.GET_ALL_OP_CODE, props.getInt(GETALL_ROUTING_TIMEOUT_MS_PROPERTY));
if (props.containsKey(GET_ROUTING_TIMEOUT_MS_PROPERTY))
timeoutConfig.setOperationTimeout(
VoldemortOpCode.GET_OP_CODE, props.getInt(GET_ROUTING_TIMEOUT_MS_PROPERTY));
if (props.containsKey(PUT_ROUTING_TIMEOUT_MS_PROPERTY)) {
long putTimeoutMs = props.getInt(PUT_ROUTING_TIMEOUT_MS_PROPERTY);
timeoutConfig.setOperationTimeout(VoldemortOpCode.PUT_OP_CODE, putTimeoutMs);
// By default, use the same thing for getVersions() also
timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, putTimeoutMs);
}
// of course, if someone overrides it, we will respect that
if (props.containsKey(GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY))
timeoutConfig.setOperationTimeout(
VoldemortOpCode.GET_VERSION_OP_CODE,
props.getInt(GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY));
if (props.containsKey(DELETE_ROUTING_TIMEOUT_MS_PROPERTY))
timeoutConfig.setOperationTimeout(
VoldemortOpCode.DELETE_OP_CODE, props.getInt(DELETE_ROUTING_TIMEOUT_MS_PROPERTY));
if (props.containsKey(ALLOW_PARTIAL_GETALLS_PROPERTY))
timeoutConfig.setPartialGetAllAllowed(props.getBoolean(ALLOW_PARTIAL_GETALLS_PROPERTY));
if (props.containsKey(SOCKET_BUFFER_SIZE_PROPERTY))
this.setSocketBufferSize(props.getInt(SOCKET_BUFFER_SIZE_PROPERTY));
if (props.containsKey(SERIALIZER_FACTORY_CLASS_PROPERTY)) {
Class<?> factoryClass =
ReflectUtils.loadClass(props.getString(SERIALIZER_FACTORY_CLASS_PROPERTY));
SerializerFactory factory =
(SerializerFactory) ReflectUtils.callConstructor(factoryClass, new Object[] {});
this.setSerializerFactory(factory);
}
if (props.containsKey(BOOTSTRAP_URLS_PROPERTY))
this.setBootstrapUrls(props.getList(BOOTSTRAP_URLS_PROPERTY));
if (props.containsKey(REQUEST_FORMAT_PROPERTY))
this.setRequestFormatType(
RequestFormatType.fromCode(props.getString(REQUEST_FORMAT_PROPERTY)));
if (props.containsKey(ENABLE_JMX_PROPERTY))
this.setEnableJmx(props.getBoolean(ENABLE_JMX_PROPERTY));
if (props.containsKey(ENABLE_LAZY_PROPERTY))
this.setEnableLazy(props.getBoolean(ENABLE_LAZY_PROPERTY));
if (props.containsKey(CLIENT_ZONE_ID)) this.setClientZoneId(props.getInt(CLIENT_ZONE_ID));
if (props.containsKey(CACHE_STORE_CLIENTS)) {
this.setCacheStoreClients(props.getBoolean(CACHE_STORE_CLIENTS));
}
if (props.containsKey(USE_DEFAULT_CLIENT))
this.enableDefaultClient(props.getBoolean(USE_DEFAULT_CLIENT));
if (props.containsKey(FAILUREDETECTOR_IMPLEMENTATION_PROPERTY))
this.setFailureDetectorImplementation(
props.getString(FAILUREDETECTOR_IMPLEMENTATION_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_MAX_TOLERABLE_FATALITIES_PROPERTY))
this.setMaximumTolerableFatalFailures(
props.getInt(FAILUREDETECTOR_MAX_TOLERABLE_FATALITIES_PROPERTY));
// We're changing the property from "node_bannage_ms" to
// "failuredetector_bannage_period" so if we have the old one, migrate
// it over.
if (props.containsKey(NODE_BANNAGE_MS_PROPERTY)
&& !props.containsKey(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY)) {
props.put(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY, props.get(NODE_BANNAGE_MS_PROPERTY));
}
if (props.containsKey(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY))
this.setFailureDetectorBannagePeriod(props.getLong(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_THRESHOLD_PROPERTY))
this.setFailureDetectorThreshold(props.getInt(FAILUREDETECTOR_THRESHOLD_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM_PROPERTY))
this.setFailureDetectorThresholdCountMinimum(
props.getInt(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_THRESHOLD_INTERVAL_PROPERTY))
this.setFailureDetectorThresholdInterval(
props.getLong(FAILUREDETECTOR_THRESHOLD_INTERVAL_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL_PROPERTY))
this.setFailureDetectorAsyncRecoveryInterval(
props.getLong(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES_PROPERTY))
this.setFailureDetectorCatastrophicErrorTypes(
props.getList(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD_PROPERTY))
this.setFailureDetectorRequestLengthThreshold(
props.getLong(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD_PROPERTY));
else this.setFailureDetectorRequestLengthThreshold(getSocketTimeout(TimeUnit.MILLISECONDS));
if (props.containsKey(MAX_BOOTSTRAP_RETRIES))
this.setMaxBootstrapRetries(props.getInt(MAX_BOOTSTRAP_RETRIES));
if (props.containsKey(CLIENT_CONTEXT_NAME)) {
this.setClientContextName(props.getString(CLIENT_CONTEXT_NAME));
}
if (props.containsKey(ASYNC_CHECK_METADATA_INTERVAL)) {
this.setAsyncMetadataRefreshInMs(props.getLong(ASYNC_CHECK_METADATA_INTERVAL));
}
if (props.containsKey(CLIENT_REGISTRY_REFRESH_INTERVAL)) {
this.setClientRegistryUpdateIntervalInSecs(props.getInt(CLIENT_REGISTRY_REFRESH_INTERVAL));
}
if (props.containsKey(ASYNC_JOB_THREAD_POOL_SIZE)) {
this.setAsyncJobThreadPoolSize(props.getInt(ASYNC_JOB_THREAD_POOL_SIZE));
}
/* Check for system store paramaters if any */
if (props.containsKey(SYS_MAX_CONNECTIONS_PER_NODE)) {
this.setSysMaxConnectionsPerNode(props.getInt(SYS_MAX_CONNECTIONS_PER_NODE));
}
if (props.containsKey(SYS_ROUTING_TIMEOUT_MS)) {
this.setSysRoutingTimeout(props.getInt(SYS_ROUTING_TIMEOUT_MS));
}
if (props.containsKey(SYS_SOCKET_TIMEOUT_MS)) {
this.setSysSocketTimeout(props.getInt(SYS_SOCKET_TIMEOUT_MS));
}
if (props.containsKey(SYS_CONNECTION_TIMEOUT_MS)) {
this.setSysConnectionTimeout(props.getInt(SYS_CONNECTION_TIMEOUT_MS));
}
if (props.containsKey(SYS_ENABLE_JMX)) {
this.setSysEnableJmx(props.getBoolean(SYS_ENABLE_JMX));
}
if (props.containsKey(ENABLE_COMPRESSION_LAYER)) {
this.setEnableCompressionLayer(props.getBoolean(ENABLE_COMPRESSION_LAYER));
}
if (props.containsKey(ENABLE_SERIALIZATION_LAYER)) {
this.setEnableSerializationLayer(props.getBoolean(ENABLE_SERIALIZATION_LAYER));
}
if (props.containsKey(ENABLE_INCONSISTENCY_RESOLVING_LAYER)) {
this.setEnableInconsistencyResolvingLayer(
props.getBoolean(ENABLE_INCONSISTENCY_RESOLVING_LAYER));
}
if (props.containsKey(FAT_CLIENT_WRAPPER_CORE_POOL_SIZE_PROPERTY)) {
this.setFatClientWrapperCorePoolSize(
props.getInt(
FAT_CLIENT_WRAPPER_CORE_POOL_SIZE_PROPERTY, this.fatClientWrapperCorePoolSize));
}
if (props.containsKey(FAT_CLIENT_WRAPPER_MAX_POOL_SIZE_PROPERTY)) {
this.setFatClientWrapperMaxPoolSize(
props.getInt(
FAT_CLIENT_WRAPPER_MAX_POOL_SIZE_PROPERTY, this.fatClientWrapperMaxPoolSize));
}
if (props.containsKey(FAT_CLIENT_WRAPPER_POOL_KEEPALIVE_IN_SECS)) {
this.setFatClientWrapperKeepAliveInSecs(
props.getInt(
FAT_CLIENT_WRAPPER_POOL_KEEPALIVE_IN_SECS, this.fatClientWrapperKeepAliveInSecs));
}
if (props.containsKey(GET_OP_ZONE_AFFINITY)) {
this.getZoneAffinity().setEnableGetOpZoneAffinity(props.getBoolean(GET_OP_ZONE_AFFINITY));
}
if (props.containsKey(GETALL_OP_ZONE_AFFINITY)) {
this.getZoneAffinity()
.setEnableGetAllOpZoneAffinity(props.getBoolean(GETALL_OP_ZONE_AFFINITY));
}
if (props.containsKey(GETVERSIONS_OP_ZONE_AFFINITY)) {
this.getZoneAffinity()
.setEnableGetVersionsOpZoneAffinity(props.getBoolean(GETVERSIONS_OP_ZONE_AFFINITY));
}
}
/**
* Initiate the client config from a set of properties. This is useful for wiring from Spring or
* for externalizing client properties to a properties file
*
* @param properties The properties to use
*/
public ClientConfig(Properties properties) {
Props props = new Props(properties);
if (props.containsKey(MAX_CONNECTIONS_PER_NODE_PROPERTY))
this.setMaxConnectionsPerNode(props.getInt(MAX_CONNECTIONS_PER_NODE_PROPERTY));
if (props.containsKey(MAX_TOTAL_CONNECTIONS_PROPERTY))
this.setMaxTotalConnections(props.getInt(MAX_TOTAL_CONNECTIONS_PROPERTY));
if (props.containsKey(MAX_THREADS_PROPERTY))
this.setMaxThreads(props.getInt(MAX_THREADS_PROPERTY));
if (props.containsKey(MAX_QUEUED_REQUESTS_PROPERTY))
this.setMaxQueuedRequests(props.getInt(MAX_QUEUED_REQUESTS_PROPERTY));
if (props.containsKey(THREAD_IDLE_MS_PROPERTY))
this.setThreadIdleTime(props.getLong(THREAD_IDLE_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(CONNECTION_TIMEOUT_MS_PROPERTY))
this.setConnectionTimeout(
props.getInt(CONNECTION_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(SOCKET_TIMEOUT_MS_PROPERTY))
this.setSocketTimeout(props.getInt(SOCKET_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(ROUTING_TIMEOUT_MS_PROPERTY))
this.setRoutingTimeout(props.getInt(ROUTING_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(NODE_BANNAGE_MS_PROPERTY))
this.setNodeBannagePeriod(props.getInt(NODE_BANNAGE_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(SOCKET_BUFFER_SIZE_PROPERTY))
this.setSocketBufferSize(props.getInt(SOCKET_BUFFER_SIZE_PROPERTY));
if (props.containsKey(SERIALIZER_FACTORY_CLASS_PROPERTY)) {
Class<?> factoryClass =
ReflectUtils.loadClass(props.getString(SERIALIZER_FACTORY_CLASS_PROPERTY));
SerializerFactory factory =
(SerializerFactory) ReflectUtils.callConstructor(factoryClass, new Object[] {});
this.setSerializerFactory(factory);
}
if (props.containsKey(BOOTSTRAP_URLS_PROPERTY))
this.setBootstrapUrls(props.getList(BOOTSTRAP_URLS_PROPERTY));
if (props.containsKey(REQUEST_FORMAT_PROPERTY))
this.setRequestFormatType(
RequestFormatType.fromCode(props.getString(REQUEST_FORMAT_PROPERTY)));
if (props.containsKey(ENABLE_JMX_PROPERTY))
this.setEnableJmx(props.getBoolean(ENABLE_JMX_PROPERTY));
if (props.containsKey(MAX_BOOTSTRAP_RETRIES))
this.setMaxBootstrapRetries(props.getInt(MAX_BOOTSTRAP_RETRIES));
}
public RebalanceClientConfig(Properties properties) {
super(properties);
Props props = new Props(properties);
if (props.containsKey(MaxParallelRebalancingString))
this.setMaxParallelRebalancing(props.getInt(MaxParallelRebalancingString));
if (props.containsKey(RebalancingClientTimeoutSeconds))
this.setRebalancingClientTimeoutSeconds(props.getInt(RebalancingClientTimeoutSeconds));
if (props.containsKey(EnableDeleteAfterRebalancing))
this.setDeleteAfterRebalancingEnabled(props.getBoolean(EnableDeleteAfterRebalancing));
if (props.containsKey(MaxTriesRebalancingString))
this.setMaxTriesRebalancing(props.getInt(MaxTriesRebalancingString));
if (props.containsKey(OutputDirectoryString))
this.setOutputDirectory(props.getString(OutputDirectoryString));
if (props.containsKey(PrimaryPartitionBatchSizeString))
this.setPrimaryPartitionBatchSize(props.getInt(PrimaryPartitionBatchSizeString));
if (props.containsKey(StealerBasedRebalancingString))
this.setStealerBasedRebalancing(props.getBoolean(StealerBasedRebalancingString));
}
public VoldemortConfig(Props props) {
try {
this.nodeId = props.getInt("node.id");
} catch (UndefinedPropertyException e) {
this.nodeId = getIntEnvVariable(VOLDEMORT_NODE_ID_VAR_NAME);
}
this.voldemortHome = props.getString("voldemort.home");
this.dataDirectory =
props.getString("data.directory", this.voldemortHome + File.separator + "data");
this.metadataDirectory =
props.getString("metadata.directory", voldemortHome + File.separator + "config");
this.bdbCacheSize = props.getBytes("bdb.cache.size", 200 * 1024 * 1024);
this.bdbWriteTransactions = props.getBoolean("bdb.write.transactions", false);
this.bdbFlushTransactions = props.getBoolean("bdb.flush.transactions", false);
this.bdbDataDirectory =
props.getString("bdb.data.directory", this.dataDirectory + File.separator + "bdb");
this.bdbMaxLogFileSize = props.getBytes("bdb.max.logfile.size", 60 * 1024 * 1024);
this.bdbBtreeFanout = props.getInt("bdb.btree.fanout", 512);
this.bdbCheckpointBytes = props.getLong("bdb.checkpoint.interval.bytes", 20 * 1024 * 1024);
this.bdbCheckpointMs = props.getLong("bdb.checkpoint.interval.ms", 30 * Time.MS_PER_SECOND);
this.bdbSortedDuplicates = props.getBoolean("bdb.enable.sorted.duplicates", true);
this.bdbOneEnvPerStore = props.getBoolean("bdb.one.env.per.store", false);
this.bdbCleanerMinFileUtilization = props.getInt("bdb.cleaner.min.file.utilization", 5);
this.bdbCleanerMinUtilization = props.getInt("bdb.cleaner.minUtilization", 50);
this.bdbCleanerThreads = props.getInt("bdb.cleaner.threads", 1);
this.bdbCleanerLookAheadCacheSize = props.getInt("bdb.cleaner.lookahead.cache.size", 8192);
this.bdbLockTimeoutMs = props.getLong("bdb.lock.timeout.ms", 500);
this.bdbLockNLockTables = props.getInt("bdb.lock.nLockTables", 1);
this.bdbLogFaultReadSize = props.getInt("bdb.log.fault.read.size", 2048);
this.bdbLogIteratorReadSize = props.getInt("bdb.log.iterator.read.size", 8192);
this.bdbFairLatches = props.getBoolean("bdb.fair.latches", false);
this.bdbCheckpointerHighPriority = props.getBoolean("bdb.checkpointer.high.priority", false);
this.bdbCleanerMaxBatchFiles = props.getInt("bdb.cleaner.max.batch.files", 0);
this.bdbReadUncommitted = props.getBoolean("bdb.lock.read_uncommitted", true);
this.bdbStatsCacheTtlMs = props.getLong("bdb.stats.cache.ttl.ms", 5 * Time.MS_PER_SECOND);
this.readOnlyBackups = props.getInt("readonly.backups", 1);
this.readOnlySearchStrategy =
props.getString("readonly.search.strategy", BinarySearchStrategy.class.getName());
this.readOnlyStorageDir =
props.getString(
"readonly.data.directory", this.dataDirectory + File.separator + "read-only");
this.readOnlyDeleteBackupTimeMs = props.getInt("readonly.delete.backup.ms", 0);
this.maxBytesPerSecond = props.getBytes("fetcher.max.bytes.per.sec", 0);
this.minBytesPerSecond = props.getBytes("fetcher.min.bytes.per.sec", 0);
this.reportingIntervalBytes =
props.getBytes("fetcher.reporting.interval.bytes", REPORTING_INTERVAL_BYTES);
this.fetcherBufferSize = (int) props.getBytes("hdfs.fetcher.buffer.size", DEFAULT_BUFFER_SIZE);
this.mysqlUsername = props.getString("mysql.user", "root");
this.mysqlPassword = props.getString("mysql.password", "");
this.mysqlHost = props.getString("mysql.host", "localhost");
this.mysqlPort = props.getInt("mysql.port", 3306);
this.mysqlDatabaseName = props.getString("mysql.database", "voldemort");
this.maxThreads = props.getInt("max.threads", 100);
this.coreThreads = props.getInt("core.threads", Math.max(1, maxThreads / 2));
// Admin client should have less threads but very high buffer size.
this.adminMaxThreads = props.getInt("admin.max.threads", 20);
this.adminCoreThreads = props.getInt("admin.core.threads", Math.max(1, adminMaxThreads / 2));
this.adminStreamBufferSize =
(int) props.getBytes("admin.streams.buffer.size", 10 * 1000 * 1000);
this.adminConnectionTimeout = props.getInt("admin.client.connection.timeout.sec", 60);
this.adminSocketTimeout = props.getInt("admin.client.socket.timeout.sec", 24 * 60 * 60);
this.streamMaxReadBytesPerSec = props.getBytes("stream.read.byte.per.sec", 10 * 1000 * 1000);
this.streamMaxWriteBytesPerSec = props.getBytes("stream.write.byte.per.sec", 10 * 1000 * 1000);
this.socketTimeoutMs = props.getInt("socket.timeout.ms", 5000);
this.socketBufferSize = (int) props.getBytes("socket.buffer.size", 64 * 1024);
this.socketKeepAlive = props.getBoolean("socket.keepalive", false);
this.useNioConnector = props.getBoolean("enable.nio.connector", false);
this.nioConnectorSelectors =
props.getInt(
"nio.connector.selectors", Math.max(8, Runtime.getRuntime().availableProcessors()));
this.nioAdminConnectorSelectors =
props.getInt(
"nio.admin.connector.selectors",
Math.max(8, Runtime.getRuntime().availableProcessors()));
this.clientSelectors = props.getInt("client.selectors", 4);
this.clientMaxConnectionsPerNode = props.getInt("client.max.connections.per.node", 50);
this.clientConnectionTimeoutMs = props.getInt("client.connection.timeout.ms", 500);
this.clientRoutingTimeoutMs = props.getInt("client.routing.timeout.ms", 15000);
this.clientMaxThreads = props.getInt("client.max.threads", 500);
this.clientThreadIdleMs = props.getInt("client.thread.idle.ms", 100000);
this.clientMaxQueuedRequests = props.getInt("client.max.queued.requests", 1000);
this.enableHttpServer = props.getBoolean("http.enable", true);
this.enableSocketServer = props.getBoolean("socket.enable", true);
this.enableAdminServer = props.getBoolean("admin.enable", true);
this.enableJmx = props.getBoolean("jmx.enable", true);
this.enablePipelineRoutedStore = props.getBoolean("enable.pipeline.routed.store", true);
this.enableSlop = props.getBoolean("slop.enable", true);
this.enableSlopPusherJob = props.getBoolean("slop.pusher.enable", true);
this.slopMaxWriteBytesPerSec = props.getBytes("slop.write.byte.per.sec", 10 * 1000 * 1000);
this.enableVerboseLogging = props.getBoolean("enable.verbose.logging", true);
this.enableStatTracking = props.getBoolean("enable.stat.tracking", true);
this.enableServerRouting = props.getBoolean("enable.server.routing", true);
this.enableMetadataChecking = props.getBoolean("enable.metadata.checking", true);
this.enableGossip = props.getBoolean("enable.gossip", false);
this.enableRebalanceService = props.getBoolean("enable.rebalancing", true);
this.enableRepair = props.getBoolean("enable.repair", true);
this.enableJmxClusterName = props.getBoolean("enable.jmx.clustername", false);
this.gossipInterval = props.getInt("gossip.interval.ms", 30 * 1000);
this.slopMaxWriteBytesPerSec = props.getBytes("slop.write.byte.per.sec", 10 * 1000 * 1000);
this.slopMaxReadBytesPerSec = props.getBytes("slop.read.byte.per.sec", 10 * 1000 * 1000);
this.slopStoreType = props.getString("slop.store.engine", BdbStorageConfiguration.TYPE_NAME);
this.slopFrequencyMs = props.getLong("slop.frequency.ms", 5 * 60 * 1000);
this.repairStartMs = props.getLong("repair.start.ms", 24 * 60 * 60 * 1000);
this.slopBatchSize = props.getInt("slop.batch.size", 100);
this.pusherType = props.getString("pusher.type", StreamingSlopPusherJob.TYPE_NAME);
this.slopZonesDownToTerminate = props.getInt("slop.zones.terminate", 0);
this.schedulerThreads = props.getInt("scheduler.threads", 6);
this.numScanPermits = props.getInt("num.scan.permits", 1);
this.storageConfigurations =
props.getList(
"storage.configs",
ImmutableList.of(
BdbStorageConfiguration.class.getName(),
MysqlStorageConfiguration.class.getName(),
InMemoryStorageConfiguration.class.getName(),
CacheStorageConfiguration.class.getName(),
ReadOnlyStorageConfiguration.class.getName()));
// start at midnight (0-23)
this.retentionCleanupFirstStartTimeInHour =
props.getInt("retention.cleanup.first.start.hour", 0);
// repeat every 24 hours
this.retentionCleanupScheduledPeriodInHour = props.getInt("retention.cleanup.period.hours", 24);
// save props for access from plugins
this.allProps = props;
String requestFormatName =
props.getString("request.format", RequestFormatType.VOLDEMORT_V1.getCode());
this.requestFormatType = RequestFormatType.fromCode(requestFormatName);
// rebalancing parameters
this.maxRebalancingAttempt = props.getInt("max.rebalancing.attempts", 3);
this.rebalancingTimeoutSec = props.getLong("rebalancing.timeout.seconds", 10 * 24 * 60 * 60);
this.maxParallelStoresRebalancing = props.getInt("max.parallel.stores.rebalancing", 3);
this.rebalancingOptimization = props.getBoolean("rebalancing.optimization", true);
this.failureDetectorImplementation =
props.getString(
"failuredetector.implementation",
FailureDetectorConfig.DEFAULT_IMPLEMENTATION_CLASS_NAME);
// We're changing the property from "client.node.bannage.ms" to
// "failuredetector.bannage.period" so if we have the old one, migrate
// it over.
if (props.containsKey("client.node.bannage.ms")
&& !props.containsKey("failuredetector.bannage.period")) {
props.put("failuredetector.bannage.period", props.get("client.node.bannage.ms"));
}
this.failureDetectorBannagePeriod =
props.getLong(
"failuredetector.bannage.period", FailureDetectorConfig.DEFAULT_BANNAGE_PERIOD);
this.failureDetectorThreshold =
props.getInt("failuredetector.threshold", FailureDetectorConfig.DEFAULT_THRESHOLD);
this.failureDetectorThresholdCountMinimum =
props.getInt(
"failuredetector.threshold.countminimum",
FailureDetectorConfig.DEFAULT_THRESHOLD_COUNT_MINIMUM);
this.failureDetectorThresholdInterval =
props.getLong(
"failuredetector.threshold.interval", FailureDetectorConfig.DEFAULT_THRESHOLD_INTERVAL);
this.failureDetectorAsyncRecoveryInterval =
props.getLong(
"failuredetector.asyncrecovery.interval",
FailureDetectorConfig.DEFAULT_ASYNC_RECOVERY_INTERVAL);
this.failureDetectorCatastrophicErrorTypes =
props.getList(
"failuredetector.catastrophic.error.types",
FailureDetectorConfig.DEFAULT_CATASTROPHIC_ERROR_TYPES);
this.failureDetectorRequestLengthThreshold =
props.getLong("failuredetector.request.length.threshold", getSocketTimeoutMs());
// network class loader disable by default.
this.enableNetworkClassLoader = props.getBoolean("enable.network.classloader", false);
validateParams();
}
/**
* Tests that, given no data completely fits in memory (realistic prod conditions), stores will
* stay within their limits, no matter how much disproportinate traffic you throw at it
*/
public void testStaticPrivateCaches() {
int totalCache = 20 * ByteUtils.BYTES_PER_MB; // total cache size
int shareA = 10 * ByteUtils.BYTES_PER_MB; // A reserves 10MB
int shareB = 5 * ByteUtils.BYTES_PER_MB; // B reserves 5MB
int shareC = totalCache - shareA - shareB; // the rest, 5 MB
int numRecords = 40;
// lets use all the default values.
Props props = new Props();
props.put("node.id", 1);
props.put("voldemort.home", "test/common/voldemort/config");
VoldemortConfig voldemortConfig = new VoldemortConfig(props);
voldemortConfig.setBdbCacheSize(totalCache);
voldemortConfig.setBdbOneEnvPerStore(true);
voldemortConfig.setBdbDataDirectory(bdbMasterDir.toURI().getPath());
bdbStorage = new BdbStorageConfiguration(voldemortConfig);
StoreDefinition defA =
TestUtils.makeStoreDefinition("storeA", shareA / (ByteUtils.BYTES_PER_MB));
BdbStorageEngine storeA = (BdbStorageEngine) bdbStorage.getStore(defA);
StoreDefinition defB =
TestUtils.makeStoreDefinition("storeB", shareB / (ByteUtils.BYTES_PER_MB));
BdbStorageEngine storeB = (BdbStorageEngine) bdbStorage.getStore(defB);
StoreDefinition defC = TestUtils.makeStoreDefinition("storeC");
BdbStorageEngine storeC = (BdbStorageEngine) bdbStorage.getStore(defC);
// before any traffic, the cache will not have grown
assertTrue(Math.abs(shareA - getCacheSize(defA)) > ByteUtils.BYTES_PER_MB);
assertTrue(Math.abs(shareB - getCacheSize(defB)) > ByteUtils.BYTES_PER_MB);
// sharedCacheSize reading 0 confirms that the store has a private cache
assertEquals(0, getStats(bdbStorage.getEnvironment(defA)).getSharedCacheTotalBytes());
assertEquals(0, getStats(bdbStorage.getEnvironment(defB)).getSharedCacheTotalBytes());
// load data into the stores; each store is guaranteed to be ~ 40MB.
// Data won't fit in memory
byte[] value = new byte[ByteUtils.BYTES_PER_MB];
for (int i = 0; i < numRecords; i++) {
storeA.put(TestUtils.toByteArray("testKey" + i), new Versioned<byte[]>(value), null);
storeB.put(TestUtils.toByteArray("testKey" + i), new Versioned<byte[]>(value), null);
storeC.put(TestUtils.toByteArray("testKey" + i), new Versioned<byte[]>(value), null);
}
// we will bring all of that data into the cache, by doing a keywalk.
// This should expand the cache as much as possible
long cacheSizeA = Long.MIN_VALUE;
long cacheSizeB = Long.MIN_VALUE;
long cacheSizeC = Long.MIN_VALUE;
for (int cycle = 0; cycle < 10; cycle++) {
for (int i = 0; i < numRecords; i++) {
long cycleCacheSizeA = getAndCheckCacheSize(storeA, defA, "testKey" + i);
long cycleCacheSizeB = getAndCheckCacheSize(storeB, defB, "testKey" + i);
long cycleCacheSizeC = getAndCheckCacheSize(storeC, defC, "testKey" + i);
// record the maximum cache size, each store every grew to
cacheSizeA = (cycleCacheSizeA > cacheSizeA) ? cycleCacheSizeA : cacheSizeA;
cacheSizeB = (cycleCacheSizeB > cacheSizeB) ? cycleCacheSizeB : cacheSizeB;
cacheSizeC = (cycleCacheSizeC > cacheSizeC) ? cycleCacheSizeC : cacheSizeC;
}
}
// check that they are certainly less than expected limits.
assertTrue(cacheSizeA <= shareA);
assertTrue(cacheSizeB <= shareB);
assertTrue(cacheSizeC <= shareC);
// check that they are not exceedingly high than their limits. Small
// overflows are okay. But should not be more than a 1MB
assertTrue(Math.abs(cacheSizeA - shareA) <= ByteUtils.BYTES_PER_MB);
assertTrue(Math.abs(cacheSizeB - shareB) <= ByteUtils.BYTES_PER_MB);
assertTrue(Math.abs(cacheSizeC - shareC) <= ByteUtils.BYTES_PER_MB);
// try doing reads on store C alone, for which we have no reservations.
// This simulates a spike on one store
long cacheSizeCNow = Long.MIN_VALUE;
for (int cycle = 0; cycle < 10; cycle++) {
for (int i = 0; i < numRecords; i++) {
long cycleCacheSizeCNow = getAndCheckCacheSize(storeC, defC, "testkey" + i);
// record the maximum cache size, each store grew to
cacheSizeCNow = (cycleCacheSizeCNow > cacheSizeCNow) ? cycleCacheSizeCNow : cacheSizeCNow;
}
}
assertTrue(cacheSizeCNow <= shareC);
storeA.close();
storeB.close();
storeC.close();
}
public void testDynamicReservations() {
int totalCache = 20 * ByteUtils.BYTES_PER_MB; // total cache size
int shareA = 10 * ByteUtils.BYTES_PER_MB; // A reserves 10MB
int shareB = totalCache - shareA;
int numRecords = 40;
// lets use all the default values.
Props props = new Props();
props.put("node.id", 1);
props.put("voldemort.home", "test/common/voldemort/config");
VoldemortConfig voldemortConfig = new VoldemortConfig(props);
voldemortConfig.setBdbCacheSize(totalCache);
voldemortConfig.setBdbOneEnvPerStore(true);
voldemortConfig.setBdbDataDirectory(bdbMasterDir.toURI().getPath());
voldemortConfig.setBdbMinimumSharedCache(5 * ByteUtils.BYTES_PER_MB);
bdbStorage = new BdbStorageConfiguration(voldemortConfig);
StoreDefinition defA = TestUtils.makeStoreDefinition("storeA", shareA / (1024 * 1024));
BdbStorageEngine storeA = (BdbStorageEngine) bdbStorage.getStore(defA);
StoreDefinition defB = TestUtils.makeStoreDefinition("storeB");
BdbStorageEngine storeB = (BdbStorageEngine) bdbStorage.getStore(defB);
// load data into the stores; each store is guaranteed to be ~ 40MB.
// Data won't fit in memory
byte[] value = new byte[ByteUtils.BYTES_PER_MB];
for (int i = 0; i < numRecords; i++) {
storeA.put(TestUtils.toByteArray("testKey" + i), new Versioned<byte[]>(value), null);
storeB.put(TestUtils.toByteArray("testKey" + i), new Versioned<byte[]>(value), null);
}
// 1. start with 10MB reserved cache for A and the rest 10MB for B
long cacheSizeA = Long.MIN_VALUE;
long cacheSizeB = Long.MIN_VALUE;
for (int cycle = 0; cycle < 10; cycle++) {
for (int i = 0; i < numRecords; i++) {
long cycleCacheSizeA = getAndCheckCacheSize(storeA, defA, "testKey" + i);
long cycleCacheSizeB = getAndCheckCacheSize(storeB, defB, "testKey" + i);
// record the maximum cache size, each store every grew to
cacheSizeA = (cycleCacheSizeA > cacheSizeA) ? cycleCacheSizeA : cacheSizeA;
cacheSizeB = (cycleCacheSizeB > cacheSizeB) ? cycleCacheSizeB : cacheSizeB;
}
}
assertTrue(Math.abs(cacheSizeA - shareA) <= ByteUtils.BYTES_PER_MB);
assertTrue(Math.abs(cacheSizeB - shareB) <= ByteUtils.BYTES_PER_MB);
// 2. dynamically grow the cache to 15MB and watch B shrink.
shareA = 15 * ByteUtils.BYTES_PER_MB;
shareB = totalCache - shareA;
defA = TestUtils.makeStoreDefinition("storeA", shareA / (1024 * 1024));
bdbStorage.update(defA);
cacheSizeA = Long.MIN_VALUE;
cacheSizeB = Long.MIN_VALUE;
for (int cycle = 0; cycle < 10; cycle++) {
for (int i = 0; i < numRecords; i++) {
long cycleCacheSizeA = getAndCheckCacheSize(storeA, defA, "testKey" + i);
long cycleCacheSizeB = getAndCheckCacheSize(storeB, defB, "testKey" + i);
// record the maximum cache size, each store every grew to
cacheSizeA = (cycleCacheSizeA > cacheSizeA) ? cycleCacheSizeA : cacheSizeA;
cacheSizeB = (cycleCacheSizeB > cacheSizeB) ? cycleCacheSizeB : cacheSizeB;
}
}
assertTrue(Math.abs(cacheSizeA - shareA) <= ByteUtils.BYTES_PER_MB);
assertTrue(Math.abs(cacheSizeB - shareB) <= ByteUtils.BYTES_PER_MB);
// 3. dynamically shrink it back to 10MB and watch B expand again.
shareA = 10 * ByteUtils.BYTES_PER_MB;
shareB = totalCache - shareA;
defA = TestUtils.makeStoreDefinition("storeA", shareA / (1024 * 1024));
bdbStorage.update(defA);
cacheSizeA = Long.MIN_VALUE;
cacheSizeB = Long.MIN_VALUE;
for (int cycle = 0; cycle < 10; cycle++) {
for (int i = 0; i < numRecords; i++) {
long cycleCacheSizeA = getAndCheckCacheSize(storeA, defA, "testKey" + i);
long cycleCacheSizeB = getAndCheckCacheSize(storeB, defB, "testKey" + i);
// record the maximum cache size, each store every grew to
cacheSizeA = (cycleCacheSizeA > cacheSizeA) ? cycleCacheSizeA : cacheSizeA;
cacheSizeB = (cycleCacheSizeB > cacheSizeB) ? cycleCacheSizeB : cacheSizeB;
}
}
// check that they are not exceedingly high than their limits. Small
// overflows are expected. But should not be more than a 1MB
assertTrue(Math.abs(cacheSizeA - shareA) <= ByteUtils.BYTES_PER_MB);
assertTrue(Math.abs(cacheSizeB - shareB) <= ByteUtils.BYTES_PER_MB);
storeA.close();
storeB.close();
}