// check and initialize versioned configuration
private Configuration checkVersionedConfiguration() {
String serviceVersion = _serviceInfo.getVersion();
String dbSchemaVersion = _dbClient.getSchemaVersion();
if (!serviceVersion.equals(dbSchemaVersion)) {
_log.warn(
"The db service version {} doesn't equals Db schema version {}, "
+ "set db service version to Db schema version",
serviceVersion,
dbSchemaVersion);
_serviceInfo.setVersion(dbSchemaVersion);
}
String kind =
_coordinator.getVersionedDbConfigPath(_serviceInfo.getName(), _serviceInfo.getVersion());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), kind, _serviceInfo.getId());
if (config == null) {
// check if it is upgraded from previous version to yoda - configuration may be stored in
// znode /config
config = _coordinator.queryConfiguration(kind, _serviceInfo.getId());
if (config != null) {
_log.info("Upgrade from pre-2.5 release, move versioned dbconfig to new location");
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
return config;
}
ConfigurationImpl cfg = new ConfigurationImpl();
cfg.setId(_serviceInfo.getId());
cfg.setKind(kind);
// persist configuration
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), cfg);
config = cfg;
}
return config;
}
/** Initializes the keystore/truststore if the paths have been provided. */
private void initKeystoreAndTruststore() {
try {
DbClientContext ctx = _dbClient.getLocalContext();
if (isGeoDbsvc()) {
ctx = _dbClient.getGeoContext();
}
String keystorePath = ctx.getKeyStoreFile();
String truststorePath = ctx.getTrustStoreFile();
if (keystorePath == null && truststorePath == null) {
_log.info("Skipping keystore/truststore initialization, no paths provided");
return;
}
String password = ctx.getTrustStorePassword();
CassandraKeystoreHandler keystoreHandler =
new CassandraKeystoreHandler(_coordinator, keystorePath, truststorePath, password);
if (keystorePath != null) {
_log.info("Initializing keystore for current node: {}", keystorePath);
keystoreHandler.saveKeyStore();
} else {
_log.info("Skipping keystore initialization, no path provided");
}
if (truststorePath != null) {
_log.info("Initializing truststore for current node: {}", truststorePath);
keystoreHandler.saveTrustStore();
} else {
_log.info("Skipping truststore initialization, no path provided");
}
} catch (Exception e) {
_log.error("Unexpected exception during initializing cassandra keystore", e);
throw new IllegalStateException(e);
}
}
/**
* Create DbClient to embedded DB
*
* @return
*/
protected static DbClient createDbClient() {
ENCRYPTION_PROVIDER.setCoordinator(ModelTestSuite.getCoordinator());
DbClientContext localCtx = new DbClientContext();
localCtx.setClusterName("Test");
localCtx.setKeyspaceName("Test");
DbClientImpl dbClient = new DbClientImpl();
dbClient.setCoordinatorClient(ModelTestSuite.getCoordinator());
dbClient.setEncryptionProvider(ENCRYPTION_PROVIDER);
dbClient.setBypassMigrationLock(true);
dbClient.setDbVersionInfo(ModelTestSuite.getDbVersionInfo());
dbClient.setLocalContext(localCtx);
VdcUtil.setDbClient(dbClient);
dbClient.start();
return dbClient;
}
public static DbClientImpl newDBClient() throws Exception {
ZkConnection zkConnection = new ZkConnection();
zkConnection.setServer(Lists.newArrayList(new URI("coordinator://localhost:2181")));
zkConnection.build();
DualInetAddress dualInetAddress = DualInetAddress.fromAddresses("127.0.0.1", "::1");
Map<String, DualInetAddress> addresses = Maps.newHashMap();
addresses.put("localhost", dualInetAddress);
CoordinatorClientInetAddressMap map = new CoordinatorClientInetAddressMap();
map.setNodeId("standalone");
map.setDualInetAddress(dualInetAddress);
map.setControllerNodeIPLookupMap(addresses);
CoordinatorClientImpl coordinatorClient = new CoordinatorClientImpl();
coordinatorClient.setZkConnection(zkConnection);
coordinatorClient.setInetAddessLookupMap(map);
coordinatorClient.start();
DbClientContext localContext = new DbClientContext();
localContext.setKeyspaceName("StorageOS");
localContext.setClusterName("StorageOs");
DbClientContext geoContext = new DbClientContext();
geoContext.setKeyspaceName("GeoStorageOs");
geoContext.setClusterName("GeoStorageOs");
DbVersionInfo versionInfo = new DbVersionInfo();
versionInfo.setSchemaVersion("2.0");
DbClientImpl client = new DbClientImpl();
client.setDbVersionInfo(versionInfo);
client.setLocalContext(localContext);
client.setGeoContext(geoContext);
client.setCoordinatorClient(coordinatorClient);
client.setLocalContext(new DbClientContext());
client.start();
VdcUtil.setDbClient(client);
return client;
}
@Override
public void start() throws IOException {
if (_log.isInfoEnabled()) {
_log.info("Starting DB service...");
}
// Suppress Sonar violation of Lazy initialization of static fields should be synchronized
// start() method will be only called one time when startup dbsvc, so it's safe to ignore sonar
// violation
instance = this; // NOSONAR ("squid:S2444")
if (backCompatPreYoda) {
_log.info(
"Pre-yoda back compatible flag detected. Initialize local keystore/truststore for Cassandra native encryption");
initKeystoreAndTruststore();
_schemaUtil.setBackCompatPreYoda(true);
}
System.setProperty("cassandra.config", _config);
System.setProperty("cassandra.config.loader", CassandraConfigLoader.class.getName());
// Set to false to clear all gossip state for the node on restart.
//
// We encounter a weird Cassandra grossip issue(COP-19246) - some nodes are missing from gossip
// when rebooting the entire cluster simultaneously. Critical Gossip
// fields(ApplicationState.STATUS, ApplicationState.TOKENS)
// are not synchronized during handshaking. It looks like some problem caused by incorrect
// gossip version/generation
// at system local table. So add this option to cleanup local gossip state during reboot
//
// Make sure add-vdc/add-standby passed when you would remove this option in the future.
//
// We need make sure majority local nodes are added as seed nodes. Otherwise cassandra may not
// see other nodes if it loses
// connection to other sites
System.setProperty("cassandra.load_ring_state", "false");
// Nodes in new data center should not auto-bootstrap.
// See
// https://docs.datastax.com/en/cassandra/2.0/cassandra/operations/ops_add_dc_to_cluster_t.html
if (_schemaUtil.isStandby()) {
System.setProperty("cassandra.auto_bootstrap", "false");
}
InterProcessLock lock = null;
Configuration config = null;
StartupMode mode = null;
try {
// we use this lock to discourage more than one node bootstrapping / joining at the same time
// Cassandra can handle this but it's generally not recommended to make changes to schema
// concurrently
lock = getLock(getSchemaLockName());
config = checkConfiguration();
checkGlobalConfiguration();
checkVersionedConfiguration();
removeStaleConfiguration();
mode = checkStartupMode(config);
_log.info("Current startup mode is {}", mode);
// Check if service is allowed to get started by querying db offline info to avoid bringing
// back stale data.
// Skipping hibernate mode for node recovery procedure to recover the overdue node.
int nodeCount = ((CoordinatorClientImpl) _coordinator).getNodeCount();
if (nodeCount != 1 && mode.type != StartupMode.StartupModeType.HIBERNATE_MODE) {
checkDBOfflineInfo();
}
// this call causes instantiation of a seed provider instance, so the check*Configuration
// calls must be preceed it
removeCassandraSavedCaches();
mode.onPreStart();
if (_jmxServer != null) {
_jmxServer.start();
System.setProperty(
"com.sun.management.jmxremote.port", Integer.toString(_jmxServer.getPort()));
}
_service = new CassandraDaemon();
_service.init(null);
_service.start();
cassandraInitialized = true;
mode.onPostStart();
} catch (Exception e) {
if (mode != null && mode.type == StartupMode.StartupModeType.HIBERNATE_MODE) {
printRecoveryWorkAround(e);
}
_log.error("e=", e);
throw new IllegalStateException(e);
} finally {
if (lock != null) {
try {
lock.release();
} catch (Exception ignore) {
_log.debug("lock release failed");
}
}
}
if (config.getConfig(DbConfigConstants.JOINED) == null) {
config.setConfig(DbConfigConstants.JOINED, Boolean.TRUE.toString());
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
_statusChecker.waitForAllNodesJoined();
_svcBeacon.start();
if (backCompatPreYoda) {
_log.info("Enable duplicated beacon in global area during pre-yoda upgrade");
startDupBeacon();
}
setDbInitializedFlag();
setDbConfigInitDone();
_dbClient.start();
if (_schemaUtil.isStandby()) {
String localDataRevision = getLocalDataRevision();
if (localDataRevision != null) {
_schemaUtil.checkDataRevision(localDataRevision);
}
}
// Setup the vdc information, so that login enabled before migration
if (!isGeoDbsvc()) {
_schemaUtil.checkAndSetupBootStrapInfo(_dbClient);
}
dbMgr.init();
if (_handler.run()) {
// Setup the bootstrap info root tenant, if root tenant migrated from local db, then skip it
if (isGeoDbsvc()) {
_schemaUtil.checkAndSetupBootStrapInfo(_dbClient);
} else {
_schemaUtil.checkAndInitStorageSystemTypes(_dbClient);
}
startBackgroundTasks();
_log.info("DB service started");
} else {
_log.error("DB migration failed. Skipping starting background tasks.");
}
}