public void setConfigValue(String key, String value) {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config != null) {
config.setConfig(key, value);
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
}
public String getConfigValue(String key) {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config != null) {
return config.getConfig(key);
}
return null;
}
private void removeStaleVersionedDbConfiguration() {
String configKind =
_coordinator.getVersionedDbConfigPath(_serviceInfo.getName(), _serviceInfo.getVersion());
List<Configuration> configs =
_coordinator.queryAllConfiguration(_coordinator.getSiteId(), configKind);
for (Configuration config : configs) {
if (isStaleConfiguration(config)) {
_coordinator.removeServiceConfiguration(_coordinator.getSiteId(), config);
_log.info("Remove stale version db config, id: {}", config.getId());
}
}
}
private RecoveryStatus.Status getRecoveryStatus() {
RecoveryStatus.Status status = null;
Configuration cfg =
coordinator.queryConfiguration(Constants.NODE_RECOVERY_STATUS, Constants.GLOBAL_ID);
if (cfg != null) {
String statusStr = cfg.getConfig(RecoveryConstants.RECOVERY_STATUS);
if (statusStr != null && statusStr.length() > 0) {
status = RecoveryStatus.Status.valueOf(statusStr);
}
}
log.info("Recovery status is: {}", status);
return status;
}
/**
* Checks and sets INIT_DONE state this means we are done with the actual cf changes on the
* cassandra side for the target version
*/
private void setDbConfigInitDone() {
String configKind =
_coordinator.getVersionedDbConfigPath(_serviceInfo.getName(), _serviceInfo.getVersion());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config != null) {
if (config.getConfig(DbConfigConstants.INIT_DONE) == null) {
config.setConfig(DbConfigConstants.INIT_DONE, Boolean.TRUE.toString());
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
} else {
// we are expecting this to exist, because its initialized from checkVersionedConfiguration
throw new IllegalStateException("unexpected error, db versioned configuration is null");
}
}
private boolean isStaleConfiguration(Configuration config) {
String delimiter = "-";
String configId = config.getId();
// Bypasses item of "global" and folders of "version", just check db configurations.
if (configId == null || configId.equals(Constants.GLOBAL_ID) || !configId.contains(delimiter)) {
return false;
}
if (_serviceInfo.getId().endsWith(Constants.STANDALONE_ID)) {
if (!configId.equals(_serviceInfo.getId())) {
return true;
}
} else {
CoordinatorClientInetAddressMap nodeMap = _coordinator.getInetAddessLookupMap();
int nodeCount = nodeMap.getControllerNodeIPLookupMap().size();
String nodeIndex = configId.split(delimiter)[1];
if (Constants.STANDALONE_ID.equalsIgnoreCase(nodeIndex)
|| Integer.parseInt(nodeIndex) > nodeCount) {
return true;
}
}
return false;
}
/**
* Read a string list(connected by ',') from given db config
*
* @param config
* @return
*/
private List<String> checkConfigList(Configuration config, String name) {
String peerIPs = config.getConfig(name);
ArrayList<String> peers = new ArrayList<String>();
if (peerIPs != null) {
for (String ip : StringUtils.split(peerIPs, ",")) {
peers.add(ip);
}
}
return peers;
}
private void removeStaleServiceConfiguration() {
boolean isGeoDBSvc = isGeoDbsvc();
boolean resetAutoBootFlag = false;
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
List<Configuration> configs =
_coordinator.queryAllConfiguration(_coordinator.getSiteId(), configKind);
for (Configuration config : configs) {
if (isStaleConfiguration(config)) {
boolean autoboot = Boolean.parseBoolean(config.getConfig(DbConfigConstants.AUTOBOOT));
String configId = config.getId();
if (isGeoDBSvc && !autoboot && (configId.equals("geodb-4") || configId.equals("geodb-5"))) {
// for geodbsvc, if restore with the backup of 5 nodes to 3 nodes and the backup is made
// on the cluster that the 'autoboot=false' is set on vipr4 or vipr5
// we should set the autoboot=false on the current node or no node with autoboot=false
// TODO:This is a temporary/safest solution in Yoda, we'll provide a better soltuion post
// Yoda
resetAutoBootFlag = true;
}
if (isStaleConfiguration(config)) {
_coordinator.removeServiceConfiguration(_coordinator.getSiteId(), config);
_log.info("Remove stale db config, id: {}", config.getId());
}
}
}
if (resetAutoBootFlag) {
_log.info("set autoboot flag to false on {}", _serviceInfo.getId());
Configuration config =
_coordinator.queryConfiguration(
_coordinator.getSiteId(), configKind, _serviceInfo.getId());
config.setConfig(DbConfigConstants.AUTOBOOT, Boolean.FALSE.toString());
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
}
/**
* Read bool value from given db config
*
* @param config
* @param name
* @return
*/
private boolean checkConfigBool(Configuration config, String name) {
String value = config.getConfig(name);
return value != null && Boolean.parseBoolean(value);
}
@Override
public void start() throws IOException {
if (_log.isInfoEnabled()) {
_log.info("Starting DB service...");
}
// Suppress Sonar violation of Lazy initialization of static fields should be synchronized
// start() method will be only called one time when startup dbsvc, so it's safe to ignore sonar
// violation
instance = this; // NOSONAR ("squid:S2444")
if (backCompatPreYoda) {
_log.info(
"Pre-yoda back compatible flag detected. Initialize local keystore/truststore for Cassandra native encryption");
initKeystoreAndTruststore();
_schemaUtil.setBackCompatPreYoda(true);
}
System.setProperty("cassandra.config", _config);
System.setProperty("cassandra.config.loader", CassandraConfigLoader.class.getName());
// Set to false to clear all gossip state for the node on restart.
//
// We encounter a weird Cassandra grossip issue(COP-19246) - some nodes are missing from gossip
// when rebooting the entire cluster simultaneously. Critical Gossip
// fields(ApplicationState.STATUS, ApplicationState.TOKENS)
// are not synchronized during handshaking. It looks like some problem caused by incorrect
// gossip version/generation
// at system local table. So add this option to cleanup local gossip state during reboot
//
// Make sure add-vdc/add-standby passed when you would remove this option in the future.
//
// We need make sure majority local nodes are added as seed nodes. Otherwise cassandra may not
// see other nodes if it loses
// connection to other sites
System.setProperty("cassandra.load_ring_state", "false");
// Nodes in new data center should not auto-bootstrap.
// See
// https://docs.datastax.com/en/cassandra/2.0/cassandra/operations/ops_add_dc_to_cluster_t.html
if (_schemaUtil.isStandby()) {
System.setProperty("cassandra.auto_bootstrap", "false");
}
InterProcessLock lock = null;
Configuration config = null;
StartupMode mode = null;
try {
// we use this lock to discourage more than one node bootstrapping / joining at the same time
// Cassandra can handle this but it's generally not recommended to make changes to schema
// concurrently
lock = getLock(getSchemaLockName());
config = checkConfiguration();
checkGlobalConfiguration();
checkVersionedConfiguration();
removeStaleConfiguration();
mode = checkStartupMode(config);
_log.info("Current startup mode is {}", mode);
// Check if service is allowed to get started by querying db offline info to avoid bringing
// back stale data.
// Skipping hibernate mode for node recovery procedure to recover the overdue node.
int nodeCount = ((CoordinatorClientImpl) _coordinator).getNodeCount();
if (nodeCount != 1 && mode.type != StartupMode.StartupModeType.HIBERNATE_MODE) {
checkDBOfflineInfo();
}
// this call causes instantiation of a seed provider instance, so the check*Configuration
// calls must be preceed it
removeCassandraSavedCaches();
mode.onPreStart();
if (_jmxServer != null) {
_jmxServer.start();
System.setProperty(
"com.sun.management.jmxremote.port", Integer.toString(_jmxServer.getPort()));
}
_service = new CassandraDaemon();
_service.init(null);
_service.start();
cassandraInitialized = true;
mode.onPostStart();
} catch (Exception e) {
if (mode != null && mode.type == StartupMode.StartupModeType.HIBERNATE_MODE) {
printRecoveryWorkAround(e);
}
_log.error("e=", e);
throw new IllegalStateException(e);
} finally {
if (lock != null) {
try {
lock.release();
} catch (Exception ignore) {
_log.debug("lock release failed");
}
}
}
if (config.getConfig(DbConfigConstants.JOINED) == null) {
config.setConfig(DbConfigConstants.JOINED, Boolean.TRUE.toString());
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
_statusChecker.waitForAllNodesJoined();
_svcBeacon.start();
if (backCompatPreYoda) {
_log.info("Enable duplicated beacon in global area during pre-yoda upgrade");
startDupBeacon();
}
setDbInitializedFlag();
setDbConfigInitDone();
_dbClient.start();
if (_schemaUtil.isStandby()) {
String localDataRevision = getLocalDataRevision();
if (localDataRevision != null) {
_schemaUtil.checkDataRevision(localDataRevision);
}
}
// Setup the vdc information, so that login enabled before migration
if (!isGeoDbsvc()) {
_schemaUtil.checkAndSetupBootStrapInfo(_dbClient);
}
dbMgr.init();
if (_handler.run()) {
// Setup the bootstrap info root tenant, if root tenant migrated from local db, then skip it
if (isGeoDbsvc()) {
_schemaUtil.checkAndSetupBootStrapInfo(_dbClient);
} else {
_schemaUtil.checkAndInitStorageSystemTypes(_dbClient);
}
startBackgroundTasks();
_log.info("DB service started");
} else {
_log.error("DB migration failed. Skipping starting background tasks.");
}
}
/**
* Remove given flag from current db config in zk
*
* @param flagName
*/
void removeFlag(String flagName) {
config.removeConfig(flagName);
coordinator.persistServiceConfiguration(config);
}
/**
* We select seeds based on the following rules - For DR standby sites, use all nodes in active
* site as seeds For DR active site, use local nodes as seeds. The rule to select local seed is -
* first boot node(AUTOBOOT = false) uses itself as seed nodes so that it could boot and
* initialize schema - subsquent node(AUTOBOOT = true) uses other successfully booted(JOINED =
* true) nodes as seeds
*/
@Override
public List<InetAddress> getSeeds() {
try {
CoordinatorClientInetAddressMap nodeMap = _client.getInetAddessLookupMap();
List<Configuration> configs =
_client.queryAllConfiguration(_client.getSiteId(), Constants.DB_CONFIG);
List<InetAddress> seeds = new ArrayList<>();
// If we are upgrading from pre-2.5 releases, dbconfig exists in zk global area
List<Configuration> leftoverConfig = _client.queryAllConfiguration(Constants.DB_CONFIG);
configs.addAll(leftoverConfig);
// Add extra seeds - seeds from remote sites
for (String seed : extraSeeds) {
if (StringUtils.isNotEmpty(seed)) {
seeds.add(InetAddress.getByName(seed));
}
}
// On DR standby site, only use seeds from active site. On active site
// we use local seeds
if (isDrActiveSite) {
for (int i = 0; i < configs.size(); i++) {
Configuration config = configs.get(i);
// Bypasses item of "global" and folders of "version", just check db configurations.
if (config.getId() == null || config.getId().equals(Constants.GLOBAL_ID)) {
continue;
}
String nodeIndex = config.getId().split("-")[1];
String nodeId = config.getConfig(DbConfigConstants.NODE_ID);
if (nodeId == null) {
// suppose that they are existing znodes from a previous version
// set the NODE_ID config, id is like db-x
nodeId = "vipr" + nodeIndex;
config.setConfig(DbConfigConstants.NODE_ID, nodeId);
config.removeConfig(DbConfigConstants.DB_IP);
_client.persistServiceConfiguration(_client.getSiteId(), config);
}
if (!Boolean.parseBoolean(config.getConfig(DbConfigConstants.AUTOBOOT))
|| (!config.getId().equals(_id)
&& Boolean.parseBoolean(config.getConfig(DbConfigConstants.JOINED)))) {
// all non autobootstrap nodes + other nodes are used as seeds
InetAddress ip = null;
if (nodeMap != null) {
String ipAddress = nodeMap.getConnectableInternalAddress(nodeId);
_logger.debug("ip[" + i + "]: " + ipAddress);
ip = InetAddress.getByName(ipAddress);
} else {
ip = InetAddress.getByName(nodeId);
}
seeds.add(ip);
_logger.info("Seed {}", ip);
}
}
}
_logger.info("Seeds list {}", StringUtils.join(seeds.toArray(), ","));
return seeds;
} catch (Exception e) {
throw new IllegalStateException(e);
}
}
