private boolean isStaleConfiguration(Configuration config) {
String delimiter = "-";
String configId = config.getId();
// Bypasses item of "global" and folders of "version", just check db configurations.
if (configId == null || configId.equals(Constants.GLOBAL_ID) || !configId.contains(delimiter)) {
return false;
}
if (_serviceInfo.getId().endsWith(Constants.STANDALONE_ID)) {
if (!configId.equals(_serviceInfo.getId())) {
return true;
}
} else {
CoordinatorClientInetAddressMap nodeMap = _coordinator.getInetAddessLookupMap();
int nodeCount = nodeMap.getControllerNodeIPLookupMap().size();
String nodeIndex = configId.split(delimiter)[1];
if (Constants.STANDALONE_ID.equalsIgnoreCase(nodeIndex)
|| Integer.parseInt(nodeIndex) > nodeCount) {
return true;
}
}
return false;
}
public void setConfigValue(String key, String value) {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config != null) {
config.setConfig(key, value);
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
}
public String getConfigValue(String key) {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config != null) {
return config.getConfig(key);
}
return null;
}
/**
* set up the ssl connectors with strong ciphers
*
* @throws Exception
*/
protected void initConnectors() throws Exception {
if (!_disableHTTP) {
if (_unsecuredConnector == null) {
_unsecuredConnector = new SelectChannelConnector();
}
if (_unsecurePort != null) {
_unsecuredConnector.setPort(Integer.parseInt(_unsecurePort));
} else {
_unsecuredConnector.setPort(_serviceInfo.getEndpoint().getPort());
}
if (_httpBindAddress != null) {
_unsecuredConnector.setHost(_httpBindAddress);
}
if (lowResourcesConnections != null) {
_unsecuredConnector.setLowResourcesConnections(lowResourcesConnections);
}
if (lowResourcesMaxIdleTime != null) {
_unsecuredConnector.setLowResourcesMaxIdleTime(lowResourcesMaxIdleTime);
}
if (threadPool != null) {
_unsecuredConnector.setThreadPool(threadPool);
}
_server.addConnector(_unsecuredConnector);
}
if (!_disableSSL) {
SslContextFactory sslFac = new SslContextFactory();
sslFac.setIncludeCipherSuites(_ciphers);
KeyStore ks = KeyStoreUtil.getViPRKeystore(_coordinatorClient);
_log.debug(
"The certificates in Jetty is {}. ",
ks.getCertificateChain(KeystoreEngine.ViPR_KEY_AND_CERTIFICATE_ALIAS));
sslFac.setCertAlias(KeystoreEngine.ViPR_KEY_AND_CERTIFICATE_ALIAS);
sslFac.setKeyStore(ks);
_securedConnector = new SslSelectChannelConnector(sslFac);
if (_securePort != null) {
_securedConnector.setPort(Integer.parseInt(_securePort));
} else {
_securedConnector.setPort(_serviceInfo.getEndpoint().getPort());
}
if (_bindAddress != null) {
_securedConnector.setHost(_bindAddress);
}
if (lowResourcesConnections != null) {
_securedConnector.setLowResourcesConnections(lowResourcesConnections);
}
if (lowResourcesMaxIdleTime != null) {
_securedConnector.setLowResourcesMaxIdleTime(lowResourcesMaxIdleTime);
}
if (threadPool != null) {
_securedConnector.setThreadPool(threadPool);
}
_server.addConnector(_securedConnector);
}
_server.setSendServerVersion(false);
}
private void removeStaleVersionedDbConfiguration() {
String configKind =
_coordinator.getVersionedDbConfigPath(_serviceInfo.getName(), _serviceInfo.getVersion());
List<Configuration> configs =
_coordinator.queryAllConfiguration(_coordinator.getSiteId(), configKind);
for (Configuration config : configs) {
if (isStaleConfiguration(config)) {
_coordinator.removeServiceConfiguration(_coordinator.getSiteId(), config);
_log.info("Remove stale version db config, id: {}", config.getId());
}
}
}
/**
* Checks and sets INIT_DONE state this means we are done with the actual cf changes on the
* cassandra side for the target version
*/
private void setDbConfigInitDone() {
String configKind =
_coordinator.getVersionedDbConfigPath(_serviceInfo.getName(), _serviceInfo.getVersion());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config != null) {
if (config.getConfig(DbConfigConstants.INIT_DONE) == null) {
config.setConfig(DbConfigConstants.INIT_DONE, Boolean.TRUE.toString());
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
} else {
// we are expecting this to exist, because its initialized from checkVersionedConfiguration
throw new IllegalStateException("unexpected error, db versioned configuration is null");
}
}
// check and initialize global configuration
private Configuration checkGlobalConfiguration() {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, Constants.GLOBAL_ID);
if (config == null) {
// check if it is upgraded from previous version to yoda - configuration may be stored in
// znode /config. Since SeedProvider still need access that, so we remove the config
// from global in migration callback after migration is done.
config = _coordinator.queryConfiguration(configKind, Constants.GLOBAL_ID);
if (config != null) {
_log.info("Upgrade from pre-yoda release, move global config to new location");
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
return config;
}
ConfigurationImpl cfg = new ConfigurationImpl();
cfg.setId(Constants.GLOBAL_ID);
cfg.setKind(configKind);
cfg.setConfig(Constants.SCHEMA_VERSION, this._serviceInfo.getVersion());
// persist configuration
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), cfg);
config = cfg;
}
return config;
}
/**
* Retrieve the vdc config info of the current site, return such info with precheck. 1. For adding
* a new vdc, the current vdc should be in ISOLATED status and is a fresh installation. 2. For
* updating an existing vdc, the current vdc should be in CONNECTED status.
*
* @param checkParam
* @return VirtualDataCenterResponse
*/
@POST
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Path("/precheck")
public VdcPreCheckResponse precheckVdcConfig(VdcPreCheckParam checkParam) {
log.info("Start vdc config precheck for {} ...", checkParam.getConfigChangeType());
if (service.getId().endsWith("standalone")) {
throw GeoException.fatals.remoteVDCWrongStandaloneInstall();
}
log.info("Loading local vdc config ...");
VirtualDataCenter vdc = VdcUtil.getLocalVdc();
Boolean isFresher = checkParam.getFresher();
if (isFresher != null && isFresher) {
// check if VDC is a fresh installation, in ISOLATED status
if (VirtualDataCenter.ConnectionStatus.ISOLATED != vdc.getConnectionStatus()) {
throw GeoException.fatals.remoteFreshVDCWrongStatus(vdc.getId());
}
} else {
// check if VDC is in CONNECTED status
// check if VDC is in CONNECTED status- remove, add; update will skip-CTRL3549
if (checkParam.getConfigChangeType().equals(VdcConfig.ConfigChangeType.CONNECT_VDC.toString())
|| (checkParam
.getConfigChangeType()
.equals(VdcConfig.ConfigChangeType.REMOVE_VDC.toString()))) {
if (vdc.getConnectionStatus() != VirtualDataCenter.ConnectionStatus.CONNECTED) {
throw GeoException.fatals.remoteVDCWrongOperationStatus(
vdc.getId(), checkParam.getConfigChangeType());
}
}
}
boolean hasData = false;
if (isFresher) {
hasData = hasDataInDb();
}
hasData |= hasDataService();
log.info("Checking software version ...");
SoftwareVersion remoteSoftVer = null;
try {
remoteSoftVer = new SoftwareVersion(checkParam.getSoftwareVersion());
log.info("Software version of remote vdc: {}", remoteSoftVer);
} catch (Exception e) {
log.info(
"Cannot get software version from checkParam, the version of remote vdc is lower than v2.3 with exception {}",
e.getMessage());
}
SoftwareVersion localSoftVer;
try {
localSoftVer = coordinator.getTargetInfo(RepositoryInfo.class).getCurrentVersion();
} catch (Exception ex) {
throw GeoException.fatals.remoteVDCFailedToGetVersion(vdc.getId());
}
return toVirtualDataCenterResponse(vdc, hasData, remoteSoftVer, localSoftVer);
}
// check and initialize versioned configuration
private Configuration checkVersionedConfiguration() {
String serviceVersion = _serviceInfo.getVersion();
String dbSchemaVersion = _dbClient.getSchemaVersion();
if (!serviceVersion.equals(dbSchemaVersion)) {
_log.warn(
"The db service version {} doesn't equals Db schema version {}, "
+ "set db service version to Db schema version",
serviceVersion,
dbSchemaVersion);
_serviceInfo.setVersion(dbSchemaVersion);
}
String kind =
_coordinator.getVersionedDbConfigPath(_serviceInfo.getName(), _serviceInfo.getVersion());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), kind, _serviceInfo.getId());
if (config == null) {
// check if it is upgraded from previous version to yoda - configuration may be stored in
// znode /config
config = _coordinator.queryConfiguration(kind, _serviceInfo.getId());
if (config != null) {
_log.info("Upgrade from pre-2.5 release, move versioned dbconfig to new location");
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
return config;
}
ConfigurationImpl cfg = new ConfigurationImpl();
cfg.setId(_serviceInfo.getId());
cfg.setKind(kind);
// persist configuration
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), cfg);
config = cfg;
}
return config;
}
private void removeStaleServiceConfiguration() {
boolean isGeoDBSvc = isGeoDbsvc();
boolean resetAutoBootFlag = false;
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
List<Configuration> configs =
_coordinator.queryAllConfiguration(_coordinator.getSiteId(), configKind);
for (Configuration config : configs) {
if (isStaleConfiguration(config)) {
boolean autoboot = Boolean.parseBoolean(config.getConfig(DbConfigConstants.AUTOBOOT));
String configId = config.getId();
if (isGeoDBSvc && !autoboot && (configId.equals("geodb-4") || configId.equals("geodb-5"))) {
// for geodbsvc, if restore with the backup of 5 nodes to 3 nodes and the backup is made
// on the cluster that the 'autoboot=false' is set on vipr4 or vipr5
// we should set the autoboot=false on the current node or no node with autoboot=false
// TODO:This is a temporary/safest solution in Yoda, we'll provide a better soltuion post
// Yoda
resetAutoBootFlag = true;
}
if (isStaleConfiguration(config)) {
_coordinator.removeServiceConfiguration(_coordinator.getSiteId(), config);
_log.info("Remove stale db config, id: {}", config.getId());
}
}
}
if (resetAutoBootFlag) {
_log.info("set autoboot flag to false on {}", _serviceInfo.getId());
Configuration config =
_coordinator.queryConfiguration(
_coordinator.getSiteId(), configKind, _serviceInfo.getId());
config.setConfig(DbConfigConstants.AUTOBOOT, Boolean.FALSE.toString());
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
}
/**
* Checks and registers db configuration information, this is one time when cluster is coming up
* for the first time
*/
private Configuration checkConfiguration() {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config == null) {
// check if it is upgraded from previous version to yoda - configuration may be stored in
// zk global area /config. Since SeedProvider still need access that, so we remove the config
// from global in migration callback after migration is done.
config = _coordinator.queryConfiguration(configKind, _serviceInfo.getId());
if (config != null) {
_log.info("Upgrade from pre-yoda release, move dbconfig to new location");
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
return config;
}
// this is a new node
// 1. register its configuration with coordinator
// 2. assume autobootstrap configuration
// this means that when a node is added, it take 1/2 of biggest token rage and
// copies its data over
ConfigurationImpl cfg = new ConfigurationImpl();
cfg.setId(_serviceInfo.getId());
cfg.setKind(configKind);
cfg.setConfig(DbConfigConstants.NODE_ID, _coordinator.getInetAddessLookupMap().getNodeId());
cfg.setConfig(DbConfigConstants.AUTOBOOT, Boolean.TRUE.toString());
// check other existing db nodes
List<Configuration> configs =
_coordinator.queryAllConfiguration(_coordinator.getSiteId(), configKind);
if (configs.isEmpty()) {
// we are the first node - turn off autobootstrap
cfg.setConfig(DbConfigConstants.AUTOBOOT, Boolean.FALSE.toString());
}
// persist configuration
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), cfg);
config = cfg;
}
return config;
}
/** Check offline event info to see if dbsvc/geodbsvc on this node could get started */
private void checkDBOfflineInfo() {
Configuration config =
_coordinator.queryConfiguration(
_coordinator.getSiteId(), Constants.DB_DOWNTIME_TRACKER_CONFIG, _serviceInfo.getName());
DbOfflineEventInfo dbOfflineEventInfo = new DbOfflineEventInfo(config);
String localNodeId = _coordinator.getInetAddessLookupMap().getNodeId();
Long lastActiveTimestamp = dbOfflineEventInfo.geLastActiveTimestamp(localNodeId);
long zkTimeStamp =
(lastActiveTimestamp == null) ? TimeUtils.getCurrentTime() : lastActiveTimestamp;
File localDbDir = new File(dbDir);
Date lastModified = getLastModified(localDbDir);
boolean isDirEmpty = lastModified == null || localDbDir.list().length == 0;
long localTimeStamp = (isDirEmpty) ? TimeUtils.getCurrentTime() : lastModified.getTime();
_log.info("Service timestamp in ZK is {}, local file is: {}", zkTimeStamp, localTimeStamp);
long diffTime = (zkTimeStamp > localTimeStamp) ? (zkTimeStamp - localTimeStamp) : 0;
if (diffTime >= MAX_SERVICE_OUTAGE_TIME) {
String errMsg =
String.format(
"We detect database files on local disk are more than %s days older "
+ "than last time it was seen in the cluster. It may bring stale data into the database, "
+ "so the service cannot continue to boot. It may be the result of a VM snapshot rollback. "
+ "Please contact with EMC support engineer for solution.",
diffTime / TimeUtils.DAYS);
alertLog.error(errMsg);
throw new IllegalStateException(errMsg);
}
Long offlineTime = dbOfflineEventInfo.getOfflineTimeInMS(localNodeId);
if (!isDirEmpty && offlineTime != null && offlineTime >= MAX_SERVICE_OUTAGE_TIME) {
String errMsg =
String.format(
"This node is offline for more than %s days. It may bring stale data into "
+ "database, so the service cannot continue to boot. Please poweroff this node and follow our "
+ "node recovery procedure to recover this node",
offlineTime / TimeUtils.DAYS);
alertLog.error(errMsg);
throw new IllegalStateException(errMsg);
}
}
/**
* check to see if the individual nodes of one vdc are visible from another
*
* @param checkParam
* @return
*/
@POST
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Path("/nodecheck")
public VdcNodeCheckResponse checkNodeConnections(
VdcNodeCheckParam checkParam, @HeaderParam("X-Forwarded-For") String clientIp) {
List<VdcConfig> vdcList = checkParam.getVirtualDataCenters();
log.info("checking nodes for vdcs {} ...", getVdcIds(vdcList));
if (service.getId().endsWith("standalone")) {
throw GeoException.fatals.remoteVDCWrongStandaloneInstall();
}
ArgValidator.checkFieldNotEmpty(vdcList, "vdc");
VirtualDataCenter localVdc = VdcUtil.getLocalVdc();
if (localVdc == null) {
throw GeoException.fatals.failedToFindLocalVDC();
}
return toVdcNodeCheckResponse(localVdc, helper.areNodesReachable(vdcList, false));
}
/** Kick off background jobs */
private void startBackgroundTasks() {
if (!_schemaUtil.isStandby()) {
if (!disableScheduledDbRepair) {
startBackgroundNodeRepairTask();
}
if (_gcExecutor != null) {
_gcExecutor.setDbServiceId(_serviceInfo.getId());
_gcExecutor.start();
}
if (_taskScrubber != null) {
_taskScrubber.start();
}
if (_eventScrubber != null) {
_eventScrubber.start();
}
}
startBackgroundDetectorTask();
startBackgroundCompactTask();
}
/**
* Do more precheck For disconnecting a vdc, check if there is a VDC that is under disconnecting
* If yes, return the VDC under disconnecting, otherwise set the VDC (given by parameter) status
* to DISCONNECTING
*
* @param checkParam
* @return VdcPreCheckResponse
*/
@POST
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Path("/precheck2")
public VdcPreCheckResponse2 precheckVdcConfig(VdcPreCheckParam2 checkParam) {
log.info("Start vdc config precheck2 for {} ...", checkParam.getConfigChangeType());
if (service.getId().endsWith("standalone")) {
throw GeoException.fatals.remoteVDCWrongStandaloneInstall();
}
VdcConfig.ConfigChangeType type = checkParam.getConfigChangeType();
VirtualDataCenter vdc = null;
VdcPreCheckResponse2 resp2 = new VdcPreCheckResponse2();
resp2.setCompatible(true);
// BZ
// TODO Need to use a different method to update info on lock on a remote system.
// Need to use a different field (not connection status of VDC object) as a locking mechanism)
boolean precheckFailed = checkParam.isPrecheckFailed();
switch (type) {
case DISCONNECT_VDC:
log.info("Precheck2 for disconnect ops");
vdc = helper.getDisconnectingVdc();
if (checkParam.getIsAllNotReachable()) {
URI targetVdcId = checkParam.getVdcIds().get(0);
log.info("Precheck2 to check the disconnect vdc {} is reachable", targetVdcId);
VirtualDataCenter targetVdc = dbClient.queryObject(VirtualDataCenter.class, targetVdcId);
resp2.setIsAllNodesNotReachable(
!helper.areNodesReachable(
getLocalVdc().getShortId(),
targetVdc.getHostIPv4AddressesMap(),
targetVdc.getHostIPv6AddressesMap(),
checkParam.getIsAllNotReachable()));
break;
}
if (precheckFailed) {
log.info("Precheck2 to update reconnect precheck fail status");
String vdcState = checkParam.getDefaultVdcState();
if (StringUtils.isNotEmpty(vdcState)) {
vdc.setConnectionStatus(VirtualDataCenter.ConnectionStatus.valueOf(vdcState));
dbClient.updateAndReindexObject(vdc);
}
break;
}
if (vdc == null) {
// no DISCONNECTING_VDC
log.info("Precheck2: there is no disconnecting vdc");
URI srcVdcId = checkParam.getVdcIds().get(1);
VirtualDataCenter srcVdc = dbClient.queryObject(VirtualDataCenter.class, srcVdcId);
if (srcVdc.getConnectionStatus() == VirtualDataCenter.ConnectionStatus.DISCONNECTED) {
resp2.setCompatible(false);
break;
}
// BZ
// TODO need to use a different field to set locks on concurrent VDC operation
URI id = checkParam.getVdcIds().get(0);
vdc = dbClient.queryObject(VirtualDataCenter.class, id);
vdc.setConnectionStatus(VirtualDataCenter.ConnectionStatus.DISCONNECTING);
dbClient.updateAndReindexObject(vdc);
} else {
resp2 = toVirtualDataCenterResponse2(vdc, true, null);
}
break;
case RECONNECT_VDC:
log.info("Precheck2 for reconnect ops checkParam={}", checkParam);
List<String> blackList = checkParam.getBlackList();
List<String> whiteList = checkParam.getWhiteList();
log.info("Precheck2 to check if two vdc disconnect each other");
resp2.setCompatible(true);
if (isDisconnectedEachOther(blackList, whiteList)) {
log.info("Precheck2: two vdc have disconnected each other");
resp2.setCompatible(false);
break;
}
if (precheckFailed) {
log.info("Precheck2 to update reconnect precheck fail status");
URI targetVdcId = checkParam.getVdcIds().get(0);
log.info("Precheck2 to check the disconnect vdc {} is reachable", targetVdcId);
VirtualDataCenter targetVdc = dbClient.queryObject(VirtualDataCenter.class, targetVdcId);
String vdcState = checkParam.getDefaultVdcState();
if (StringUtils.isNotEmpty(vdcState)) {
targetVdc.setConnectionStatus(VirtualDataCenter.ConnectionStatus.valueOf(vdcState));
dbClient.updateAndReindexObject(targetVdc);
}
break;
}
break;
}
log.info("Precheck2 done, resp is {}", resp2.toString());
return resp2;
}
