public String getConfigValue(String key) {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config != null) {
return config.getConfig(key);
}
return null;
}
public void setConfigValue(String key, String value) {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config != null) {
config.setConfig(key, value);
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
}
private void removeStaleVersionedDbConfiguration() {
String configKind =
_coordinator.getVersionedDbConfigPath(_serviceInfo.getName(), _serviceInfo.getVersion());
List<Configuration> configs =
_coordinator.queryAllConfiguration(_coordinator.getSiteId(), configKind);
for (Configuration config : configs) {
if (isStaleConfiguration(config)) {
_coordinator.removeServiceConfiguration(_coordinator.getSiteId(), config);
_log.info("Remove stale version db config, id: {}", config.getId());
}
}
}
// check and initialize global configuration
private Configuration checkGlobalConfiguration() {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, Constants.GLOBAL_ID);
if (config == null) {
// check if it is upgraded from previous version to yoda - configuration may be stored in
// znode /config. Since SeedProvider still need access that, so we remove the config
// from global in migration callback after migration is done.
config = _coordinator.queryConfiguration(configKind, Constants.GLOBAL_ID);
if (config != null) {
_log.info("Upgrade from pre-yoda release, move global config to new location");
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
return config;
}
ConfigurationImpl cfg = new ConfigurationImpl();
cfg.setId(Constants.GLOBAL_ID);
cfg.setKind(configKind);
cfg.setConfig(Constants.SCHEMA_VERSION, this._serviceInfo.getVersion());
// persist configuration
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), cfg);
config = cfg;
}
return config;
}
// check and initialize versioned configuration
private Configuration checkVersionedConfiguration() {
String serviceVersion = _serviceInfo.getVersion();
String dbSchemaVersion = _dbClient.getSchemaVersion();
if (!serviceVersion.equals(dbSchemaVersion)) {
_log.warn(
"The db service version {} doesn't equals Db schema version {}, "
+ "set db service version to Db schema version",
serviceVersion,
dbSchemaVersion);
_serviceInfo.setVersion(dbSchemaVersion);
}
String kind =
_coordinator.getVersionedDbConfigPath(_serviceInfo.getName(), _serviceInfo.getVersion());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), kind, _serviceInfo.getId());
if (config == null) {
// check if it is upgraded from previous version to yoda - configuration may be stored in
// znode /config
config = _coordinator.queryConfiguration(kind, _serviceInfo.getId());
if (config != null) {
_log.info("Upgrade from pre-2.5 release, move versioned dbconfig to new location");
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
return config;
}
ConfigurationImpl cfg = new ConfigurationImpl();
cfg.setId(_serviceInfo.getId());
cfg.setKind(kind);
// persist configuration
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), cfg);
config = cfg;
}
return config;
}
/**
* Checks and sets INIT_DONE state this means we are done with the actual cf changes on the
* cassandra side for the target version
*/
private void setDbConfigInitDone() {
String configKind =
_coordinator.getVersionedDbConfigPath(_serviceInfo.getName(), _serviceInfo.getVersion());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config != null) {
if (config.getConfig(DbConfigConstants.INIT_DONE) == null) {
config.setConfig(DbConfigConstants.INIT_DONE, Boolean.TRUE.toString());
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
} else {
// we are expecting this to exist, because its initialized from checkVersionedConfiguration
throw new IllegalStateException("unexpected error, db versioned configuration is null");
}
}
private InterProcessLock getLock(String name) throws Exception {
InterProcessLock lock = null;
while (true) {
try {
lock = _coordinator.getSiteLocalLock(name);
lock.acquire();
break; // got lock
} catch (Exception e) {
if (_coordinator.isConnected()) {
throw e;
}
}
}
return lock;
}
public void vcenterClusterAddHostOperation(
URI vcenterId, URI vcenterDataCenterId, URI clusterId, URI hostId, String stepId) {
VcenterApiClient vcenterApiClient = null;
try {
WorkflowStepCompleter.stepExecuting(stepId);
VcenterDataCenter vcenterDataCenter =
_dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
Host host = _dbClient.queryObject(Host.class, hostId);
vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
String key =
vcenterApiClient.addHost(
vcenterDataCenter.getLabel(),
cluster.getExternalId(),
host.getHostName(),
host.getUsername(),
host.getPassword());
_log.info("Successfully added or located host " + host.getHostName() + " " + key);
host.setVcenterDataCenter(vcenterDataCenter.getId());
_dbClient.updateAndReindexObject(host);
WorkflowStepCompleter.stepSucceded(stepId);
} catch (Exception e) {
_log.error("vcenterClusterAddHostOperation exception " + e);
WorkflowStepCompleter.stepFailed(
stepId, VcenterControllerException.exceptions.hostException(e.getLocalizedMessage(), e));
} finally {
if (vcenterApiClient != null) vcenterApiClient.destroy();
}
}
/**
* Retrieve the vdc config info of the current site, return such info with precheck. 1. For adding
* a new vdc, the current vdc should be in ISOLATED status and is a fresh installation. 2. For
* updating an existing vdc, the current vdc should be in CONNECTED status.
*
* @param checkParam
* @return VirtualDataCenterResponse
*/
@POST
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Path("/precheck")
public VdcPreCheckResponse precheckVdcConfig(VdcPreCheckParam checkParam) {
log.info("Start vdc config precheck for {} ...", checkParam.getConfigChangeType());
if (service.getId().endsWith("standalone")) {
throw GeoException.fatals.remoteVDCWrongStandaloneInstall();
}
log.info("Loading local vdc config ...");
VirtualDataCenter vdc = VdcUtil.getLocalVdc();
Boolean isFresher = checkParam.getFresher();
if (isFresher != null && isFresher) {
// check if VDC is a fresh installation, in ISOLATED status
if (VirtualDataCenter.ConnectionStatus.ISOLATED != vdc.getConnectionStatus()) {
throw GeoException.fatals.remoteFreshVDCWrongStatus(vdc.getId());
}
} else {
// check if VDC is in CONNECTED status
// check if VDC is in CONNECTED status- remove, add; update will skip-CTRL3549
if (checkParam.getConfigChangeType().equals(VdcConfig.ConfigChangeType.CONNECT_VDC.toString())
|| (checkParam
.getConfigChangeType()
.equals(VdcConfig.ConfigChangeType.REMOVE_VDC.toString()))) {
if (vdc.getConnectionStatus() != VirtualDataCenter.ConnectionStatus.CONNECTED) {
throw GeoException.fatals.remoteVDCWrongOperationStatus(
vdc.getId(), checkParam.getConfigChangeType());
}
}
}
boolean hasData = false;
if (isFresher) {
hasData = hasDataInDb();
}
hasData |= hasDataService();
log.info("Checking software version ...");
SoftwareVersion remoteSoftVer = null;
try {
remoteSoftVer = new SoftwareVersion(checkParam.getSoftwareVersion());
log.info("Software version of remote vdc: {}", remoteSoftVer);
} catch (Exception e) {
log.info(
"Cannot get software version from checkParam, the version of remote vdc is lower than v2.3 with exception {}",
e.getMessage());
}
SoftwareVersion localSoftVer;
try {
localSoftVer = coordinator.getTargetInfo(RepositoryInfo.class).getCurrentVersion();
} catch (Exception ex) {
throw GeoException.fatals.remoteVDCFailedToGetVersion(vdc.getId());
}
return toVirtualDataCenterResponse(vdc, hasData, remoteSoftVer, localSoftVer);
}
@Override
public void removeVcenterCluster(URI datacenterUri, URI clusterUri) throws InternalException {
VcenterApiClient vcenterApiClient = null;
try {
VcenterDataCenter vcenterDataCenter =
_dbClient.queryObject(VcenterDataCenter.class, datacenterUri);
Cluster cluster = _dbClient.queryObject(Cluster.class, clusterUri);
Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterDataCenter.getVcenter());
_log.info(
"Request to remove cluster "
+ vcenter.getLabel()
+ "/"
+ vcenterDataCenter.getLabel()
+ "/"
+ cluster.getLabel());
vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
vcenterApiClient.removeCluster(vcenterDataCenter.getLabel(), cluster.getExternalId());
} catch (VcenterObjectConnectionException e) {
throw VcenterControllerException.exceptions.objectConnectionException(
e.getLocalizedMessage(), e);
} catch (VcenterObjectNotFoundException e) {
throw VcenterControllerException.exceptions.objectNotFoundException(
e.getLocalizedMessage(), e);
} catch (VcenterServerConnectionException e) {
throw VcenterControllerException.exceptions.serverConnectionException(
e.getLocalizedMessage(), e);
} catch (Exception e) {
_log.error("removeVcenterCluster exception " + e);
throw VcenterControllerException.exceptions.unexpectedException(e.getLocalizedMessage(), e);
} finally {
if (vcenterApiClient != null) vcenterApiClient.destroy();
}
}
/**
* Invoke a failure if the artificial_failure variable is set. This is an internal-only setting
* that allows testers and automated suites to inject a failure into a workflow step at key
* locations to test rollback and Task states.
*
* @param coordinator coordinator
* @param failureKey key from above
*/
public static void internalOnlyInvokeTestFailure(String failureKey) {
// Invoke an artificial failure, if set (experimental, testing only)
String invokeArtificialFailure = _coordinator.getPropertyInfo().getProperty(ARTIFICIAL_FAILURE);
if (invokeArtificialFailure != null && invokeArtificialFailure.equalsIgnoreCase(failureKey)) {
throw new NullPointerException("Artificially Thrown Exception");
}
}
private boolean isStaleConfiguration(Configuration config) {
String delimiter = "-";
String configId = config.getId();
// Bypasses item of "global" and folders of "version", just check db configurations.
if (configId == null || configId.equals(Constants.GLOBAL_ID) || !configId.contains(delimiter)) {
return false;
}
if (_serviceInfo.getId().endsWith(Constants.STANDALONE_ID)) {
if (!configId.equals(_serviceInfo.getId())) {
return true;
}
} else {
CoordinatorClientInetAddressMap nodeMap = _coordinator.getInetAddessLookupMap();
int nodeCount = nodeMap.getControllerNodeIPLookupMap().size();
String nodeIndex = configId.split(delimiter)[1];
if (Constants.STANDALONE_ID.equalsIgnoreCase(nodeIndex)
|| Integer.parseInt(nodeIndex) > nodeCount) {
return true;
}
}
return false;
}
public void createUpdateVcenterClusterOperation(
boolean createCluster, URI vcenterId, URI vcenterDataCenterId, URI clusterId, String stepId) {
VcenterApiClient vcenterApiClient = null;
try {
WorkflowStepCompleter.stepExecuting(stepId);
VcenterDataCenter vcenterDataCenter =
_dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
String vcenterClusterId = null;
if (createCluster) {
_log.info("Create cluster with name " + cluster.getLabel());
vcenterClusterId =
vcenterApiClient.createCluster(vcenterDataCenter.getLabel(), cluster.getLabel());
} else {
// Use MoRef if present but don't stop if we fail to find cluster based off this because
// we'll fall back and try on name
String externalId = cluster.getExternalId();
if (externalId != null && !externalId.trim().equals("")) {
_log.info("Update cluster with MoRef " + externalId);
try {
vcenterClusterId =
vcenterApiClient.updateCluster(vcenterDataCenter.getLabel(), externalId);
} catch (VcenterObjectNotFoundException e) {
_log.info(
"Ignore VcenterObjectNotFoundException updateCluster when using MoRef... Try name based search next");
}
}
if (vcenterClusterId == null) {
_log.info("Update cluster with name " + cluster.getLabel());
vcenterClusterId =
vcenterApiClient.updateCluster(vcenterDataCenter.getLabel(), cluster.getLabel());
}
}
_log.info(
"Successfully created or updated cluster " + cluster.getLabel() + " " + vcenterClusterId);
cluster.setVcenterDataCenter(vcenterDataCenter.getId());
cluster.setExternalId(vcenterClusterId);
_dbClient.updateAndReindexObject(cluster);
WorkflowStepCompleter.stepSucceded(stepId);
} catch (Exception e) {
_log.error("createUpdateVcenterClusterOperation exception " + e);
WorkflowStepCompleter.stepFailed(
stepId,
VcenterControllerException.exceptions.clusterException(e.getLocalizedMessage(), e));
} finally {
if (vcenterApiClient != null) vcenterApiClient.destroy();
}
}
/** Check offline event info to see if dbsvc/geodbsvc on this node could get started */
private void checkDBOfflineInfo() {
Configuration config =
_coordinator.queryConfiguration(
_coordinator.getSiteId(), Constants.DB_DOWNTIME_TRACKER_CONFIG, _serviceInfo.getName());
DbOfflineEventInfo dbOfflineEventInfo = new DbOfflineEventInfo(config);
String localNodeId = _coordinator.getInetAddessLookupMap().getNodeId();
Long lastActiveTimestamp = dbOfflineEventInfo.geLastActiveTimestamp(localNodeId);
long zkTimeStamp =
(lastActiveTimestamp == null) ? TimeUtils.getCurrentTime() : lastActiveTimestamp;
File localDbDir = new File(dbDir);
Date lastModified = getLastModified(localDbDir);
boolean isDirEmpty = lastModified == null || localDbDir.list().length == 0;
long localTimeStamp = (isDirEmpty) ? TimeUtils.getCurrentTime() : lastModified.getTime();
_log.info("Service timestamp in ZK is {}, local file is: {}", zkTimeStamp, localTimeStamp);
long diffTime = (zkTimeStamp > localTimeStamp) ? (zkTimeStamp - localTimeStamp) : 0;
if (diffTime >= MAX_SERVICE_OUTAGE_TIME) {
String errMsg =
String.format(
"We detect database files on local disk are more than %s days older "
+ "than last time it was seen in the cluster. It may bring stale data into the database, "
+ "so the service cannot continue to boot. It may be the result of a VM snapshot rollback. "
+ "Please contact with EMC support engineer for solution.",
diffTime / TimeUtils.DAYS);
alertLog.error(errMsg);
throw new IllegalStateException(errMsg);
}
Long offlineTime = dbOfflineEventInfo.getOfflineTimeInMS(localNodeId);
if (!isDirEmpty && offlineTime != null && offlineTime >= MAX_SERVICE_OUTAGE_TIME) {
String errMsg =
String.format(
"This node is offline for more than %s days. It may bring stale data into "
+ "database, so the service cannot continue to boot. Please poweroff this node and follow our "
+ "node recovery procedure to recover this node",
offlineTime / TimeUtils.DAYS);
alertLog.error(errMsg);
throw new IllegalStateException(errMsg);
}
}
// Find a host connected and powered on then refresh it
public void vcenterClusterSelectHostOperation(
URI vcenterId, URI vcenterDataCenterId, URI clusterId, URI[] hostUris, String stepId) {
VcenterApiClient vcenterApiClient = null;
try {
WorkflowStepCompleter.stepExecuting(stepId);
VcenterDataCenter vcenterDataCenter =
_dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
Collection<Host> hosts = _dbClient.queryObject(Host.class, hostUris);
vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
Host hostForStorageOperations = null;
for (Host host : hosts) {
try {
vcenterApiClient.checkHostConnectedPoweredOn(
vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName());
hostForStorageOperations = host;
_log.info("Host " + host.getHostName() + " to be used for storage operations");
break;
} catch (Exception e) {
_log.info(
"Host "
+ host.getHostName()
+ " not valid for storage operations due to exception "
+ e.getLocalizedMessage());
}
}
if (hostForStorageOperations == null) {
_log.error(
"No host valid for performing storage operations thus cannot perform datastore operations");
throw new Exception(
"No host valid for performing storage operations thus cannot perform datastore operations");
}
vcenterApiClient.refreshRescanHostStorage(
vcenterDataCenter.getLabel(),
cluster.getExternalId(),
hostForStorageOperations.getHostName());
// persist hostForStorageOperations ID in wf data
_workflowService.storeStepData(stepId, hostForStorageOperations.getId());
WorkflowStepCompleter.stepSucceded(stepId);
} catch (Exception e) {
_log.error("vcenterClusterSelectHostOperation exception " + e);
WorkflowStepCompleter.stepFailed(
stepId, VcenterControllerException.exceptions.hostException(e.getLocalizedMessage(), e));
} finally {
if (vcenterApiClient != null) vcenterApiClient.destroy();
}
}
/** @return true if there are data services */
private boolean hasDataService() {
try {
List<Service> services =
coordinator.locateAllServices(DataServiceName, DataServiceVersion, null, null);
if (services.isEmpty()) {
return false;
}
} catch (Exception e) {
return false;
}
return true;
}
private List<Service> getServiceInfoListInternal() {
LOG.debug("Retrieving " + SA_SVC_NAME + " service info from coordinator service");
try {
List<Service> services =
coordinator.locateAllServices(SA_SVC_NAME, SA_SVC_VERSION, null, null);
if (services.isEmpty()) {
throw new RuntimeException("No endpoint found for " + SA_SVC_NAME, null);
}
return services;
} catch (Exception e) {
throw new RuntimeException("Error whilst fetch " + SA_SVC_NAME + " information", e);
}
}
public void vcenterClusterCreateDatastoreOperation(
URI vcenterId,
URI vcenterDataCenterId,
URI clusterId,
URI volumeId,
String selectHostStepId,
String stepId) {
VcenterApiClient vcenterApiClient = null;
try {
WorkflowStepCompleter.stepExecuting(stepId);
URI hostId = (URI) _workflowService.loadStepData(selectHostStepId);
if (hostId == null) {
_log.error("Workflow loadStepData on " + selectHostStepId + " is null");
throw new Exception("Workflow loadStepData on " + selectHostStepId + " is null");
}
VcenterDataCenter vcenterDataCenter =
_dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
Host host = _dbClient.queryObject(Host.class, hostId);
Volume volume = _dbClient.queryObject(Volume.class, volumeId);
vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
String key =
vcenterApiClient.createDatastore(
vcenterDataCenter.getLabel(),
cluster.getExternalId(),
host.getHostName(),
volume.getWWN(),
volume.getLabel());
_log.info("Successfully created or located datastore " + volume.getLabel() + " " + key);
host.setVcenterDataCenter(vcenterDataCenter.getId());
_dbClient.updateAndReindexObject(host);
WorkflowStepCompleter.stepSucceded(stepId);
} catch (Exception e) {
_log.error("vcenterClusterCreateDatastoreOperation exception " + e);
WorkflowStepCompleter.stepFailed(
stepId, VcenterControllerException.exceptions.hostException(e.getLocalizedMessage(), e));
} finally {
if (vcenterApiClient != null) vcenterApiClient.destroy();
}
}
private void removeStaleServiceConfiguration() {
boolean isGeoDBSvc = isGeoDbsvc();
boolean resetAutoBootFlag = false;
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
List<Configuration> configs =
_coordinator.queryAllConfiguration(_coordinator.getSiteId(), configKind);
for (Configuration config : configs) {
if (isStaleConfiguration(config)) {
boolean autoboot = Boolean.parseBoolean(config.getConfig(DbConfigConstants.AUTOBOOT));
String configId = config.getId();
if (isGeoDBSvc && !autoboot && (configId.equals("geodb-4") || configId.equals("geodb-5"))) {
// for geodbsvc, if restore with the backup of 5 nodes to 3 nodes and the backup is made
// on the cluster that the 'autoboot=false' is set on vipr4 or vipr5
// we should set the autoboot=false on the current node or no node with autoboot=false
// TODO:This is a temporary/safest solution in Yoda, we'll provide a better soltuion post
// Yoda
resetAutoBootFlag = true;
}
if (isStaleConfiguration(config)) {
_coordinator.removeServiceConfiguration(_coordinator.getSiteId(), config);
_log.info("Remove stale db config, id: {}", config.getId());
}
}
}
if (resetAutoBootFlag) {
_log.info("set autoboot flag to false on {}", _serviceInfo.getId());
Configuration config =
_coordinator.queryConfiguration(
_coordinator.getSiteId(), configKind, _serviceInfo.getId());
config.setConfig(DbConfigConstants.AUTOBOOT, Boolean.FALSE.toString());
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
}
private List<String> getVirtualMachines(
URI datacenterUri, URI clusterUri, URI hostUri, boolean runningOnly)
throws InternalException {
VcenterApiClient vcenterApiClient = null;
try {
Host host = _dbClient.queryObject(Host.class, hostUri);
VcenterDataCenter vcenterDataCenter =
_dbClient.queryObject(VcenterDataCenter.class, datacenterUri);
Cluster cluster = _dbClient.queryObject(Cluster.class, clusterUri);
Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterDataCenter.getVcenter());
_log.info(
"Request to get virtual machines for "
+ vcenter.getLabel()
+ "/"
+ vcenterDataCenter.getLabel()
+ "/"
+ cluster.getLabel()
+ "/"
+ host.getHostName());
vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
return runningOnly
? vcenterApiClient.getRunningVirtualMachines(
vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName())
: vcenterApiClient.getVirtualMachines(
vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName());
} catch (VcenterObjectConnectionException e) {
throw VcenterControllerException.exceptions.objectConnectionException(
e.getLocalizedMessage(), e);
} catch (VcenterObjectNotFoundException e) {
throw VcenterControllerException.exceptions.objectNotFoundException(
e.getLocalizedMessage(), e);
} catch (VcenterServerConnectionException e) {
throw VcenterControllerException.exceptions.serverConnectionException(
e.getLocalizedMessage(), e);
} catch (Exception e) {
_log.error("getVirtualMachines exception " + e);
throw VcenterControllerException.exceptions.unexpectedException(e.getLocalizedMessage(), e);
} finally {
if (vcenterApiClient != null) vcenterApiClient.destroy();
}
}
/** Setup geosvc job queue */
public void start() {
_log.info("Get vipr keystore");
try {
viprKeyStore = KeyStoreUtil.getViPRKeystore(_coordinator);
} catch (Exception e) {
_log.error("Failed to load the VIPR keystore", e);
throw new IllegalStateException(e);
}
_log.info("Starting geosvc job queue");
try {
_queue =
_coordinator.getQueue(
GeoServiceHelper.GEOSVC_QUEUE_NAME,
this,
new GeoServiceJobSerializer(),
GeoServiceHelper.DEFAULT_MAX_THREADS);
} catch (Exception e) {
_log.error("can not startup geosvc job queue", e);
}
}
/**
* Attempts to acquire the passed lock.
*
* @param lockName The name of the lock to acquire.
* @param waitInSeconds The amount of time to wait to acquire the lock in seconds. A value less
* than 0 will cause the function to wait indefinitely for the lock.
* @return true if lock acquired, false otherwise.
*/
public boolean acquireLock(String lockName, long waitInSeconds) {
if (lockName == null || lockName.isEmpty()) {
s_logger.info("No lock name specified.");
return false;
}
try {
InterProcessLock lock = _coordinator.getLock(lockName);
if (lock != null) {
if (waitInSeconds >= 0) {
s_logger.info(
"Attempting to acquire lock: "
+ lockName
+ " for a maximum of "
+ waitInSeconds
+ " seconds.");
if (!lock.acquire(waitInSeconds, TimeUnit.SECONDS)) {
s_logger.info("Failed to acquire lock: " + lockName);
return false;
}
} else {
s_logger.info("Attempting to acquire lock: " + lockName + " for as long as it takes.");
lock.acquire(); // will only throw exception or pass
}
s_acquiredLocks.put(lockName, lock);
} else {
return false;
}
s_logger.info("Acquired lock: " + lockName);
return true;
} catch (Exception e) {
s_logger.error("Acquisition of lock: {} failed with Exception: ", lockName, e);
return false;
}
}
@Override
public void start() throws IOException {
if (_log.isInfoEnabled()) {
_log.info("Starting DB service...");
}
// Suppress Sonar violation of Lazy initialization of static fields should be synchronized
// start() method will be only called one time when startup dbsvc, so it's safe to ignore sonar
// violation
instance = this; // NOSONAR ("squid:S2444")
if (backCompatPreYoda) {
_log.info(
"Pre-yoda back compatible flag detected. Initialize local keystore/truststore for Cassandra native encryption");
initKeystoreAndTruststore();
_schemaUtil.setBackCompatPreYoda(true);
}
System.setProperty("cassandra.config", _config);
System.setProperty("cassandra.config.loader", CassandraConfigLoader.class.getName());
// Set to false to clear all gossip state for the node on restart.
//
// We encounter a weird Cassandra grossip issue(COP-19246) - some nodes are missing from gossip
// when rebooting the entire cluster simultaneously. Critical Gossip
// fields(ApplicationState.STATUS, ApplicationState.TOKENS)
// are not synchronized during handshaking. It looks like some problem caused by incorrect
// gossip version/generation
// at system local table. So add this option to cleanup local gossip state during reboot
//
// Make sure add-vdc/add-standby passed when you would remove this option in the future.
//
// We need make sure majority local nodes are added as seed nodes. Otherwise cassandra may not
// see other nodes if it loses
// connection to other sites
System.setProperty("cassandra.load_ring_state", "false");
// Nodes in new data center should not auto-bootstrap.
// See
// https://docs.datastax.com/en/cassandra/2.0/cassandra/operations/ops_add_dc_to_cluster_t.html
if (_schemaUtil.isStandby()) {
System.setProperty("cassandra.auto_bootstrap", "false");
}
InterProcessLock lock = null;
Configuration config = null;
StartupMode mode = null;
try {
// we use this lock to discourage more than one node bootstrapping / joining at the same time
// Cassandra can handle this but it's generally not recommended to make changes to schema
// concurrently
lock = getLock(getSchemaLockName());
config = checkConfiguration();
checkGlobalConfiguration();
checkVersionedConfiguration();
removeStaleConfiguration();
mode = checkStartupMode(config);
_log.info("Current startup mode is {}", mode);
// Check if service is allowed to get started by querying db offline info to avoid bringing
// back stale data.
// Skipping hibernate mode for node recovery procedure to recover the overdue node.
int nodeCount = ((CoordinatorClientImpl) _coordinator).getNodeCount();
if (nodeCount != 1 && mode.type != StartupMode.StartupModeType.HIBERNATE_MODE) {
checkDBOfflineInfo();
}
// this call causes instantiation of a seed provider instance, so the check*Configuration
// calls must be preceed it
removeCassandraSavedCaches();
mode.onPreStart();
if (_jmxServer != null) {
_jmxServer.start();
System.setProperty(
"com.sun.management.jmxremote.port", Integer.toString(_jmxServer.getPort()));
}
_service = new CassandraDaemon();
_service.init(null);
_service.start();
cassandraInitialized = true;
mode.onPostStart();
} catch (Exception e) {
if (mode != null && mode.type == StartupMode.StartupModeType.HIBERNATE_MODE) {
printRecoveryWorkAround(e);
}
_log.error("e=", e);
throw new IllegalStateException(e);
} finally {
if (lock != null) {
try {
lock.release();
} catch (Exception ignore) {
_log.debug("lock release failed");
}
}
}
if (config.getConfig(DbConfigConstants.JOINED) == null) {
config.setConfig(DbConfigConstants.JOINED, Boolean.TRUE.toString());
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
}
_statusChecker.waitForAllNodesJoined();
_svcBeacon.start();
if (backCompatPreYoda) {
_log.info("Enable duplicated beacon in global area during pre-yoda upgrade");
startDupBeacon();
}
setDbInitializedFlag();
setDbConfigInitDone();
_dbClient.start();
if (_schemaUtil.isStandby()) {
String localDataRevision = getLocalDataRevision();
if (localDataRevision != null) {
_schemaUtil.checkDataRevision(localDataRevision);
}
}
// Setup the vdc information, so that login enabled before migration
if (!isGeoDbsvc()) {
_schemaUtil.checkAndSetupBootStrapInfo(_dbClient);
}
dbMgr.init();
if (_handler.run()) {
// Setup the bootstrap info root tenant, if root tenant migrated from local db, then skip it
if (isGeoDbsvc()) {
_schemaUtil.checkAndSetupBootStrapInfo(_dbClient);
} else {
_schemaUtil.checkAndInitStorageSystemTypes(_dbClient);
}
startBackgroundTasks();
_log.info("DB service started");
} else {
_log.error("DB migration failed. Skipping starting background tasks.");
}
}
/**
* Remove given flag from current db config in zk
*
* @param flagName
*/
void removeFlag(String flagName) {
config.removeConfig(flagName);
coordinator.persistServiceConfiguration(config);
}
/**
* Checks and registers db configuration information, this is one time when cluster is coming up
* for the first time
*/
private Configuration checkConfiguration() {
String configKind = _coordinator.getDbConfigPath(_serviceInfo.getName());
Configuration config =
_coordinator.queryConfiguration(_coordinator.getSiteId(), configKind, _serviceInfo.getId());
if (config == null) {
// check if it is upgraded from previous version to yoda - configuration may be stored in
// zk global area /config. Since SeedProvider still need access that, so we remove the config
// from global in migration callback after migration is done.
config = _coordinator.queryConfiguration(configKind, _serviceInfo.getId());
if (config != null) {
_log.info("Upgrade from pre-yoda release, move dbconfig to new location");
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), config);
return config;
}
// this is a new node
// 1. register its configuration with coordinator
// 2. assume autobootstrap configuration
// this means that when a node is added, it take 1/2 of biggest token rage and
// copies its data over
ConfigurationImpl cfg = new ConfigurationImpl();
cfg.setId(_serviceInfo.getId());
cfg.setKind(configKind);
cfg.setConfig(DbConfigConstants.NODE_ID, _coordinator.getInetAddessLookupMap().getNodeId());
cfg.setConfig(DbConfigConstants.AUTOBOOT, Boolean.TRUE.toString());
// check other existing db nodes
List<Configuration> configs =
_coordinator.queryAllConfiguration(_coordinator.getSiteId(), configKind);
if (configs.isEmpty()) {
// we are the first node - turn off autobootstrap
cfg.setConfig(DbConfigConstants.AUTOBOOT, Boolean.FALSE.toString());
}
// persist configuration
_coordinator.persistServiceConfiguration(_coordinator.getSiteId(), cfg);
config = cfg;
}
return config;
}
