/**
* Update vdc properties and reboot the node if
*
* @param svcId node service id
* @throws Exception
*/
private void updateVdcProperties(String svcId) throws Exception {
// If the change is being done to create a multi VDC configuration or to reduce to a
// multi VDC configuration a reboot is needed. If only operating on a single VDC
// do not reboot the nodes.
if (targetVdcPropInfo.getProperty(VdcConfigUtil.VDC_IDS).contains(",")
|| localVdcPropInfo.getProperty(VdcConfigUtil.VDC_IDS).contains(",")) {
log.info("Step4: Acquiring property lock for vdc properties change.");
if (!getPropertyLock(svcId)) {
retrySleep();
} else if (!isQuorumMaintained()) {
try {
coordinator.releasePersistentLock(svcId, propertyLockId);
} catch (Exception e) {
log.error("Failed to release the property lock:", e);
}
retrySleep();
} else {
log.info("Step4: Setting vdc properties and rebooting for multi-vdc config change");
localRepository.setVdcPropertyInfo(targetVdcPropInfo);
reboot();
}
} else {
log.info("Step4: Setting vdc properties not rebooting for single VDC change");
localRepository.setVdcPropertyInfo(targetVdcPropInfo);
}
}
/**
* Combine nodeScopePropInfo with targetPropInfo
*
* @param targetPropInfo target property info
* @param nodeScopePropInfo node scope property info
* @return combined property info
*/
private PropertyInfoExt combineProps(
final PropertyInfoExt targetPropInfo, final PropertyInfoExt nodeScopePropInfo) {
PropertyInfoExt combinedProps = new PropertyInfoExt();
for (Entry<String, String> entry : targetPropInfo.getAllProperties().entrySet()) {
combinedProps.addProperty(entry.getKey(), entry.getValue());
}
for (Entry<String, String> entry : nodeScopePropInfo.getAllProperties().entrySet()) {
combinedProps.addProperty(entry.getKey(), entry.getValue());
}
return combinedProps;
}
/**
* Get node scope properties UpgradeManager will publish the node scope properties as node
* information into coordinator Node scope properties are invariants.
*
* <p>We check to see if a property is in metadata or not. If it is, it is a target property; If
* not, it is a local property
*
* @param localPropInfo local property info read from /etc/config.properties
* @return node scope properties
*/
private PropertyInfoExt getNodeScopeProperties(final PropertyInfoExt localPropInfo) {
Map<String, PropertyMetadata> metadata = PropertiesMetadata.getGlobalMetadata();
PropertyInfoExt localScopeProps = new PropertyInfoExt();
for (Entry<String, String> entry : localPropInfo.getAllProperties().entrySet()) {
final String key = entry.getKey();
final String value = entry.getValue();
if (!metadata.containsKey(key)) {
localScopeProps.addProperty(key, value);
}
}
return localScopeProps;
}
/**
* Check if VDC configuration is different in the database vs. what is stored locally
*
* @return
*/
private boolean vdcPropertiesChanged() {
if (!coordinator.isControlNode()) {
return false;
}
int localVdcConfigHashcode =
localVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE) == null
? 0
: Integer.parseInt(localVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE));
int targetVdcConfigHashcode =
targetVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE) == null
? 0
: Integer.parseInt(targetVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE));
return localVdcConfigHashcode != targetVdcConfigHashcode;
}
private void notifyPropertyChanges(PropertyInfoExt diffProperties) {
List<String> notifierTags = diffProperties.getNotifierTags();
for (String notifierTag : notifierTags) {
log.info("Step3a: Calling notifier {}", notifierTag);
try {
Notifier notifier = Notifier.getInstance(notifierTag);
if (notifier != null) notifier.doNotify();
} catch (Exception e) {
log.error("Step3a: Fail to invoke notifier {}", notifierTag, e);
}
}
}
private void reconfigProperties(PropertyInfoExt diffProperties) {
// only get the notifiers that requires reconfig as well
List<String> notifierTagList = diffProperties.getNotifierTags(true);
String notifierTags = StringUtils.join(notifierTagList, " ");
log.info("Step3a: Reconfiguring properties related to {}", notifierTags);
try {
localRepository.reconfigProperties(notifierTags);
} catch (Exception e) {
log.error("Step3a: Fail to reconfig properties related to {}", notifierTags, e);
}
}
/**
* Get local target property info
*
* <p>For control node, properties that can be found in metadata are target properties For extra
* node, not only exist in metadata, but also ControlNodeOnly==false
*
* @param localPropInfo
* @return
*/
private PropertyInfoExt getLocalTargetPropInfo(final PropertyInfoExt localPropInfo) {
Map<String, PropertyMetadata> metadata = PropertiesMetadata.getGlobalMetadata();
PropertyInfoExt localTargetProps = new PropertyInfoExt();
for (Entry<String, String> entry : localPropInfo.getAllProperties().entrySet()) {
final String key = entry.getKey();
final String value = entry.getValue();
if (metadata.containsKey(key)) {
if (coordinator.isControlNode()) {
localTargetProps.addProperty(key, value);
} else {
if (!metadata.get(key).getControlNodeOnly()) {
localTargetProps.addProperty(key, value);
}
}
}
}
return localTargetProps;
}
/**
* * Update SSL property
*
* @param state
* @throws LocalRepositoryException
*/
public void setSslPropertyInfo(PropertyInfoExt state) throws LocalRepositoryException {
final String prefix = "setSslPropertyInfo(): to=" + state;
_log.debug(prefix);
final Path tmpFilePath = FileSystems.getDefault().getPath(SSL_PROPERTY_TMP);
createTmpFile(tmpFilePath, state.toString(false), prefix);
try {
final String[] cmd = {_SYSTOOL_CMD, _SYSTOOL_SET_SSL_PROPS, SSL_PROPERTY_TMP};
exec(prefix, cmd);
_log.info(prefix + "Success");
} finally {
cleanupTmpFile(tmpFilePath);
}
}
/**
* * Update property
*
* @param state
* @throws LocalRepositoryException
*/
public void setOverrideProperties(PropertyInfoExt state) throws LocalRepositoryException {
final String prefix = "setOverrideProperties(): to=" + state;
_log.debug(prefix);
final Path tmpFilePath = FileSystems.getDefault().getPath(TMP_CONFIG_USER_CHANGED_PROPS_PATH);
createTmpFile(tmpFilePath, state.toString(false), prefix);
try {
final String[] cmd = {_SYSTOOL_CMD, _SYSTOOL_SET_OPROPS, TMP_CONFIG_USER_CHANGED_PROPS_PATH};
exec(prefix, cmd);
_log.info(prefix + "Success");
} finally {
cleanupTmpFile(tmpFilePath);
}
}
/**
* * Update data revision properties to local
*
* @param localRevisionProps
* @throws LocalRepositoryException
*/
public void setDataRevisionPropertyInfo(PropertyInfoExt localRevisionProps)
throws LocalRepositoryException {
final String prefix =
String.format("setDataRevisionPropertyInfo(): to=%s ", localRevisionProps);
_log.debug(prefix);
final Path tmpFilePath = FileSystems.getDefault().getPath(DATA_REVISION_TMP);
createTmpFile(tmpFilePath, localRevisionProps.toString(false), prefix);
try {
final String[] cmd = {_SYSTOOL_CMD, _SYSTOOL_SET_DATA_REVISION, DATA_REVISION_TMP};
exec(prefix, cmd);
_log.info(prefix + " Success");
} finally {
cleanupTmpFile(tmpFilePath);
}
}
/**
* Update properties
*
* @param svcId node service id
* @throws Exception
*/
private void updateProperties(String svcId) throws Exception {
if (targetPropInfo.TARGET_PROPERTY.equals(targetPropInfo.OLD_TARGET_PROPERTY)) {
coordinator.removeTargetInfo(targetPropInfo, true);
}
PropertyInfoExt diffProperties =
new PropertyInfoExt(targetPropInfo.getDiffProperties(localTargetPropInfo));
PropertyInfoExt override_properties =
new PropertyInfoExt(localRepository.getOverrideProperties().getAllProperties());
log.info("Step3a: Updating User Changed properties file: {}", override_properties);
PropertyInfoExt updatedUserChangedProps = combineProps(override_properties, diffProperties);
if (diffProperties.hasRebootProperty()) {
if (!getPropertyLock(svcId)) {
retrySleep();
} else if (!isQuorumMaintained()) {
try {
coordinator.releasePersistentLock(svcId, propertyLockId);
} catch (Exception e) {
log.error("Failed to release the property lock:", e);
}
retrySleep();
} else {
log.info("Step3a: Reboot property found.");
localRepository.setOverrideProperties(updatedUserChangedProps);
log.info("Step3a: Updating properties: {}", updatedUserChangedProps);
reboot();
}
} else if (diffProperties.hasReconfigProperty()
|| !diffProperties.getNotifierTags().isEmpty()) {
log.info("Step3a: Reconfig property found or notifiers specified.");
// CTRL-9860: don't update the local config version until everything is done.
String targetVersion = targetPropInfo.getProperty(PropertyInfoExt.CONFIG_VERSION);
updatedUserChangedProps.addProperty(PropertyInfoExt.CONFIG_VERSION, localConfigVersion);
localRepository.setOverrideProperties(updatedUserChangedProps);
log.info(
"Step3a: Updating properties without updating the config version: {}",
updatedUserChangedProps);
if (diffProperties.hasReconfigAttributeWithoutNotifiers()) {
// this is the old-school "complete" reconfig that takes no notifiers as arguments.
// moving forward this will diminish
// i.e., all reconfigRequired properties will have notifier specified.
localRepository.reconfig();
} else if (diffProperties.hasReconfigProperty()) {
reconfigProperties(diffProperties);
}
// the notifier list can be empty, in which case nothing will be done.
notifyPropertyChanges(diffProperties);
// update the local config version to target version now
log.info("Step3a: Updating the config version to {}", targetVersion);
updatedUserChangedProps.addProperty(PropertyInfoExt.CONFIG_VERSION, targetVersion);
localRepository.setOverrideProperties(updatedUserChangedProps);
} else {
log.info("Step3a: No reboot property found.");
localRepository.setOverrideProperties(updatedUserChangedProps);
log.info("Step3a: Updating properties: {}", updatedUserChangedProps);
}
}
/**
* Initialize local and target info
*
* @throws Exception
*/
private void initializeLocalAndTargetInfo(String svcId) throws Exception {
// publish config_version which is also a target property
// used as a flag denoting whether target properties have been changed
PropertyInfoExt localPropInfo = localRepository.getOverrideProperties();
localConfigVersion = localPropInfo.getProperty(PropertyInfoExt.CONFIG_VERSION);
if (localConfigVersion == null) {
localConfigVersion = "0";
localPropInfo.addProperty(PropertyInfoExt.CONFIG_VERSION, localConfigVersion);
}
coordinator.setNodeSessionScopeInfo(new ConfigVersion(localConfigVersion));
log.info("Step1a: Local config version: {}", localConfigVersion);
// set node scope properties for the 1st time only since they are invariants.
localNodePropInfo =
coordinator.getNodeGlobalScopeInfo(PropertyInfoExt.class, "propertyinfo", svcId);
// The PropertyInfoExt object will be persisted in the zookeeper path
// config/propertyinfo/(svcId)
if (localNodePropInfo == null) {
localNodePropInfo = getNodeScopeProperties(localPropInfo);
coordinator.setNodeGlobalScopeInfo(localNodePropInfo, "propertyinfo", svcId);
// The PropertyInfoExt object can be fetched from the zookeeper path
// config/propertyinfo/(svcId)
log.info("Step1a: Local node scope properties: {}", localNodePropInfo);
}
// get local target property info
localTargetPropInfo = getLocalTargetPropInfo(localPropInfo);
log.debug("Step1a: Local target properties: {}", localTargetPropInfo);
// set target if empty
targetPropInfo = coordinator.getTargetInfo(PropertyInfoExt.class);
targetPowerOffState = coordinator.getTargetInfo(PowerOffState.class);
if (targetPropInfo == null || targetPowerOffState == null) {
// only control node can set target
try {
// Set the updated propperty info in coordinator
coordinator.setTargetInfo(localPropInfo);
coordinator.setTargetInfo(new PowerOffState(PowerOffState.State.NONE));
targetPropInfo = coordinator.getTargetInfo(PropertyInfoExt.class);
log.info("Step1b: Target property set to local state: {}", targetPropInfo);
targetPowerOffState = coordinator.getTargetInfo(PowerOffState.class);
log.info("Step1b: Target poweroff state set to: {}", PowerOffState.State.NONE);
} catch (CoordinatorClientException e) {
log.info("Step1b: Wait another control node to set target");
retrySleep();
throw e;
}
}
// Initialize vdc prop info
localVdcPropInfo = localRepository.getVdcPropertyInfo();
targetVdcPropInfo = loadVdcConfigFromDatabase();
if (localVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE) == null) {
localRepository.setVdcPropertyInfo(targetVdcPropInfo);
localVdcPropInfo = localRepository.getVdcPropertyInfo();
String vdc_ids = targetVdcPropInfo.getProperty(VDC_IDS_KEY);
String[] vdcIds = vdc_ids.split(",");
if (vdcIds.length > 1) {
log.info("More than one Vdc, so set reboot flag");
shouldReboot = true;
}
}
}
@Override
protected void innerRun() {
final String svcId = coordinator.getMySvcId();
while (doRun) {
log.debug("Main loop: Start");
shortSleep = false;
if (shouldReboot) {
reboot();
}
// Step0: check if we have the property lock
boolean hasLock;
try {
hasLock = coordinator.hasPersistentLock(svcId, propertyLockId);
} catch (Exception e) {
log.info("Step1: Failed to verify if the current node has the property lock ", e);
retrySleep();
continue;
}
if (hasLock) {
try {
coordinator.releasePersistentLock(svcId, propertyLockId);
log.info("Step0: Released property lock for node: {}", svcId);
wakeupOtherNodes();
} catch (InvalidLockOwnerException e) {
log.error("Step0: Failed to release the property lock: Not owner.");
} catch (Exception e) {
log.info("Step0: Failed to release the property lock and will retry: {}", e.getMessage());
retrySleep();
continue;
}
}
// Step1: publish current state, and set target if empty
try {
initializeLocalAndTargetInfo(svcId);
} catch (Exception e) {
log.info("Step1b failed and will be retried: {}", e.getMessage());
retrySleep();
continue;
}
// Step2: power off if all nodes agree.
log.info("Step2: Power off if poweroff state != NONE. {}", targetPowerOffState);
try {
gracefulPoweroffCluster();
} catch (Exception e) {
log.error("Step2: Failed to poweroff. {}", e);
}
// Step3: if target property is changed, update
log.info("Step3: If target property is changed, update");
if (localTargetPropInfo != null
&& targetPropInfo != null
&& !localConfigVersion.equals(
targetPropInfo.getProperty(PropertyInfoExt.CONFIG_VERSION))) {
log.info("Step3a: Current properties are not same as target properties. Updating.");
log.debug("Current local target properties: " + localTargetPropInfo);
log.debug("Target properties: " + targetPropInfo);
try {
updateProperties(svcId);
} catch (Exception e) {
log.info("Step3a: Update failed and will be retried: {}", e.getMessage());
// Restart the loop immediately so that we release the upgrade lock.
continue;
}
continue;
}
log.info("Step4: If VDC configuration is changed update");
if (vdcPropertiesChanged()) {
log.info("Step4: Current vdc properties are not same as target vdc properties. Updating.");
log.debug("Current local vdc properties: " + localVdcPropInfo);
log.debug("Target vdc properties: " + targetVdcPropInfo);
try {
updateVdcProperties(svcId);
} catch (Exception e) {
log.info("Step4: VDC properties update failed and will be retried: {}", e.getMessage());
// Restart the loop immediately so that we release the upgrade lock.
continue;
}
continue;
}
if (shouldReboot == false) {
// Step5: sleep
log.info("Step5: sleep");
longSleep();
}
}
}