/** * Update vdc properties and reboot the node if * * @param svcId node service id * @throws Exception */ private void updateVdcProperties(String svcId) throws Exception { // If the change is being done to create a multi VDC configuration or to reduce to a // multi VDC configuration a reboot is needed. If only operating on a single VDC // do not reboot the nodes. if (targetVdcPropInfo.getProperty(VdcConfigUtil.VDC_IDS).contains(",") || localVdcPropInfo.getProperty(VdcConfigUtil.VDC_IDS).contains(",")) { log.info("Step4: Acquiring property lock for vdc properties change."); if (!getPropertyLock(svcId)) { retrySleep(); } else if (!isQuorumMaintained()) { try { coordinator.releasePersistentLock(svcId, propertyLockId); } catch (Exception e) { log.error("Failed to release the property lock:", e); } retrySleep(); } else { log.info("Step4: Setting vdc properties and rebooting for multi-vdc config change"); localRepository.setVdcPropertyInfo(targetVdcPropInfo); reboot(); } } else { log.info("Step4: Setting vdc properties not rebooting for single VDC change"); localRepository.setVdcPropertyInfo(targetVdcPropInfo); } }
/** * Combine nodeScopePropInfo with targetPropInfo * * @param targetPropInfo target property info * @param nodeScopePropInfo node scope property info * @return combined property info */ private PropertyInfoExt combineProps( final PropertyInfoExt targetPropInfo, final PropertyInfoExt nodeScopePropInfo) { PropertyInfoExt combinedProps = new PropertyInfoExt(); for (Entry<String, String> entry : targetPropInfo.getAllProperties().entrySet()) { combinedProps.addProperty(entry.getKey(), entry.getValue()); } for (Entry<String, String> entry : nodeScopePropInfo.getAllProperties().entrySet()) { combinedProps.addProperty(entry.getKey(), entry.getValue()); } return combinedProps; }
/** * Get node scope properties UpgradeManager will publish the node scope properties as node * information into coordinator Node scope properties are invariants. * * <p>We check to see if a property is in metadata or not. If it is, it is a target property; If * not, it is a local property * * @param localPropInfo local property info read from /etc/config.properties * @return node scope properties */ private PropertyInfoExt getNodeScopeProperties(final PropertyInfoExt localPropInfo) { Map<String, PropertyMetadata> metadata = PropertiesMetadata.getGlobalMetadata(); PropertyInfoExt localScopeProps = new PropertyInfoExt(); for (Entry<String, String> entry : localPropInfo.getAllProperties().entrySet()) { final String key = entry.getKey(); final String value = entry.getValue(); if (!metadata.containsKey(key)) { localScopeProps.addProperty(key, value); } } return localScopeProps; }
/** * Check if VDC configuration is different in the database vs. what is stored locally * * @return */ private boolean vdcPropertiesChanged() { if (!coordinator.isControlNode()) { return false; } int localVdcConfigHashcode = localVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE) == null ? 0 : Integer.parseInt(localVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE)); int targetVdcConfigHashcode = targetVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE) == null ? 0 : Integer.parseInt(targetVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE)); return localVdcConfigHashcode != targetVdcConfigHashcode; }
private void notifyPropertyChanges(PropertyInfoExt diffProperties) { List<String> notifierTags = diffProperties.getNotifierTags(); for (String notifierTag : notifierTags) { log.info("Step3a: Calling notifier {}", notifierTag); try { Notifier notifier = Notifier.getInstance(notifierTag); if (notifier != null) notifier.doNotify(); } catch (Exception e) { log.error("Step3a: Fail to invoke notifier {}", notifierTag, e); } } }
private void reconfigProperties(PropertyInfoExt diffProperties) { // only get the notifiers that requires reconfig as well List<String> notifierTagList = diffProperties.getNotifierTags(true); String notifierTags = StringUtils.join(notifierTagList, " "); log.info("Step3a: Reconfiguring properties related to {}", notifierTags); try { localRepository.reconfigProperties(notifierTags); } catch (Exception e) { log.error("Step3a: Fail to reconfig properties related to {}", notifierTags, e); } }
/** * Get local target property info * * <p>For control node, properties that can be found in metadata are target properties For extra * node, not only exist in metadata, but also ControlNodeOnly==false * * @param localPropInfo * @return */ private PropertyInfoExt getLocalTargetPropInfo(final PropertyInfoExt localPropInfo) { Map<String, PropertyMetadata> metadata = PropertiesMetadata.getGlobalMetadata(); PropertyInfoExt localTargetProps = new PropertyInfoExt(); for (Entry<String, String> entry : localPropInfo.getAllProperties().entrySet()) { final String key = entry.getKey(); final String value = entry.getValue(); if (metadata.containsKey(key)) { if (coordinator.isControlNode()) { localTargetProps.addProperty(key, value); } else { if (!metadata.get(key).getControlNodeOnly()) { localTargetProps.addProperty(key, value); } } } } return localTargetProps; }
/** * * Update SSL property * * @param state * @throws LocalRepositoryException */ public void setSslPropertyInfo(PropertyInfoExt state) throws LocalRepositoryException { final String prefix = "setSslPropertyInfo(): to=" + state; _log.debug(prefix); final Path tmpFilePath = FileSystems.getDefault().getPath(SSL_PROPERTY_TMP); createTmpFile(tmpFilePath, state.toString(false), prefix); try { final String[] cmd = {_SYSTOOL_CMD, _SYSTOOL_SET_SSL_PROPS, SSL_PROPERTY_TMP}; exec(prefix, cmd); _log.info(prefix + "Success"); } finally { cleanupTmpFile(tmpFilePath); } }
/** * * Update property * * @param state * @throws LocalRepositoryException */ public void setOverrideProperties(PropertyInfoExt state) throws LocalRepositoryException { final String prefix = "setOverrideProperties(): to=" + state; _log.debug(prefix); final Path tmpFilePath = FileSystems.getDefault().getPath(TMP_CONFIG_USER_CHANGED_PROPS_PATH); createTmpFile(tmpFilePath, state.toString(false), prefix); try { final String[] cmd = {_SYSTOOL_CMD, _SYSTOOL_SET_OPROPS, TMP_CONFIG_USER_CHANGED_PROPS_PATH}; exec(prefix, cmd); _log.info(prefix + "Success"); } finally { cleanupTmpFile(tmpFilePath); } }
/** * * Update data revision properties to local * * @param localRevisionProps * @throws LocalRepositoryException */ public void setDataRevisionPropertyInfo(PropertyInfoExt localRevisionProps) throws LocalRepositoryException { final String prefix = String.format("setDataRevisionPropertyInfo(): to=%s ", localRevisionProps); _log.debug(prefix); final Path tmpFilePath = FileSystems.getDefault().getPath(DATA_REVISION_TMP); createTmpFile(tmpFilePath, localRevisionProps.toString(false), prefix); try { final String[] cmd = {_SYSTOOL_CMD, _SYSTOOL_SET_DATA_REVISION, DATA_REVISION_TMP}; exec(prefix, cmd); _log.info(prefix + " Success"); } finally { cleanupTmpFile(tmpFilePath); } }
/** * Update properties * * @param svcId node service id * @throws Exception */ private void updateProperties(String svcId) throws Exception { if (targetPropInfo.TARGET_PROPERTY.equals(targetPropInfo.OLD_TARGET_PROPERTY)) { coordinator.removeTargetInfo(targetPropInfo, true); } PropertyInfoExt diffProperties = new PropertyInfoExt(targetPropInfo.getDiffProperties(localTargetPropInfo)); PropertyInfoExt override_properties = new PropertyInfoExt(localRepository.getOverrideProperties().getAllProperties()); log.info("Step3a: Updating User Changed properties file: {}", override_properties); PropertyInfoExt updatedUserChangedProps = combineProps(override_properties, diffProperties); if (diffProperties.hasRebootProperty()) { if (!getPropertyLock(svcId)) { retrySleep(); } else if (!isQuorumMaintained()) { try { coordinator.releasePersistentLock(svcId, propertyLockId); } catch (Exception e) { log.error("Failed to release the property lock:", e); } retrySleep(); } else { log.info("Step3a: Reboot property found."); localRepository.setOverrideProperties(updatedUserChangedProps); log.info("Step3a: Updating properties: {}", updatedUserChangedProps); reboot(); } } else if (diffProperties.hasReconfigProperty() || !diffProperties.getNotifierTags().isEmpty()) { log.info("Step3a: Reconfig property found or notifiers specified."); // CTRL-9860: don't update the local config version until everything is done. String targetVersion = targetPropInfo.getProperty(PropertyInfoExt.CONFIG_VERSION); updatedUserChangedProps.addProperty(PropertyInfoExt.CONFIG_VERSION, localConfigVersion); localRepository.setOverrideProperties(updatedUserChangedProps); log.info( "Step3a: Updating properties without updating the config version: {}", updatedUserChangedProps); if (diffProperties.hasReconfigAttributeWithoutNotifiers()) { // this is the old-school "complete" reconfig that takes no notifiers as arguments. // moving forward this will diminish // i.e., all reconfigRequired properties will have notifier specified. localRepository.reconfig(); } else if (diffProperties.hasReconfigProperty()) { reconfigProperties(diffProperties); } // the notifier list can be empty, in which case nothing will be done. notifyPropertyChanges(diffProperties); // update the local config version to target version now log.info("Step3a: Updating the config version to {}", targetVersion); updatedUserChangedProps.addProperty(PropertyInfoExt.CONFIG_VERSION, targetVersion); localRepository.setOverrideProperties(updatedUserChangedProps); } else { log.info("Step3a: No reboot property found."); localRepository.setOverrideProperties(updatedUserChangedProps); log.info("Step3a: Updating properties: {}", updatedUserChangedProps); } }
/** * Initialize local and target info * * @throws Exception */ private void initializeLocalAndTargetInfo(String svcId) throws Exception { // publish config_version which is also a target property // used as a flag denoting whether target properties have been changed PropertyInfoExt localPropInfo = localRepository.getOverrideProperties(); localConfigVersion = localPropInfo.getProperty(PropertyInfoExt.CONFIG_VERSION); if (localConfigVersion == null) { localConfigVersion = "0"; localPropInfo.addProperty(PropertyInfoExt.CONFIG_VERSION, localConfigVersion); } coordinator.setNodeSessionScopeInfo(new ConfigVersion(localConfigVersion)); log.info("Step1a: Local config version: {}", localConfigVersion); // set node scope properties for the 1st time only since they are invariants. localNodePropInfo = coordinator.getNodeGlobalScopeInfo(PropertyInfoExt.class, "propertyinfo", svcId); // The PropertyInfoExt object will be persisted in the zookeeper path // config/propertyinfo/(svcId) if (localNodePropInfo == null) { localNodePropInfo = getNodeScopeProperties(localPropInfo); coordinator.setNodeGlobalScopeInfo(localNodePropInfo, "propertyinfo", svcId); // The PropertyInfoExt object can be fetched from the zookeeper path // config/propertyinfo/(svcId) log.info("Step1a: Local node scope properties: {}", localNodePropInfo); } // get local target property info localTargetPropInfo = getLocalTargetPropInfo(localPropInfo); log.debug("Step1a: Local target properties: {}", localTargetPropInfo); // set target if empty targetPropInfo = coordinator.getTargetInfo(PropertyInfoExt.class); targetPowerOffState = coordinator.getTargetInfo(PowerOffState.class); if (targetPropInfo == null || targetPowerOffState == null) { // only control node can set target try { // Set the updated propperty info in coordinator coordinator.setTargetInfo(localPropInfo); coordinator.setTargetInfo(new PowerOffState(PowerOffState.State.NONE)); targetPropInfo = coordinator.getTargetInfo(PropertyInfoExt.class); log.info("Step1b: Target property set to local state: {}", targetPropInfo); targetPowerOffState = coordinator.getTargetInfo(PowerOffState.class); log.info("Step1b: Target poweroff state set to: {}", PowerOffState.State.NONE); } catch (CoordinatorClientException e) { log.info("Step1b: Wait another control node to set target"); retrySleep(); throw e; } } // Initialize vdc prop info localVdcPropInfo = localRepository.getVdcPropertyInfo(); targetVdcPropInfo = loadVdcConfigFromDatabase(); if (localVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE) == null) { localRepository.setVdcPropertyInfo(targetVdcPropInfo); localVdcPropInfo = localRepository.getVdcPropertyInfo(); String vdc_ids = targetVdcPropInfo.getProperty(VDC_IDS_KEY); String[] vdcIds = vdc_ids.split(","); if (vdcIds.length > 1) { log.info("More than one Vdc, so set reboot flag"); shouldReboot = true; } } }
@Override protected void innerRun() { final String svcId = coordinator.getMySvcId(); while (doRun) { log.debug("Main loop: Start"); shortSleep = false; if (shouldReboot) { reboot(); } // Step0: check if we have the property lock boolean hasLock; try { hasLock = coordinator.hasPersistentLock(svcId, propertyLockId); } catch (Exception e) { log.info("Step1: Failed to verify if the current node has the property lock ", e); retrySleep(); continue; } if (hasLock) { try { coordinator.releasePersistentLock(svcId, propertyLockId); log.info("Step0: Released property lock for node: {}", svcId); wakeupOtherNodes(); } catch (InvalidLockOwnerException e) { log.error("Step0: Failed to release the property lock: Not owner."); } catch (Exception e) { log.info("Step0: Failed to release the property lock and will retry: {}", e.getMessage()); retrySleep(); continue; } } // Step1: publish current state, and set target if empty try { initializeLocalAndTargetInfo(svcId); } catch (Exception e) { log.info("Step1b failed and will be retried: {}", e.getMessage()); retrySleep(); continue; } // Step2: power off if all nodes agree. log.info("Step2: Power off if poweroff state != NONE. {}", targetPowerOffState); try { gracefulPoweroffCluster(); } catch (Exception e) { log.error("Step2: Failed to poweroff. {}", e); } // Step3: if target property is changed, update log.info("Step3: If target property is changed, update"); if (localTargetPropInfo != null && targetPropInfo != null && !localConfigVersion.equals( targetPropInfo.getProperty(PropertyInfoExt.CONFIG_VERSION))) { log.info("Step3a: Current properties are not same as target properties. Updating."); log.debug("Current local target properties: " + localTargetPropInfo); log.debug("Target properties: " + targetPropInfo); try { updateProperties(svcId); } catch (Exception e) { log.info("Step3a: Update failed and will be retried: {}", e.getMessage()); // Restart the loop immediately so that we release the upgrade lock. continue; } continue; } log.info("Step4: If VDC configuration is changed update"); if (vdcPropertiesChanged()) { log.info("Step4: Current vdc properties are not same as target vdc properties. Updating."); log.debug("Current local vdc properties: " + localVdcPropInfo); log.debug("Target vdc properties: " + targetVdcPropInfo); try { updateVdcProperties(svcId); } catch (Exception e) { log.info("Step4: VDC properties update failed and will be retried: {}", e.getMessage()); // Restart the loop immediately so that we release the upgrade lock. continue; } continue; } if (shouldReboot == false) { // Step5: sleep log.info("Step5: sleep"); longSleep(); } } }