/**
  * Update vdc properties and reboot the node if
  *
  * @param svcId node service id
  * @throws Exception
  */
 private void updateVdcProperties(String svcId) throws Exception {
   // If the change is being done to create a multi VDC configuration or to reduce to a
   // multi VDC configuration a reboot is needed.  If only operating on a single VDC
   // do not reboot the nodes.
   if (targetVdcPropInfo.getProperty(VdcConfigUtil.VDC_IDS).contains(",")
       || localVdcPropInfo.getProperty(VdcConfigUtil.VDC_IDS).contains(",")) {
     log.info("Step4: Acquiring property lock for vdc properties change.");
     if (!getPropertyLock(svcId)) {
       retrySleep();
     } else if (!isQuorumMaintained()) {
       try {
         coordinator.releasePersistentLock(svcId, propertyLockId);
       } catch (Exception e) {
         log.error("Failed to release the property lock:", e);
       }
       retrySleep();
     } else {
       log.info("Step4: Setting vdc properties and rebooting for multi-vdc config change");
       localRepository.setVdcPropertyInfo(targetVdcPropInfo);
       reboot();
     }
   } else {
     log.info("Step4: Setting vdc properties not rebooting for single VDC change");
     localRepository.setVdcPropertyInfo(targetVdcPropInfo);
   }
 }
  /**
   * Combine nodeScopePropInfo with targetPropInfo
   *
   * @param targetPropInfo target property info
   * @param nodeScopePropInfo node scope property info
   * @return combined property info
   */
  private PropertyInfoExt combineProps(
      final PropertyInfoExt targetPropInfo, final PropertyInfoExt nodeScopePropInfo) {
    PropertyInfoExt combinedProps = new PropertyInfoExt();

    for (Entry<String, String> entry : targetPropInfo.getAllProperties().entrySet()) {
      combinedProps.addProperty(entry.getKey(), entry.getValue());
    }

    for (Entry<String, String> entry : nodeScopePropInfo.getAllProperties().entrySet()) {
      combinedProps.addProperty(entry.getKey(), entry.getValue());
    }

    return combinedProps;
  }
  /**
   * Get node scope properties UpgradeManager will publish the node scope properties as node
   * information into coordinator Node scope properties are invariants.
   *
   * <p>We check to see if a property is in metadata or not. If it is, it is a target property; If
   * not, it is a local property
   *
   * @param localPropInfo local property info read from /etc/config.properties
   * @return node scope properties
   */
  private PropertyInfoExt getNodeScopeProperties(final PropertyInfoExt localPropInfo) {
    Map<String, PropertyMetadata> metadata = PropertiesMetadata.getGlobalMetadata();
    PropertyInfoExt localScopeProps = new PropertyInfoExt();

    for (Entry<String, String> entry : localPropInfo.getAllProperties().entrySet()) {
      final String key = entry.getKey();
      final String value = entry.getValue();

      if (!metadata.containsKey(key)) {
        localScopeProps.addProperty(key, value);
      }
    }

    return localScopeProps;
  }
  /**
   * Check if VDC configuration is different in the database vs. what is stored locally
   *
   * @return
   */
  private boolean vdcPropertiesChanged() {
    if (!coordinator.isControlNode()) {
      return false;
    }

    int localVdcConfigHashcode =
        localVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE) == null
            ? 0
            : Integer.parseInt(localVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE));
    int targetVdcConfigHashcode =
        targetVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE) == null
            ? 0
            : Integer.parseInt(targetVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE));

    return localVdcConfigHashcode != targetVdcConfigHashcode;
  }
 private void notifyPropertyChanges(PropertyInfoExt diffProperties) {
   List<String> notifierTags = diffProperties.getNotifierTags();
   for (String notifierTag : notifierTags) {
     log.info("Step3a: Calling notifier {}", notifierTag);
     try {
       Notifier notifier = Notifier.getInstance(notifierTag);
       if (notifier != null) notifier.doNotify();
     } catch (Exception e) {
       log.error("Step3a: Fail to invoke notifier {}", notifierTag, e);
     }
   }
 }
  private void reconfigProperties(PropertyInfoExt diffProperties) {
    // only get the notifiers that requires reconfig as well
    List<String> notifierTagList = diffProperties.getNotifierTags(true);
    String notifierTags = StringUtils.join(notifierTagList, " ");
    log.info("Step3a: Reconfiguring properties related to {}", notifierTags);

    try {
      localRepository.reconfigProperties(notifierTags);
    } catch (Exception e) {
      log.error("Step3a: Fail to reconfig properties related to {}", notifierTags, e);
    }
  }
  /**
   * Get local target property info
   *
   * <p>For control node, properties that can be found in metadata are target properties For extra
   * node, not only exist in metadata, but also ControlNodeOnly==false
   *
   * @param localPropInfo
   * @return
   */
  private PropertyInfoExt getLocalTargetPropInfo(final PropertyInfoExt localPropInfo) {
    Map<String, PropertyMetadata> metadata = PropertiesMetadata.getGlobalMetadata();
    PropertyInfoExt localTargetProps = new PropertyInfoExt();

    for (Entry<String, String> entry : localPropInfo.getAllProperties().entrySet()) {
      final String key = entry.getKey();
      final String value = entry.getValue();

      if (metadata.containsKey(key)) {
        if (coordinator.isControlNode()) {
          localTargetProps.addProperty(key, value);
        } else {
          if (!metadata.get(key).getControlNodeOnly()) {
            localTargetProps.addProperty(key, value);
          }
        }
      }
    }

    return localTargetProps;
  }
  /**
   * * Update SSL property
   *
   * @param state
   * @throws LocalRepositoryException
   */
  public void setSslPropertyInfo(PropertyInfoExt state) throws LocalRepositoryException {
    final String prefix = "setSslPropertyInfo(): to=" + state;
    _log.debug(prefix);

    final Path tmpFilePath = FileSystems.getDefault().getPath(SSL_PROPERTY_TMP);
    createTmpFile(tmpFilePath, state.toString(false), prefix);

    try {
      final String[] cmd = {_SYSTOOL_CMD, _SYSTOOL_SET_SSL_PROPS, SSL_PROPERTY_TMP};
      exec(prefix, cmd);
      _log.info(prefix + "Success");
    } finally {
      cleanupTmpFile(tmpFilePath);
    }
  }
  /**
   * * Update property
   *
   * @param state
   * @throws LocalRepositoryException
   */
  public void setOverrideProperties(PropertyInfoExt state) throws LocalRepositoryException {
    final String prefix = "setOverrideProperties(): to=" + state;
    _log.debug(prefix);

    final Path tmpFilePath = FileSystems.getDefault().getPath(TMP_CONFIG_USER_CHANGED_PROPS_PATH);
    createTmpFile(tmpFilePath, state.toString(false), prefix);

    try {
      final String[] cmd = {_SYSTOOL_CMD, _SYSTOOL_SET_OPROPS, TMP_CONFIG_USER_CHANGED_PROPS_PATH};
      exec(prefix, cmd);
      _log.info(prefix + "Success");
    } finally {
      cleanupTmpFile(tmpFilePath);
    }
  }
  /**
   * * Update data revision properties to local
   *
   * @param localRevisionProps
   * @throws LocalRepositoryException
   */
  public void setDataRevisionPropertyInfo(PropertyInfoExt localRevisionProps)
      throws LocalRepositoryException {
    final String prefix =
        String.format("setDataRevisionPropertyInfo(): to=%s ", localRevisionProps);
    _log.debug(prefix);

    final Path tmpFilePath = FileSystems.getDefault().getPath(DATA_REVISION_TMP);
    createTmpFile(tmpFilePath, localRevisionProps.toString(false), prefix);

    try {
      final String[] cmd = {_SYSTOOL_CMD, _SYSTOOL_SET_DATA_REVISION, DATA_REVISION_TMP};
      exec(prefix, cmd);
      _log.info(prefix + " Success");
    } finally {
      cleanupTmpFile(tmpFilePath);
    }
  }
  /**
   * Update properties
   *
   * @param svcId node service id
   * @throws Exception
   */
  private void updateProperties(String svcId) throws Exception {
    if (targetPropInfo.TARGET_PROPERTY.equals(targetPropInfo.OLD_TARGET_PROPERTY)) {
      coordinator.removeTargetInfo(targetPropInfo, true);
    }
    PropertyInfoExt diffProperties =
        new PropertyInfoExt(targetPropInfo.getDiffProperties(localTargetPropInfo));
    PropertyInfoExt override_properties =
        new PropertyInfoExt(localRepository.getOverrideProperties().getAllProperties());
    log.info("Step3a: Updating User Changed properties file: {}", override_properties);
    PropertyInfoExt updatedUserChangedProps = combineProps(override_properties, diffProperties);
    if (diffProperties.hasRebootProperty()) {
      if (!getPropertyLock(svcId)) {
        retrySleep();
      } else if (!isQuorumMaintained()) {
        try {
          coordinator.releasePersistentLock(svcId, propertyLockId);
        } catch (Exception e) {
          log.error("Failed to release the property lock:", e);
        }
        retrySleep();
      } else {
        log.info("Step3a: Reboot property found.");
        localRepository.setOverrideProperties(updatedUserChangedProps);
        log.info("Step3a: Updating properties: {}", updatedUserChangedProps);
        reboot();
      }
    } else if (diffProperties.hasReconfigProperty()
        || !diffProperties.getNotifierTags().isEmpty()) {
      log.info("Step3a: Reconfig property found or notifiers specified.");

      // CTRL-9860: don't update the local config version until everything is done.
      String targetVersion = targetPropInfo.getProperty(PropertyInfoExt.CONFIG_VERSION);
      updatedUserChangedProps.addProperty(PropertyInfoExt.CONFIG_VERSION, localConfigVersion);
      localRepository.setOverrideProperties(updatedUserChangedProps);
      log.info(
          "Step3a: Updating properties without updating the config version: {}",
          updatedUserChangedProps);
      if (diffProperties.hasReconfigAttributeWithoutNotifiers()) {
        // this is the old-school "complete" reconfig that takes no notifiers as arguments.
        // moving forward this will diminish
        // i.e., all reconfigRequired properties will have notifier specified.
        localRepository.reconfig();
      } else if (diffProperties.hasReconfigProperty()) {
        reconfigProperties(diffProperties);
      }

      // the notifier list can be empty, in which case nothing will be done.
      notifyPropertyChanges(diffProperties);

      // update the local config version to target version now
      log.info("Step3a: Updating the config version to {}", targetVersion);
      updatedUserChangedProps.addProperty(PropertyInfoExt.CONFIG_VERSION, targetVersion);
      localRepository.setOverrideProperties(updatedUserChangedProps);

    } else {
      log.info("Step3a: No reboot property found.");
      localRepository.setOverrideProperties(updatedUserChangedProps);
      log.info("Step3a: Updating properties: {}", updatedUserChangedProps);
    }
  }
  /**
   * Initialize local and target info
   *
   * @throws Exception
   */
  private void initializeLocalAndTargetInfo(String svcId) throws Exception {
    // publish config_version which is also a target property
    //  used as a flag denoting whether target properties have been changed
    PropertyInfoExt localPropInfo = localRepository.getOverrideProperties();
    localConfigVersion = localPropInfo.getProperty(PropertyInfoExt.CONFIG_VERSION);
    if (localConfigVersion == null) {
      localConfigVersion = "0";
      localPropInfo.addProperty(PropertyInfoExt.CONFIG_VERSION, localConfigVersion);
    }
    coordinator.setNodeSessionScopeInfo(new ConfigVersion(localConfigVersion));
    log.info("Step1a: Local config version: {}", localConfigVersion);
    // set node scope properties for the 1st time only since they are invariants.
    localNodePropInfo =
        coordinator.getNodeGlobalScopeInfo(PropertyInfoExt.class, "propertyinfo", svcId);
    // The PropertyInfoExt object will be persisted in the zookeeper path
    // config/propertyinfo/(svcId)
    if (localNodePropInfo == null) {
      localNodePropInfo = getNodeScopeProperties(localPropInfo);
      coordinator.setNodeGlobalScopeInfo(localNodePropInfo, "propertyinfo", svcId);
      // The PropertyInfoExt object can be fetched from the zookeeper path
      // config/propertyinfo/(svcId)
      log.info("Step1a: Local node scope properties: {}", localNodePropInfo);
    }
    // get local target property info
    localTargetPropInfo = getLocalTargetPropInfo(localPropInfo);
    log.debug("Step1a: Local target properties: {}", localTargetPropInfo);

    // set target if empty
    targetPropInfo = coordinator.getTargetInfo(PropertyInfoExt.class);
    targetPowerOffState = coordinator.getTargetInfo(PowerOffState.class);
    if (targetPropInfo == null || targetPowerOffState == null) {
      // only control node can set target
      try {
        // Set the updated propperty info in coordinator
        coordinator.setTargetInfo(localPropInfo);
        coordinator.setTargetInfo(new PowerOffState(PowerOffState.State.NONE));

        targetPropInfo = coordinator.getTargetInfo(PropertyInfoExt.class);
        log.info("Step1b: Target property set to local state: {}", targetPropInfo);
        targetPowerOffState = coordinator.getTargetInfo(PowerOffState.class);
        log.info("Step1b: Target poweroff state set to: {}", PowerOffState.State.NONE);
      } catch (CoordinatorClientException e) {
        log.info("Step1b: Wait another control node to set target");
        retrySleep();
        throw e;
      }
    }

    // Initialize vdc prop info
    localVdcPropInfo = localRepository.getVdcPropertyInfo();
    targetVdcPropInfo = loadVdcConfigFromDatabase();
    if (localVdcPropInfo.getProperty(VdcConfigUtil.VDC_CONFIG_HASHCODE) == null) {
      localRepository.setVdcPropertyInfo(targetVdcPropInfo);
      localVdcPropInfo = localRepository.getVdcPropertyInfo();
      String vdc_ids = targetVdcPropInfo.getProperty(VDC_IDS_KEY);
      String[] vdcIds = vdc_ids.split(",");
      if (vdcIds.length > 1) {
        log.info("More than one Vdc, so set reboot flag");
        shouldReboot = true;
      }
    }
  }
  @Override
  protected void innerRun() {
    final String svcId = coordinator.getMySvcId();

    while (doRun) {
      log.debug("Main loop: Start");

      shortSleep = false;

      if (shouldReboot) {
        reboot();
      }

      // Step0: check if we have the property lock
      boolean hasLock;
      try {
        hasLock = coordinator.hasPersistentLock(svcId, propertyLockId);
      } catch (Exception e) {
        log.info("Step1: Failed to verify if the current node has the property lock ", e);
        retrySleep();
        continue;
      }

      if (hasLock) {
        try {
          coordinator.releasePersistentLock(svcId, propertyLockId);
          log.info("Step0: Released property lock for node: {}", svcId);
          wakeupOtherNodes();
        } catch (InvalidLockOwnerException e) {
          log.error("Step0: Failed to release the property lock: Not owner.");
        } catch (Exception e) {
          log.info("Step0: Failed to release the property lock and will retry: {}", e.getMessage());
          retrySleep();
          continue;
        }
      }

      // Step1: publish current state, and set target if empty
      try {
        initializeLocalAndTargetInfo(svcId);
      } catch (Exception e) {
        log.info("Step1b failed and will be retried: {}", e.getMessage());
        retrySleep();
        continue;
      }

      // Step2: power off if all nodes agree.
      log.info("Step2: Power off if poweroff state != NONE. {}", targetPowerOffState);
      try {
        gracefulPoweroffCluster();
      } catch (Exception e) {
        log.error("Step2: Failed to poweroff. {}", e);
      }

      // Step3: if target property is changed, update
      log.info("Step3: If target property is changed, update");
      if (localTargetPropInfo != null
          && targetPropInfo != null
          && !localConfigVersion.equals(
              targetPropInfo.getProperty(PropertyInfoExt.CONFIG_VERSION))) {
        log.info("Step3a: Current properties are not same as target properties. Updating.");
        log.debug("Current local target properties: " + localTargetPropInfo);
        log.debug("Target properties: " + targetPropInfo);

        try {
          updateProperties(svcId);
        } catch (Exception e) {
          log.info("Step3a: Update failed and will be retried: {}", e.getMessage());
          // Restart the loop immediately so that we release the upgrade lock.
          continue;
        }
        continue;
      }

      log.info("Step4: If VDC configuration is changed update");
      if (vdcPropertiesChanged()) {
        log.info("Step4: Current vdc properties are not same as target vdc properties. Updating.");
        log.debug("Current local vdc properties: " + localVdcPropInfo);
        log.debug("Target vdc properties: " + targetVdcPropInfo);

        try {
          updateVdcProperties(svcId);
        } catch (Exception e) {
          log.info("Step4: VDC properties update failed and will be retried: {}", e.getMessage());
          // Restart the loop immediately so that we release the upgrade lock.
          continue;
        }
        continue;
      }

      if (shouldReboot == false) {
        // Step5: sleep
        log.info("Step5: sleep");
        longSleep();
      }
    }
  }