public void vcenterClusterAddHostOperation(
      URI vcenterId, URI vcenterDataCenterId, URI clusterId, URI hostId, String stepId) {
    VcenterApiClient vcenterApiClient = null;
    try {
      WorkflowStepCompleter.stepExecuting(stepId);

      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
      Host host = _dbClient.queryObject(Host.class, hostId);

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
      String key =
          vcenterApiClient.addHost(
              vcenterDataCenter.getLabel(),
              cluster.getExternalId(),
              host.getHostName(),
              host.getUsername(),
              host.getPassword());
      _log.info("Successfully added or located host " + host.getHostName() + " " + key);

      host.setVcenterDataCenter(vcenterDataCenter.getId());
      _dbClient.updateAndReindexObject(host);

      WorkflowStepCompleter.stepSucceded(stepId);
    } catch (Exception e) {
      _log.error("vcenterClusterAddHostOperation exception " + e);
      WorkflowStepCompleter.stepFailed(
          stepId, VcenterControllerException.exceptions.hostException(e.getLocalizedMessage(), e));
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  @Override
  public void removeVcenterCluster(URI datacenterUri, URI clusterUri) throws InternalException {
    VcenterApiClient vcenterApiClient = null;
    try {
      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, datacenterUri);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterUri);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterDataCenter.getVcenter());
      _log.info(
          "Request to remove cluster "
              + vcenter.getLabel()
              + "/"
              + vcenterDataCenter.getLabel()
              + "/"
              + cluster.getLabel());

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
      vcenterApiClient.removeCluster(vcenterDataCenter.getLabel(), cluster.getExternalId());
    } catch (VcenterObjectConnectionException e) {
      throw VcenterControllerException.exceptions.objectConnectionException(
          e.getLocalizedMessage(), e);
    } catch (VcenterObjectNotFoundException e) {
      throw VcenterControllerException.exceptions.objectNotFoundException(
          e.getLocalizedMessage(), e);
    } catch (VcenterServerConnectionException e) {
      throw VcenterControllerException.exceptions.serverConnectionException(
          e.getLocalizedMessage(), e);
    } catch (Exception e) {
      _log.error("removeVcenterCluster exception " + e);
      throw VcenterControllerException.exceptions.unexpectedException(e.getLocalizedMessage(), e);
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  public void createUpdateVcenterClusterOperation(
      boolean createCluster, URI vcenterId, URI vcenterDataCenterId, URI clusterId, String stepId) {
    VcenterApiClient vcenterApiClient = null;
    try {
      WorkflowStepCompleter.stepExecuting(stepId);

      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
      String vcenterClusterId = null;
      if (createCluster) {
        _log.info("Create cluster with name " + cluster.getLabel());
        vcenterClusterId =
            vcenterApiClient.createCluster(vcenterDataCenter.getLabel(), cluster.getLabel());
      } else {
        // Use MoRef if present but don't stop if we fail to find cluster based off this because
        // we'll fall back and try on name
        String externalId = cluster.getExternalId();
        if (externalId != null && !externalId.trim().equals("")) {
          _log.info("Update cluster with MoRef " + externalId);
          try {
            vcenterClusterId =
                vcenterApiClient.updateCluster(vcenterDataCenter.getLabel(), externalId);
          } catch (VcenterObjectNotFoundException e) {
            _log.info(
                "Ignore VcenterObjectNotFoundException updateCluster when using MoRef... Try name based search next");
          }
        }
        if (vcenterClusterId == null) {
          _log.info("Update cluster with name " + cluster.getLabel());
          vcenterClusterId =
              vcenterApiClient.updateCluster(vcenterDataCenter.getLabel(), cluster.getLabel());
        }
      }
      _log.info(
          "Successfully created or updated cluster " + cluster.getLabel() + " " + vcenterClusterId);

      cluster.setVcenterDataCenter(vcenterDataCenter.getId());
      cluster.setExternalId(vcenterClusterId);
      _dbClient.updateAndReindexObject(cluster);

      WorkflowStepCompleter.stepSucceded(stepId);
    } catch (Exception e) {
      _log.error("createUpdateVcenterClusterOperation exception " + e);
      WorkflowStepCompleter.stepFailed(
          stepId,
          VcenterControllerException.exceptions.clusterException(e.getLocalizedMessage(), e));
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  // Find a host connected and powered on then refresh it
  public void vcenterClusterSelectHostOperation(
      URI vcenterId, URI vcenterDataCenterId, URI clusterId, URI[] hostUris, String stepId) {
    VcenterApiClient vcenterApiClient = null;
    try {
      WorkflowStepCompleter.stepExecuting(stepId);

      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
      Collection<Host> hosts = _dbClient.queryObject(Host.class, hostUris);

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());

      Host hostForStorageOperations = null;
      for (Host host : hosts) {
        try {
          vcenterApiClient.checkHostConnectedPoweredOn(
              vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName());
          hostForStorageOperations = host;
          _log.info("Host " + host.getHostName() + " to be used for storage operations");
          break;
        } catch (Exception e) {
          _log.info(
              "Host "
                  + host.getHostName()
                  + " not valid for storage operations due to exception "
                  + e.getLocalizedMessage());
        }
      }
      if (hostForStorageOperations == null) {
        _log.error(
            "No host valid for performing storage operations thus cannot perform datastore operations");
        throw new Exception(
            "No host valid for performing storage operations thus cannot perform datastore operations");
      }
      vcenterApiClient.refreshRescanHostStorage(
          vcenterDataCenter.getLabel(),
          cluster.getExternalId(),
          hostForStorageOperations.getHostName());

      // persist hostForStorageOperations ID in wf data
      _workflowService.storeStepData(stepId, hostForStorageOperations.getId());

      WorkflowStepCompleter.stepSucceded(stepId);
    } catch (Exception e) {
      _log.error("vcenterClusterSelectHostOperation exception " + e);
      WorkflowStepCompleter.stepFailed(
          stepId, VcenterControllerException.exceptions.hostException(e.getLocalizedMessage(), e));
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  public void vcenterClusterCreateDatastoreOperation(
      URI vcenterId,
      URI vcenterDataCenterId,
      URI clusterId,
      URI volumeId,
      String selectHostStepId,
      String stepId) {
    VcenterApiClient vcenterApiClient = null;
    try {
      WorkflowStepCompleter.stepExecuting(stepId);

      URI hostId = (URI) _workflowService.loadStepData(selectHostStepId);
      if (hostId == null) {
        _log.error("Workflow loadStepData on " + selectHostStepId + " is null");
        throw new Exception("Workflow loadStepData on " + selectHostStepId + " is null");
      }

      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
      Host host = _dbClient.queryObject(Host.class, hostId);
      Volume volume = _dbClient.queryObject(Volume.class, volumeId);

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
      String key =
          vcenterApiClient.createDatastore(
              vcenterDataCenter.getLabel(),
              cluster.getExternalId(),
              host.getHostName(),
              volume.getWWN(),
              volume.getLabel());
      _log.info("Successfully created or located datastore " + volume.getLabel() + " " + key);

      host.setVcenterDataCenter(vcenterDataCenter.getId());
      _dbClient.updateAndReindexObject(host);

      WorkflowStepCompleter.stepSucceded(stepId);
    } catch (Exception e) {
      _log.error("vcenterClusterCreateDatastoreOperation exception " + e);
      WorkflowStepCompleter.stepFailed(
          stepId, VcenterControllerException.exceptions.hostException(e.getLocalizedMessage(), e));
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  private List<String> getVirtualMachines(
      URI datacenterUri, URI clusterUri, URI hostUri, boolean runningOnly)
      throws InternalException {
    VcenterApiClient vcenterApiClient = null;
    try {
      Host host = _dbClient.queryObject(Host.class, hostUri);
      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, datacenterUri);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterUri);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterDataCenter.getVcenter());

      _log.info(
          "Request to get virtual machines for "
              + vcenter.getLabel()
              + "/"
              + vcenterDataCenter.getLabel()
              + "/"
              + cluster.getLabel()
              + "/"
              + host.getHostName());

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
      return runningOnly
          ? vcenterApiClient.getRunningVirtualMachines(
              vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName())
          : vcenterApiClient.getVirtualMachines(
              vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName());
    } catch (VcenterObjectConnectionException e) {
      throw VcenterControllerException.exceptions.objectConnectionException(
          e.getLocalizedMessage(), e);
    } catch (VcenterObjectNotFoundException e) {
      throw VcenterControllerException.exceptions.objectNotFoundException(
          e.getLocalizedMessage(), e);
    } catch (VcenterServerConnectionException e) {
      throw VcenterControllerException.exceptions.serverConnectionException(
          e.getLocalizedMessage(), e);
    } catch (Exception e) {
      _log.error("getVirtualMachines exception " + e);
      throw VcenterControllerException.exceptions.unexpectedException(e.getLocalizedMessage(), e);
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  /**
   * Validates the create/update vCenter input data
   *
   * @param param the input parameter
   * @param vcenter the vcenter being updated in case of update operation. This parameter must be
   *     null for create operations.
   */
  protected void validateVcenter(VcenterParam param, Vcenter vcenter, Boolean validateConnection) {
    if (vcenter == null
        || (param.findIpAddress() != null
            && !param.findIpAddress().equals(vcenter.getIpAddress()))) {
      checkDuplicateAltId(Vcenter.class, "ipAddress", param.findIpAddress(), "vcenter");
    }
    if (vcenter == null
        || (param.getName() != null && !param.getName().equals(vcenter.getLabel()))) {
      checkDuplicateLabel(Vcenter.class, param.getName(), "vcenter");
    }
    validateVcenterCredentials(param, vcenter);

    if (validateConnection != null && validateConnection == true) {
      String errorMessage = VCenterConnectionValidator.isVCenterConnectionValid(param);
      if (StringUtils.isNotBlank(errorMessage)) {
        throw APIException.badRequests.invalidVCenterConnection(errorMessage);
      }
    }
  }
  private void createOrUpdateVcenterCluster(
      boolean createCluster,
      AsyncTask task,
      URI clusterUri,
      URI[] addHostUris,
      URI[] removeHostUris,
      URI[] volumeUris)
      throws InternalException {
    TaskCompleter completer = null;
    try {
      _log.info(
          "createOrUpdateVcenterCluster "
              + createCluster
              + " "
              + task
              + " "
              + clusterUri
              + " "
              + addHostUris
              + " "
              + removeHostUris);

      if (task == null) {
        _log.error("AsyncTask is null");
        throw new Exception("AsyncTask is null");
      }
      URI vcenterDataCenterId = task._id;
      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);

      if (clusterUri == null) {
        _log.error("Cluster URI is null");
        throw new Exception("Cluster URI is null");
      }
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterUri);

      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterDataCenter.getVcenter());
      _log.info(
          "Request to create or update cluster "
              + vcenter.getIpAddress()
              + "/"
              + vcenterDataCenter.getLabel()
              + "/"
              + cluster.getLabel());

      Collection<Host> addHosts = new ArrayList<Host>();
      if (addHostUris == null || addHostUris.length == 0) {
        _log.info("Add host URIs is null or empty - Cluster will be created without hosts");
      } else {
        for (URI hostUri : addHostUris) {
          _log.info("createOrUpdateVcenterCluster " + clusterUri + " with add host " + hostUri);
        }
        addHosts = _dbClient.queryObject(Host.class, addHostUris);
      }

      Collection<Host> removeHosts = new ArrayList<Host>();
      if (removeHostUris == null || removeHostUris.length == 0) {
        _log.info("Remove host URIs is null or empty - Cluster will have no removed hosts");
      } else {
        for (URI hostUri : removeHostUris) {
          _log.info("createOrUpdateVcenterCluster " + clusterUri + " with remove host " + hostUri);
        }
        removeHosts = _dbClient.queryObject(Host.class, removeHostUris);
      }

      Collection<Volume> volumes = new ArrayList<Volume>();
      if (volumeUris == null || volumeUris.length == 0) {
        _log.info("Volume URIs is null or empty - Cluster will be created without datastores");
      } else {
        for (URI volumeUri : volumeUris) {
          _log.info("createOrUpdateVcenterCluster " + clusterUri + " with volume " + volumeUri);
        }
        volumes = _dbClient.queryObject(Volume.class, volumeUris);
      }

      completer =
          new VcenterClusterCompleter(
              vcenterDataCenterId,
              task._opId,
              OperationTypeEnum.CREATE_UPDATE_VCENTER_CLUSTER,
              "VCENTER_CONTROLLER");
      Workflow workflow =
          _workflowService.getNewWorkflow(
              this, "CREATE_UPDATE_VCENTER_CLUSTER_WORKFLOW", true, task._opId);
      String clusterStep =
          workflow.createStep(
              "CREATE_UPDATE_VCENTER_CLUSTER_STEP",
              String.format(
                  "vCenter cluster operation in vCenter datacenter %s", vcenterDataCenterId),
              null,
              vcenterDataCenterId,
              vcenterDataCenterId.toString(),
              this.getClass(),
              new Workflow.Method(
                  "createUpdateVcenterClusterOperation",
                  createCluster,
                  vcenter.getId(),
                  vcenterDataCenter.getId(),
                  cluster.getId()),
              null,
              null);

      String lastStep = clusterStep;
      if (removeHosts.size() > 0) {
        for (Host host : removeHosts) {
          String hostStep =
              workflow.createStep(
                  "VCENTER_CLUSTER_REMOVE_HOST",
                  String.format("vCenter cluster remove host operation %s", host.getId()),
                  clusterStep,
                  vcenterDataCenterId,
                  vcenterDataCenterId.toString(),
                  this.getClass(),
                  new Workflow.Method(
                      "vcenterClusterRemoveHostOperation",
                      vcenter.getId(),
                      vcenterDataCenter.getId(),
                      cluster.getId(),
                      host.getId()),
                  null,
                  null);
          lastStep = hostStep; // add host will wait on last of these
        }
      }

      if (addHosts.size() > 0) {
        for (Host host : addHosts) {
          String hostStep =
              workflow.createStep(
                  "VCENTER_CLUSTER_ADD_HOST",
                  String.format("vCenter cluster add host operation %s", host.getId()),
                  lastStep,
                  vcenterDataCenterId,
                  vcenterDataCenterId.toString(),
                  this.getClass(),
                  new Workflow.Method(
                      "vcenterClusterAddHostOperation",
                      vcenter.getId(),
                      vcenterDataCenter.getId(),
                      cluster.getId(),
                      host.getId()),
                  null,
                  null);
        }

        if (volumes.size() > 0) {
          // Once all hosts in cluster select a host to use for shared storage operations
          String selectHostForStorageOperationsStep =
              workflow.createStep(
                  "VCENTER_CLUSTER_SELECT_HOST",
                  String.format(
                      "vCenter cluster select host for storage operations operation vCenter datacenter %s",
                      vcenterDataCenterId),
                  "VCENTER_CLUSTER_ADD_HOST",
                  vcenterDataCenterId,
                  vcenterDataCenterId.toString(),
                  this.getClass(),
                  new Workflow.Method(
                      "vcenterClusterSelectHostOperation",
                      vcenter.getId(),
                      vcenterDataCenter.getId(),
                      cluster.getId(),
                      addHostUris),
                  null,
                  null);

          // Do not run datastore creation in parallel
          // First datastore waits on selectHostForStorageOperationsStep step then next wait on the
          // previous datastore operation
          String volumeStep = null;
          for (Volume volume : volumes) {
            volumeStep =
                workflow.createStep(
                    "VCENTER_CLUSTER_CREATE_DATASTORE",
                    String.format("vCenter cluster create datastore operation %s", volume.getId()),
                    volumeStep == null ? selectHostForStorageOperationsStep : volumeStep,
                    vcenterDataCenterId,
                    vcenterDataCenterId.toString(),
                    this.getClass(),
                    new Workflow.Method(
                        "vcenterClusterCreateDatastoreOperation",
                        vcenter.getId(),
                        vcenterDataCenter.getId(),
                        cluster.getId(),
                        volume.getId(),
                        selectHostForStorageOperationsStep),
                    null,
                    null);
          }
        }
      }

      workflow.executePlan(completer, "Success");
    } catch (Exception e) {
      _log.error("createOrUpdateVcenterCluster caught an exception.", e);
      ServiceError serviceError = DeviceControllerException.errors.jobFailed(e);
      completer.error(_dbClient, serviceError);
    }
  }