public void vcenterClusterRemoveHostOperation(
      URI vcenterId, URI vcenterDataCenterId, URI clusterId, URI hostId, String stepId) {
    VcenterApiClient vcenterApiClient = null;
    try {
      WorkflowStepCompleter.stepExecuting(stepId);

      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
      Host host = _dbClient.queryObject(Host.class, hostId);

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
      vcenterApiClient.removeHost(
          vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName());
      _log.info("Successfully removed host " + host.getHostName());

      host.setVcenterDataCenter(null); // Probably doesnt work
      _dbClient.updateAndReindexObject(host);

      WorkflowStepCompleter.stepSucceded(stepId);
    } catch (Exception e) {
      _log.error("vcenterClusterAddHostOperation exception " + e);
      WorkflowStepCompleter.stepFailed(
          stepId, VcenterControllerException.exceptions.hostException(e.getLocalizedMessage(), e));
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  @Override
  protected void complete(DbClient dbClient, Status status, ServiceCoded coded)
      throws DeviceControllerException {
    if (isNotifyWorkflow()) {
      // If there is a workflow, update the step to complete.
      updateWorkflowStatus(status, coded);
    }

    // if export updates were successful, remove all old initiators and deleted hosts
    if (status.equals(Status.ready)) {
      for (HostStateChange hostChange : changes) {
        for (URI initiatorId : hostChange.getOldInitiators()) {
          Initiator initiator = dbClient.queryObject(Initiator.class, initiatorId);
          dbClient.markForDeletion(initiator);
          _logger.info("Initiator marked for deletion: " + this.getId());
        }
      }

      for (URI hostId : deletedHosts) {
        Host host = dbClient.queryObject(Host.class, hostId);
        // don't delete host if it was provisioned by Vipr
        if (!NullColumnValueGetter.isNullURI(host.getComputeElement())) {
          _logger.info(
              "do not delete provisioned host {} - disassociate it from vcenter", host.getLabel());
          host.setVcenterDataCenter(NullColumnValueGetter.getNullURI());
          dbClient.persistObject(host);
        } else {
          ComputeSystemHelper.doDeactivateHost(dbClient, host);
          _logger.info("Deactivating Host: " + host.getId());
        }
      }

      for (URI clusterId : deletedClusters) {
        Cluster cluster = dbClient.queryObject(Cluster.class, clusterId);
        List<URI> clusterHosts =
            ComputeSystemHelper.getChildrenUris(dbClient, clusterId, Host.class, "cluster");
        // don't delete cluster if all hosts weren't deleted (ex: hosts provisioned by ViPR)
        if (!clusterHosts.isEmpty()) {
          _logger.info(
              "do not delete cluster {} - it still has hosts - disassociate it from vcenter",
              cluster.getLabel());
          cluster.setVcenterDataCenter(NullColumnValueGetter.getNullURI());
          cluster.setExternalId(NullColumnValueGetter.getNullStr());
          dbClient.persistObject(cluster);
        } else {
          ComputeSystemHelper.doDeactivateCluster(dbClient, cluster);
          _logger.info("Deactivating Cluster: " + cluster.getId());
        }
      }
    }
  }
  public void vcenterClusterCreateDatastoreOperation(
      URI vcenterId,
      URI vcenterDataCenterId,
      URI clusterId,
      URI volumeId,
      String selectHostStepId,
      String stepId) {
    VcenterApiClient vcenterApiClient = null;
    try {
      WorkflowStepCompleter.stepExecuting(stepId);

      URI hostId = (URI) _workflowService.loadStepData(selectHostStepId);
      if (hostId == null) {
        _log.error("Workflow loadStepData on " + selectHostStepId + " is null");
        throw new Exception("Workflow loadStepData on " + selectHostStepId + " is null");
      }

      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
      Host host = _dbClient.queryObject(Host.class, hostId);
      Volume volume = _dbClient.queryObject(Volume.class, volumeId);

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
      String key =
          vcenterApiClient.createDatastore(
              vcenterDataCenter.getLabel(),
              cluster.getExternalId(),
              host.getHostName(),
              volume.getWWN(),
              volume.getLabel());
      _log.info("Successfully created or located datastore " + volume.getLabel() + " " + key);

      host.setVcenterDataCenter(vcenterDataCenter.getId());
      _dbClient.updateAndReindexObject(host);

      WorkflowStepCompleter.stepSucceded(stepId);
    } catch (Exception e) {
      _log.error("vcenterClusterCreateDatastoreOperation exception " + e);
      WorkflowStepCompleter.stepFailed(
          stepId, VcenterControllerException.exceptions.hostException(e.getLocalizedMessage(), e));
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  private List<String> getVirtualMachines(
      URI datacenterUri, URI clusterUri, URI hostUri, boolean runningOnly)
      throws InternalException {
    VcenterApiClient vcenterApiClient = null;
    try {
      Host host = _dbClient.queryObject(Host.class, hostUri);
      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, datacenterUri);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterUri);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterDataCenter.getVcenter());

      _log.info(
          "Request to get virtual machines for "
              + vcenter.getLabel()
              + "/"
              + vcenterDataCenter.getLabel()
              + "/"
              + cluster.getLabel()
              + "/"
              + host.getHostName());

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
      return runningOnly
          ? vcenterApiClient.getRunningVirtualMachines(
              vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName())
          : vcenterApiClient.getVirtualMachines(
              vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName());
    } catch (VcenterObjectConnectionException e) {
      throw VcenterControllerException.exceptions.objectConnectionException(
          e.getLocalizedMessage(), e);
    } catch (VcenterObjectNotFoundException e) {
      throw VcenterControllerException.exceptions.objectNotFoundException(
          e.getLocalizedMessage(), e);
    } catch (VcenterServerConnectionException e) {
      throw VcenterControllerException.exceptions.serverConnectionException(
          e.getLocalizedMessage(), e);
    } catch (Exception e) {
      _log.error("getVirtualMachines exception " + e);
      throw VcenterControllerException.exceptions.unexpectedException(e.getLocalizedMessage(), e);
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  @Override
  public void exitMaintenanceMode(URI datacenterUri, URI clusterUri, URI hostUri)
      throws InternalException {
    VcenterApiClient vcenterApiClient = null;
    try {
      Host host = _dbClient.queryObject(Host.class, hostUri);
      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, datacenterUri);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterUri);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterDataCenter.getVcenter());
      _log.info(
          "Request to exit maintenance mode for "
              + vcenter.getLabel()
              + "/"
              + vcenterDataCenter.getLabel()
              + "/"
              + cluster.getLabel()
              + "/"
              + host.getHostName());

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());
      vcenterApiClient.exitMaintenanceMode(
          vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName());
    } catch (VcenterObjectConnectionException e) {
      throw VcenterControllerException.exceptions.objectConnectionException(
          e.getLocalizedMessage(), e);
    } catch (VcenterObjectNotFoundException e) {
      throw VcenterControllerException.exceptions.objectNotFoundException(
          e.getLocalizedMessage(), e);
    } catch (VcenterServerConnectionException e) {
      throw VcenterControllerException.exceptions.serverConnectionException(
          e.getLocalizedMessage(), e);
    } catch (Exception e) {
      _log.error("exitMaintenanceMode exception " + e);
      throw VcenterControllerException.exceptions.unexpectedException(e.getLocalizedMessage(), e);
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  // Find a host connected and powered on then refresh it
  public void vcenterClusterSelectHostOperation(
      URI vcenterId, URI vcenterDataCenterId, URI clusterId, URI[] hostUris, String stepId) {
    VcenterApiClient vcenterApiClient = null;
    try {
      WorkflowStepCompleter.stepExecuting(stepId);

      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterId);
      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterId);
      Collection<Host> hosts = _dbClient.queryObject(Host.class, hostUris);

      vcenterApiClient = new VcenterApiClient(_coordinator.getPropertyInfo());
      vcenterApiClient.setup(vcenter.getIpAddress(), vcenter.getUsername(), vcenter.getPassword());

      Host hostForStorageOperations = null;
      for (Host host : hosts) {
        try {
          vcenterApiClient.checkHostConnectedPoweredOn(
              vcenterDataCenter.getLabel(), cluster.getExternalId(), host.getHostName());
          hostForStorageOperations = host;
          _log.info("Host " + host.getHostName() + " to be used for storage operations");
          break;
        } catch (Exception e) {
          _log.info(
              "Host "
                  + host.getHostName()
                  + " not valid for storage operations due to exception "
                  + e.getLocalizedMessage());
        }
      }
      if (hostForStorageOperations == null) {
        _log.error(
            "No host valid for performing storage operations thus cannot perform datastore operations");
        throw new Exception(
            "No host valid for performing storage operations thus cannot perform datastore operations");
      }
      vcenterApiClient.refreshRescanHostStorage(
          vcenterDataCenter.getLabel(),
          cluster.getExternalId(),
          hostForStorageOperations.getHostName());

      // persist hostForStorageOperations ID in wf data
      _workflowService.storeStepData(stepId, hostForStorageOperations.getId());

      WorkflowStepCompleter.stepSucceded(stepId);
    } catch (Exception e) {
      _log.error("vcenterClusterSelectHostOperation exception " + e);
      WorkflowStepCompleter.stepFailed(
          stepId, VcenterControllerException.exceptions.hostException(e.getLocalizedMessage(), e));
    } finally {
      if (vcenterApiClient != null) vcenterApiClient.destroy();
    }
  }
  private void createOrUpdateVcenterCluster(
      boolean createCluster,
      AsyncTask task,
      URI clusterUri,
      URI[] addHostUris,
      URI[] removeHostUris,
      URI[] volumeUris)
      throws InternalException {
    TaskCompleter completer = null;
    try {
      _log.info(
          "createOrUpdateVcenterCluster "
              + createCluster
              + " "
              + task
              + " "
              + clusterUri
              + " "
              + addHostUris
              + " "
              + removeHostUris);

      if (task == null) {
        _log.error("AsyncTask is null");
        throw new Exception("AsyncTask is null");
      }
      URI vcenterDataCenterId = task._id;
      VcenterDataCenter vcenterDataCenter =
          _dbClient.queryObject(VcenterDataCenter.class, vcenterDataCenterId);

      if (clusterUri == null) {
        _log.error("Cluster URI is null");
        throw new Exception("Cluster URI is null");
      }
      Cluster cluster = _dbClient.queryObject(Cluster.class, clusterUri);

      Vcenter vcenter = _dbClient.queryObject(Vcenter.class, vcenterDataCenter.getVcenter());
      _log.info(
          "Request to create or update cluster "
              + vcenter.getIpAddress()
              + "/"
              + vcenterDataCenter.getLabel()
              + "/"
              + cluster.getLabel());

      Collection<Host> addHosts = new ArrayList<Host>();
      if (addHostUris == null || addHostUris.length == 0) {
        _log.info("Add host URIs is null or empty - Cluster will be created without hosts");
      } else {
        for (URI hostUri : addHostUris) {
          _log.info("createOrUpdateVcenterCluster " + clusterUri + " with add host " + hostUri);
        }
        addHosts = _dbClient.queryObject(Host.class, addHostUris);
      }

      Collection<Host> removeHosts = new ArrayList<Host>();
      if (removeHostUris == null || removeHostUris.length == 0) {
        _log.info("Remove host URIs is null or empty - Cluster will have no removed hosts");
      } else {
        for (URI hostUri : removeHostUris) {
          _log.info("createOrUpdateVcenterCluster " + clusterUri + " with remove host " + hostUri);
        }
        removeHosts = _dbClient.queryObject(Host.class, removeHostUris);
      }

      Collection<Volume> volumes = new ArrayList<Volume>();
      if (volumeUris == null || volumeUris.length == 0) {
        _log.info("Volume URIs is null or empty - Cluster will be created without datastores");
      } else {
        for (URI volumeUri : volumeUris) {
          _log.info("createOrUpdateVcenterCluster " + clusterUri + " with volume " + volumeUri);
        }
        volumes = _dbClient.queryObject(Volume.class, volumeUris);
      }

      completer =
          new VcenterClusterCompleter(
              vcenterDataCenterId,
              task._opId,
              OperationTypeEnum.CREATE_UPDATE_VCENTER_CLUSTER,
              "VCENTER_CONTROLLER");
      Workflow workflow =
          _workflowService.getNewWorkflow(
              this, "CREATE_UPDATE_VCENTER_CLUSTER_WORKFLOW", true, task._opId);
      String clusterStep =
          workflow.createStep(
              "CREATE_UPDATE_VCENTER_CLUSTER_STEP",
              String.format(
                  "vCenter cluster operation in vCenter datacenter %s", vcenterDataCenterId),
              null,
              vcenterDataCenterId,
              vcenterDataCenterId.toString(),
              this.getClass(),
              new Workflow.Method(
                  "createUpdateVcenterClusterOperation",
                  createCluster,
                  vcenter.getId(),
                  vcenterDataCenter.getId(),
                  cluster.getId()),
              null,
              null);

      String lastStep = clusterStep;
      if (removeHosts.size() > 0) {
        for (Host host : removeHosts) {
          String hostStep =
              workflow.createStep(
                  "VCENTER_CLUSTER_REMOVE_HOST",
                  String.format("vCenter cluster remove host operation %s", host.getId()),
                  clusterStep,
                  vcenterDataCenterId,
                  vcenterDataCenterId.toString(),
                  this.getClass(),
                  new Workflow.Method(
                      "vcenterClusterRemoveHostOperation",
                      vcenter.getId(),
                      vcenterDataCenter.getId(),
                      cluster.getId(),
                      host.getId()),
                  null,
                  null);
          lastStep = hostStep; // add host will wait on last of these
        }
      }

      if (addHosts.size() > 0) {
        for (Host host : addHosts) {
          String hostStep =
              workflow.createStep(
                  "VCENTER_CLUSTER_ADD_HOST",
                  String.format("vCenter cluster add host operation %s", host.getId()),
                  lastStep,
                  vcenterDataCenterId,
                  vcenterDataCenterId.toString(),
                  this.getClass(),
                  new Workflow.Method(
                      "vcenterClusterAddHostOperation",
                      vcenter.getId(),
                      vcenterDataCenter.getId(),
                      cluster.getId(),
                      host.getId()),
                  null,
                  null);
        }

        if (volumes.size() > 0) {
          // Once all hosts in cluster select a host to use for shared storage operations
          String selectHostForStorageOperationsStep =
              workflow.createStep(
                  "VCENTER_CLUSTER_SELECT_HOST",
                  String.format(
                      "vCenter cluster select host for storage operations operation vCenter datacenter %s",
                      vcenterDataCenterId),
                  "VCENTER_CLUSTER_ADD_HOST",
                  vcenterDataCenterId,
                  vcenterDataCenterId.toString(),
                  this.getClass(),
                  new Workflow.Method(
                      "vcenterClusterSelectHostOperation",
                      vcenter.getId(),
                      vcenterDataCenter.getId(),
                      cluster.getId(),
                      addHostUris),
                  null,
                  null);

          // Do not run datastore creation in parallel
          // First datastore waits on selectHostForStorageOperationsStep step then next wait on the
          // previous datastore operation
          String volumeStep = null;
          for (Volume volume : volumes) {
            volumeStep =
                workflow.createStep(
                    "VCENTER_CLUSTER_CREATE_DATASTORE",
                    String.format("vCenter cluster create datastore operation %s", volume.getId()),
                    volumeStep == null ? selectHostForStorageOperationsStep : volumeStep,
                    vcenterDataCenterId,
                    vcenterDataCenterId.toString(),
                    this.getClass(),
                    new Workflow.Method(
                        "vcenterClusterCreateDatastoreOperation",
                        vcenter.getId(),
                        vcenterDataCenter.getId(),
                        cluster.getId(),
                        volume.getId(),
                        selectHostForStorageOperationsStep),
                    null,
                    null);
          }
        }
      }

      workflow.executePlan(completer, "Success");
    } catch (Exception e) {
      _log.error("createOrUpdateVcenterCluster caught an exception.", e);
      ServiceError serviceError = DeviceControllerException.errors.jobFailed(e);
      completer.error(_dbClient, serviceError);
    }
  }