@Override
  public void doPolling(Guid cmdId, List<Guid> childCmdIds) {

    boolean anyFailed = false;
    for (Guid childCmdId : childCmdIds) {
      CommandStatus commandStatus = CommandCoordinatorUtil.getCommandStatus(childCmdId);
      switch (commandStatus) {
        case NOT_STARTED:
        case ACTIVE:
          log.info("Waiting for child commands to complete");
          return;
        case SUCCEEDED:
          break;
        case FAILED:
        case FAILED_RESTARTED:
        case UNKNOWN:
          anyFailed = true;
          break;
        default:
          log.error("Invalid command status: '{}", commandStatus);
          break;
      }
    }

    T command = getCommand(cmdId);
    command.getParameters().setTaskGroupSuccess(!anyFailed);
    command.setCommandStatus(anyFailed ? CommandStatus.FAILED : CommandStatus.SUCCEEDED);
    log.info("All commands have completed, status '{}'", command.getCommandStatus());
  }
 @Override
 public void onSucceeded(Guid cmdId, List<Guid> childCmdIds) {
   super.onSucceeded(cmdId, childCmdIds);
   log.info("Volume/Snapshot has been successfully deleted from Cinder. ID: {}", getDiskId());
   getCommand().endAction();
   CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
 }
Пример #3
0
  @Override
  public void doPolling(Guid cmdId, List<Guid> childCmdIds) {
    RemoveVmPoolCommand<? extends VmPoolParametersBase> command = getCommand(cmdId);

    boolean anyFailed = false;
    for (Guid childCmdId : childCmdIds) {
      CommandEntity entity = CommandCoordinatorUtil.getCommandEntity(childCmdId);
      switch (entity.getCommandStatus()) {
        case ENDED_WITH_FAILURE:
        case FAILED:
        case EXECUTION_FAILED:
        case UNKNOWN:
          anyFailed = true;
          break;

        default:
          break;
      }
    }

    if (anyFailed) {
      command.setCommandStatus(CommandStatus.FAILED);
    } else {
      VmPool pool = DbFacade.getInstance().getVmPoolDao().get(command.getVmPoolId());
      if (pool == null || pool.getRunningVmsCount() == 0) {
        command.setCommandStatus(CommandStatus.SUCCEEDED);
      }
    }
  }
Пример #4
0
 @Override
 public void onSucceeded(Guid cmdId, List<Guid> childCmdIds) {
   RemoveVmPoolCommand<? extends VmPoolParametersBase> cmd = getCommand(cmdId);
   CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
   cmd.getParameters().setTaskGroupSuccess(true);
   cmd.endAction();
 }
  @Override
  public void doPolling(Guid cmdId, List<Guid> childCmdIds) {

    boolean anyFailed = false;
    for (Guid childCmdId : childCmdIds) {
      switch (CommandCoordinatorUtil.getCommandStatus(childCmdId)) {
        case ACTIVE:
          log.info("Waiting on Live Merge child commands to complete");
          return;
        case FAILED:
        case FAILED_RESTARTED:
        case UNKNOWN:
          anyFailed = true;
          break;
        default:
          break;
      }
    }

    RemoveSnapshotCommand<RemoveSnapshotParameters> command = getCommand(cmdId);
    command.getParameters().setTaskGroupSuccess(!anyFailed);
    command.setCommandStatus(anyFailed ? CommandStatus.FAILED : CommandStatus.SUCCEEDED);
    log.info(
        "All Live Merge child commands have completed, status '{}'", command.getCommandStatus());
  }
 @Override
 public void onFailed(Guid cmdId, List<Guid> childCmdIds) {
   super.onFailed(cmdId, childCmdIds);
   getCommand().getParameters().setTaskGroupSuccess(false);
   log.error("Failed deleting volume/snapshot from Cinder. ID: {}", getDiskId());
   getCommand().endAction();
   CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
 }
  /**
   * This method is called upon the bean creation as part of the management Service bean life cycle.
   */
  @Override
  @PostConstruct
  public void create() {

    try {
      // This must be done before starting to sample the hosts status from VDSM since the sampling
      // will turn such host from Reboot to NonResponsive
      loadService(PmHealthCheckManager.class);
      loadService(EngineBackupAwarenessManager.class);
      CommandCoordinatorUtil.initAsyncTaskManager();
      loadService(ResourceManager.class);
      OvfDataUpdater.getInstance().initOvfDataUpdater();
      ThreadPoolUtil.execute(
          new Runnable() {
            @Override
            public void run() {
              MacPoolPerDcSingleton.getInstance().initialize();
            }
          });
      StoragePoolStatusHandler.init();

      GlusterJobsManager.init();

      try {
        log.info("Init VM custom properties utilities");
        VmPropertiesUtils.getInstance().init();
      } catch (InitializationException e) {
        log.error("Initialization of vm custom properties failed.", e);
      }

      try {
        log.info("Init device custom properties utilities");
        DevicePropertiesUtils.getInstance().init();
      } catch (InitializationException e) {
        log.error("Initialization of device custom properties failed.", e);
      }

      loadService(SchedulingManager.class);

      SessionDataContainer.getInstance().cleanupEngineSessionsOnStartup();

      loadService(HostDeviceManager.class);
      loadService(DwhHeartBeat.class);

      if (Config.<Boolean>getValue(ConfigValues.AffinityRulesEnforcementManagerEnabled)) {
        loadService(AffinityRulesEnforcementManager.class);
      }

      loadService(CertificationValidityChecker.class);
    } catch (Exception ex) {
      log.error("Failed to initialize backend", ex);
      throw ex;
    }
  }
 /**
  * Updates (but does not persist) the parameters.childCommands list to ensure the current child
  * command is present. This is necessary in various entry points called externally (e.g. by
  * endAction()), which can be called after a child command is started but before the main
  * proceedCommandExecution() loop has persisted the updated child list.
  */
 private void syncChildCommandList() {
   List<Guid> childCommandIds = CommandCoordinatorUtil.getChildCommandIds(getCommandId());
   if (childCommandIds.size() != getParameters().getChildCommands().size()) {
     for (Guid id : childCommandIds) {
       if (!getParameters().getChildCommands().containsValue(id)) {
         getParameters().getChildCommands().put(getParameters().getCommandStep(), id);
         break;
       }
     }
   }
 }
  @Override
  public void onSucceeded(Guid cmdId, List<Guid> childCmdIds) {
    super.onSucceeded(cmdId, childCmdIds);

    getCommand().performDiskUpdate();
    log.error("Disk has been successfully extended. ID: {}", getDiskId());
    updateAuditLog(AuditLogType.USER_EXTEND_DISK_SIZE_SUCCESS, getCommand().getNewDiskSizeInGB());

    getCommand().endAction();
    CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
  }
  @Override
  public void doPolling(Guid cmdId, List<Guid> childCmdIds) {
    super.doPolling(cmdId, childCmdIds);
    boolean anyFailed = false;
    for (Guid childCmdId : childCmdIds) {
      CommandStatus commandStatus = CommandCoordinatorUtil.getCommandStatus(childCmdId);
      switch (commandStatus) {
        case NOT_STARTED:
          break;
        case ACTIVE:
          log.info("Waiting on RemoveCinderDiskVolumeCommandCallback child commands to complete");
          return;
        case SUCCEEDED:
          if (!getCommand().getParameters().getFinishedChildCmdIds().contains(childCmdId)) {
            int removedVolumeIndex = getCommand().getParameters().getRemovedVolumeIndex();
            final CinderDisk cinderVolume =
                getCommand()
                    .getParameters()
                    .getChildCommandsParameters()
                    .get(removedVolumeIndex)
                    .getRemovedVolume();
            getCommand().removeDiskFromDbCallBack(cinderVolume);
            getCommand().getParameters().getFinishedChildCmdIds().add(childCmdId);
          }

          break;
        case FAILED:
        case FAILED_RESTARTED:
        case UNKNOWN:
          anyFailed = true;
          if (!getCommand().getParameters().getFinishedChildCmdIds().contains(childCmdId)) {
            getCommand().getParameters().getFinishedChildCmdIds().add(childCmdId);
          }
          break;
        default:
          log.error("Invalid command status: '{}", commandStatus);
          break;
      }
    }

    if (allChildCommandWereExecuted()) {
      getCommand().setCommandStatus(anyFailed ? CommandStatus.FAILED : CommandStatus.SUCCEEDED);
    } else if (allChildCmdIdsFinished(childCmdIds)) {
      if (anyFailed) {
        getCommand().setCommandStatus(CommandStatus.FAILED);
      } else {
        int removedVolumeIndex = getCommand().getParameters().getRemovedVolumeIndex();
        removedVolumeIndex++;
        getCommand().getParameters().setRemovedVolumeIndex(removedVolumeIndex);
        getCommand().removeCinderVolume(removedVolumeIndex);
      }
    }
  }
  @Override
  public void onFailed(Guid cmdId, List<Guid> childCmdIds) {
    super.onFailed(cmdId, childCmdIds);

    ImagesHandler.updateImageStatus(getDiskId(), ImageStatus.ILLEGAL);
    log.error("Failed extending disk. ID: {}", getDiskId());
    updateAuditLog(AuditLogType.USER_EXTEND_DISK_SIZE_FAILURE, getCommand().getNewDiskSizeInGB());

    getCommand().getParameters().setTaskGroupSuccess(false);
    getCommand().endAction();
    CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
  }
Пример #12
0
  private void extendImageSize() {
    Guid diskImageId = getParameters().getBaseImage().getImageId();
    long sizeInBytes = getParameters().getTopImage().getSize();
    ExtendImageSizeParameters parameters =
        new ExtendImageSizeParameters(diskImageId, sizeInBytes, true);
    parameters.setStoragePoolId(getParameters().getBaseImage().getStoragePoolId());
    parameters.setStorageDomainId(getParameters().getBaseImage().getStorageIds().get(0));
    parameters.setImageGroupID(getParameters().getBaseImage().getId());
    parameters.setParentCommand(VdcActionType.MergeExtend);
    parameters.setParentParameters(getParameters());

    CommandCoordinatorUtil.executeAsyncCommand(
        VdcActionType.ExtendImageSize, parameters, cloneContextAndDetachFromParent());
    log.info("Extending size of base volume {} to {} bytes", diskImageId, sizeInBytes);
  }
 /**
  * The following method performs a removing of all cinder disks from vm. These is only DB
  * operation
  */
 private void removeCinderDisks(List<CinderDisk> cinderDisks) {
   RemoveAllVmCinderDisksParameters removeParam =
       new RemoveAllVmCinderDisksParameters(getVmTemplateId(), cinderDisks);
   removeParam.setParentHasTasks(!getReturnValue().getVdsmTaskIdList().isEmpty());
   Future<VdcReturnValueBase> future =
       CommandCoordinatorUtil.executeAsyncCommand(
           VdcActionType.RemoveAllVmCinderDisks,
           withRootCommandInfo(removeParam),
           cloneContextAndDetachFromParent(),
           CINDERStorageHelper.getStorageEntities(cinderDisks));
   try {
     future.get().getActionReturnValue();
   } catch (InterruptedException | ExecutionException e) {
     log.error("Exception", e);
   }
 }
Пример #14
0
  @Override
  protected void executeCommand() {
    VDSStatus statusBeforeUpgrade = getVds().getStatus();
    if (statusBeforeUpgrade != VDSStatus.Maintenance) {
      Future<VdcReturnValueBase> maintenanceCmd =
          CommandCoordinatorUtil.executeAsyncCommand(
              VdcActionType.MaintenanceNumberOfVdss, createMaintenanceParams(), cloneContext());

      VdcReturnValueBase result;
      try {
        result = maintenanceCmd.get();
        if (!result.getSucceeded()) {
          propagateFailure(result);
          return;
        }
      } catch (InterruptedException | ExecutionException e) {
        log.error("Exception", e);
        return;
      }
    }

    setSucceeded(true);
  }
Пример #15
0
 /**
  * The following method will perform a removing of all cinder disks from vm. These is only DB
  * operation
  */
 private Collection<CinderDisk> removeCinderDisks() {
   Collection<CinderDisk> failedRemoveCinderDisks = null;
   if (getParameters().isRemoveDisks()) {
     List<CinderDisk> cinderDisks = getCinderDisks();
     if (cinderDisks.isEmpty()) {
       return Collections.emptyList();
     }
     RemoveAllVmCinderDisksParameters param =
         new RemoveAllVmCinderDisksParameters(getVmId(), cinderDisks);
     param.setEndProcedure(EndProcedure.COMMAND_MANAGED);
     Future<VdcReturnValueBase> future =
         CommandCoordinatorUtil.executeAsyncCommand(
             VdcActionType.RemoveAllVmCinderDisks,
             withRootCommandInfo(param),
             cloneContextAndDetachFromParent());
     try {
       failedRemoveCinderDisks = future.get().getActionReturnValue();
     } catch (InterruptedException | ExecutionException e) {
       failedRemoveCinderDisks = cinderDisks;
       log.error("Exception", e);
     }
   }
   return failedRemoveCinderDisks;
 }
Пример #16
0
 @Override
 public void storagePoolUpEvent(StoragePool storagePool) {
   CommandCoordinatorUtil.addStoragePoolExistingTasks(storagePool);
 }
 private void endAction(Guid commandId, boolean succeeded) {
   RemoveSnapshotCommand<RemoveSnapshotParameters> command = getCommand(commandId);
   command.endAction();
   CommandCoordinatorUtil.removeAllCommandsInHierarchy(commandId);
   ExecutionHandler.endJob(command.getExecutionContext(), succeeded);
 }
Пример #18
0
 private RemoveVmPoolCommand<? extends VmPoolParametersBase> getCommand(Guid cmdId) {
   return CommandCoordinatorUtil.retrieveCommand(cmdId);
 }
 private RemoveSnapshotCommand<RemoveSnapshotParameters> getCommand(Guid cmdId) {
   return CommandCoordinatorUtil.retrieveCommand(cmdId);
 }
 private T getCommand(Guid cmdId) {
   return CommandCoordinatorUtil.retrieveCommand(cmdId);
 }
 @Override
 public void onFailed(Guid cmdId, List<Guid> childCmdIds) {
   getCommand(cmdId).endAction();
   CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
 }
  public void proceedCommandExecution() {
    // Steps are executed such that:
    //  a) all logic before the command runs is idempotent
    //  b) the command is the last action in the step
    // This allows for recovery after a crash at any point during command execution.

    log.debug("Proceeding with execution of RemoveSnapshotSingleDiskLiveCommand");
    if (getParameters().getCommandStep() == null) {
      getParameters().setCommandStep(getInitialMergeStepForImage(getParameters().getImageId()));
      getParameters().setChildCommands(new HashMap<RemoveSnapshotSingleDiskLiveStep, Guid>());
    }

    // Upon recovery or after invoking a new child command, our map may be missing an entry
    syncChildCommandList();
    Guid currentChildId = getCurrentChildId();

    VdcReturnValueBase vdcReturnValue = null;
    if (currentChildId != null) {
      switch (CommandCoordinatorUtil.getCommandStatus(currentChildId)) {
        case ACTIVE:
        case NOT_STARTED:
          log.info(
              "Waiting on Live Merge command step '{}' to complete",
              getParameters().getCommandStep());
          return;

        case SUCCEEDED:
          CommandEntity cmdEntity = CommandCoordinatorUtil.getCommandEntity(currentChildId);
          if (cmdEntity.isCallbackEnabled() && !cmdEntity.isCallbackNotified()) {
            log.info(
                "Waiting on Live Merge command step '{}' to finalize",
                getParameters().getCommandStep());
            return;
          }

          vdcReturnValue = CommandCoordinatorUtil.getCommandReturnValue(currentChildId);
          if (vdcReturnValue != null && vdcReturnValue.getSucceeded()) {
            log.debug("Child command '{}' succeeded", getParameters().getCommandStep());
            getParameters().setCommandStep(getParameters().getNextCommandStep());
            break;
          } else {
            log.error(
                "Child command '{}' failed: {}",
                getParameters().getCommandStep(),
                (vdcReturnValue != null
                    ? vdcReturnValue.getExecuteFailedMessages()
                    : "null return value"));
            setCommandStatus(CommandStatus.FAILED);
            return;
          }

        case FAILED:
        case FAILED_RESTARTED:
          log.error("Failed child command status for step '{}'", getParameters().getCommandStep());
          setCommandStatus(CommandStatus.FAILED);
          return;

        case UNKNOWN:
          log.error("Unknown child command status for step '{}'", getParameters().getCommandStep());
          setCommandStatus(CommandStatus.FAILED);
          return;
      }
    }

    log.info("Executing Live Merge command step '{}'", getParameters().getCommandStep());

    Pair<VdcActionType, ? extends VdcActionParametersBase> nextCommand = null;
    switch (getParameters().getCommandStep()) {
      case EXTEND:
        nextCommand = new Pair<>(VdcActionType.MergeExtend, buildMergeParameters());
        getParameters().setNextCommandStep(RemoveSnapshotSingleDiskLiveStep.MERGE);
        break;
      case MERGE:
        nextCommand = new Pair<>(VdcActionType.Merge, buildMergeParameters());
        getParameters().setNextCommandStep(RemoveSnapshotSingleDiskLiveStep.MERGE_STATUS);
        break;
      case MERGE_STATUS:
        getParameters().setMergeCommandComplete(true);
        nextCommand = new Pair<>(VdcActionType.MergeStatus, buildMergeParameters());
        getParameters().setNextCommandStep(RemoveSnapshotSingleDiskLiveStep.DESTROY_IMAGE);
        break;
      case DESTROY_IMAGE:
        if (vdcReturnValue != null) {
          getParameters()
              .setMergeStatusReturnValue(
                  (MergeStatusReturnValue) vdcReturnValue.getActionReturnValue());
        } else if (getParameters().getMergeStatusReturnValue() == null) {
          // If the images were already merged, just add the orphaned image
          getParameters().setMergeStatusReturnValue(synthesizeMergeStatusReturnValue());
        }
        nextCommand = new Pair<>(VdcActionType.DestroyImage, buildDestroyImageParameters());
        getParameters().setNextCommandStep(RemoveSnapshotSingleDiskLiveStep.COMPLETE);
        break;
      case COMPLETE:
        getParameters().setDestroyImageCommandComplete(true);
        setCommandStatus(CommandStatus.SUCCEEDED);
        break;
    }

    persistCommand(getParameters().getParentCommand(), true);
    if (nextCommand != null) {
      CommandCoordinatorUtil.executeAsyncCommand(
          nextCommand.getFirst(), nextCommand.getSecond(), cloneContextAndDetachFromParent());
      // Add the child, but wait, it's a race!  child will start, task may spawn, get polled, and we
      // won't have the child id
    }
  }
Пример #23
0
  @Override
  protected boolean validate() {
    if (getVm() == null) {
      return failValidation(EngineMessage.ACTION_TYPE_FAILED_VM_NOT_FOUND);
    }

    if (!canRunActionOnNonManagedVm()) {
      return false;
    }

    if (getVm().isDeleteProtected()) {
      return failValidation(EngineMessage.ACTION_TYPE_FAILED_DELETE_PROTECTION_ENABLED);
    }

    vmHandler.updateDisksFromDb(getVm());
    getParameters()
        .setUseCinderCommandCallback(
            getParameters().isRemoveDisks() && !getCinderDisks().isEmpty());

    if (!getParameters().isRemoveDisks() && !canRemoveVmWithDetachDisks()) {
      return false;
    }

    switch (getVm().getStatus()) {
      case Unassigned:
      case Down:
      case ImageIllegal:
      case ImageLocked:
        break;
      case Suspended:
        return failValidation(EngineMessage.VM_CANNOT_REMOVE_VM_WHEN_STATUS_IS_NOT_DOWN);
      default:
        return (getVm().isHostedEngine() && isInternalExecution())
            || failValidation(EngineMessage.ACTION_TYPE_FAILED_VM_IS_RUNNING);
    }

    if (getVm().getVmPoolId() != null) {
      return failValidation(EngineMessage.ACTION_TYPE_FAILED_VM_ATTACHED_TO_POOL);
    }

    // enable to remove vms without images
    SnapshotsValidator snapshotsValidator = new SnapshotsValidator();
    if (!validate(snapshotsValidator.vmNotDuringSnapshot(getVmId()))) {
      return false;
    }

    if (!getVm().getDiskMap().isEmpty()
        && !validate(new StoragePoolValidator(getStoragePool()).isUp())) {
      return false;
    }

    Collection<Disk> vmDisks = getVm().getDiskMap().values();
    List<DiskImage> vmImages =
        DisksFilter.filterImageDisks(vmDisks, ONLY_NOT_SHAREABLE, ONLY_ACTIVE);
    vmImages.addAll(DisksFilter.filterCinderDisks(vmDisks));
    if (!vmImages.isEmpty()) {
      Set<Guid> storageIds = ImagesHandler.getAllStorageIdsForImageIds(vmImages);
      MultipleStorageDomainsValidator storageValidator =
          new MultipleStorageDomainsValidator(getVm().getStoragePoolId(), storageIds);
      if (!validate(storageValidator.allDomainsExistAndActive())) {
        return false;
      }

      DiskImagesValidator diskImagesValidator = new DiskImagesValidator(vmImages);
      if (!getParameters().getForce() && !validate(diskImagesValidator.diskImagesNotLocked())) {
        return false;
      }
    }

    // Handle VM status with ImageLocked
    VmValidator vmValidator = new VmValidator(getVm());
    ValidationResult vmLockedValidatorResult = vmValidator.vmNotLocked();
    if (!vmLockedValidatorResult.isValid()) {
      // without force remove, we can't remove the VM
      if (!getParameters().getForce()) {
        return failValidation(vmLockedValidatorResult.getMessages());
      }

      // If it is force, we cannot remove if there are task
      if (CommandCoordinatorUtil.hasTasksByStoragePoolId(getVm().getStoragePoolId())) {
        return failValidation(EngineMessage.VM_CANNOT_REMOVE_HAS_RUNNING_TASKS);
      }
    }

    if (getParameters().isRemoveDisks()
        && !validate(vmValidator.vmNotHavingDeviceSnapshotsAttachedToOtherVms(false))) {
      return false;
    }

    return true;
  }