@Override
public void doPolling(Guid cmdId, List<Guid> childCmdIds) {
boolean anyFailed = false;
for (Guid childCmdId : childCmdIds) {
CommandStatus commandStatus = CommandCoordinatorUtil.getCommandStatus(childCmdId);
switch (commandStatus) {
case NOT_STARTED:
case ACTIVE:
log.info("Waiting for child commands to complete");
return;
case SUCCEEDED:
break;
case FAILED:
case FAILED_RESTARTED:
case UNKNOWN:
anyFailed = true;
break;
default:
log.error("Invalid command status: '{}", commandStatus);
break;
}
}
T command = getCommand(cmdId);
command.getParameters().setTaskGroupSuccess(!anyFailed);
command.setCommandStatus(anyFailed ? CommandStatus.FAILED : CommandStatus.SUCCEEDED);
log.info("All commands have completed, status '{}'", command.getCommandStatus());
}
@Override
public void onSucceeded(Guid cmdId, List<Guid> childCmdIds) {
super.onSucceeded(cmdId, childCmdIds);
log.info("Volume/Snapshot has been successfully deleted from Cinder. ID: {}", getDiskId());
getCommand().endAction();
CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
}
@Override
public void doPolling(Guid cmdId, List<Guid> childCmdIds) {
RemoveVmPoolCommand<? extends VmPoolParametersBase> command = getCommand(cmdId);
boolean anyFailed = false;
for (Guid childCmdId : childCmdIds) {
CommandEntity entity = CommandCoordinatorUtil.getCommandEntity(childCmdId);
switch (entity.getCommandStatus()) {
case ENDED_WITH_FAILURE:
case FAILED:
case EXECUTION_FAILED:
case UNKNOWN:
anyFailed = true;
break;
default:
break;
}
}
if (anyFailed) {
command.setCommandStatus(CommandStatus.FAILED);
} else {
VmPool pool = DbFacade.getInstance().getVmPoolDao().get(command.getVmPoolId());
if (pool == null || pool.getRunningVmsCount() == 0) {
command.setCommandStatus(CommandStatus.SUCCEEDED);
}
}
}
@Override
public void onSucceeded(Guid cmdId, List<Guid> childCmdIds) {
RemoveVmPoolCommand<? extends VmPoolParametersBase> cmd = getCommand(cmdId);
CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
cmd.getParameters().setTaskGroupSuccess(true);
cmd.endAction();
}
@Override
public void doPolling(Guid cmdId, List<Guid> childCmdIds) {
boolean anyFailed = false;
for (Guid childCmdId : childCmdIds) {
switch (CommandCoordinatorUtil.getCommandStatus(childCmdId)) {
case ACTIVE:
log.info("Waiting on Live Merge child commands to complete");
return;
case FAILED:
case FAILED_RESTARTED:
case UNKNOWN:
anyFailed = true;
break;
default:
break;
}
}
RemoveSnapshotCommand<RemoveSnapshotParameters> command = getCommand(cmdId);
command.getParameters().setTaskGroupSuccess(!anyFailed);
command.setCommandStatus(anyFailed ? CommandStatus.FAILED : CommandStatus.SUCCEEDED);
log.info(
"All Live Merge child commands have completed, status '{}'", command.getCommandStatus());
}
@Override
public void onFailed(Guid cmdId, List<Guid> childCmdIds) {
super.onFailed(cmdId, childCmdIds);
getCommand().getParameters().setTaskGroupSuccess(false);
log.error("Failed deleting volume/snapshot from Cinder. ID: {}", getDiskId());
getCommand().endAction();
CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
}
/**
* This method is called upon the bean creation as part of the management Service bean life cycle.
*/
@Override
@PostConstruct
public void create() {
try {
// This must be done before starting to sample the hosts status from VDSM since the sampling
// will turn such host from Reboot to NonResponsive
loadService(PmHealthCheckManager.class);
loadService(EngineBackupAwarenessManager.class);
CommandCoordinatorUtil.initAsyncTaskManager();
loadService(ResourceManager.class);
OvfDataUpdater.getInstance().initOvfDataUpdater();
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
MacPoolPerDcSingleton.getInstance().initialize();
}
});
StoragePoolStatusHandler.init();
GlusterJobsManager.init();
try {
log.info("Init VM custom properties utilities");
VmPropertiesUtils.getInstance().init();
} catch (InitializationException e) {
log.error("Initialization of vm custom properties failed.", e);
}
try {
log.info("Init device custom properties utilities");
DevicePropertiesUtils.getInstance().init();
} catch (InitializationException e) {
log.error("Initialization of device custom properties failed.", e);
}
loadService(SchedulingManager.class);
SessionDataContainer.getInstance().cleanupEngineSessionsOnStartup();
loadService(HostDeviceManager.class);
loadService(DwhHeartBeat.class);
if (Config.<Boolean>getValue(ConfigValues.AffinityRulesEnforcementManagerEnabled)) {
loadService(AffinityRulesEnforcementManager.class);
}
loadService(CertificationValidityChecker.class);
} catch (Exception ex) {
log.error("Failed to initialize backend", ex);
throw ex;
}
}
/**
* Updates (but does not persist) the parameters.childCommands list to ensure the current child
* command is present. This is necessary in various entry points called externally (e.g. by
* endAction()), which can be called after a child command is started but before the main
* proceedCommandExecution() loop has persisted the updated child list.
*/
private void syncChildCommandList() {
List<Guid> childCommandIds = CommandCoordinatorUtil.getChildCommandIds(getCommandId());
if (childCommandIds.size() != getParameters().getChildCommands().size()) {
for (Guid id : childCommandIds) {
if (!getParameters().getChildCommands().containsValue(id)) {
getParameters().getChildCommands().put(getParameters().getCommandStep(), id);
break;
}
}
}
}
@Override
public void onSucceeded(Guid cmdId, List<Guid> childCmdIds) {
super.onSucceeded(cmdId, childCmdIds);
getCommand().performDiskUpdate();
log.error("Disk has been successfully extended. ID: {}", getDiskId());
updateAuditLog(AuditLogType.USER_EXTEND_DISK_SIZE_SUCCESS, getCommand().getNewDiskSizeInGB());
getCommand().endAction();
CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
}
@Override
public void doPolling(Guid cmdId, List<Guid> childCmdIds) {
super.doPolling(cmdId, childCmdIds);
boolean anyFailed = false;
for (Guid childCmdId : childCmdIds) {
CommandStatus commandStatus = CommandCoordinatorUtil.getCommandStatus(childCmdId);
switch (commandStatus) {
case NOT_STARTED:
break;
case ACTIVE:
log.info("Waiting on RemoveCinderDiskVolumeCommandCallback child commands to complete");
return;
case SUCCEEDED:
if (!getCommand().getParameters().getFinishedChildCmdIds().contains(childCmdId)) {
int removedVolumeIndex = getCommand().getParameters().getRemovedVolumeIndex();
final CinderDisk cinderVolume =
getCommand()
.getParameters()
.getChildCommandsParameters()
.get(removedVolumeIndex)
.getRemovedVolume();
getCommand().removeDiskFromDbCallBack(cinderVolume);
getCommand().getParameters().getFinishedChildCmdIds().add(childCmdId);
}
break;
case FAILED:
case FAILED_RESTARTED:
case UNKNOWN:
anyFailed = true;
if (!getCommand().getParameters().getFinishedChildCmdIds().contains(childCmdId)) {
getCommand().getParameters().getFinishedChildCmdIds().add(childCmdId);
}
break;
default:
log.error("Invalid command status: '{}", commandStatus);
break;
}
}
if (allChildCommandWereExecuted()) {
getCommand().setCommandStatus(anyFailed ? CommandStatus.FAILED : CommandStatus.SUCCEEDED);
} else if (allChildCmdIdsFinished(childCmdIds)) {
if (anyFailed) {
getCommand().setCommandStatus(CommandStatus.FAILED);
} else {
int removedVolumeIndex = getCommand().getParameters().getRemovedVolumeIndex();
removedVolumeIndex++;
getCommand().getParameters().setRemovedVolumeIndex(removedVolumeIndex);
getCommand().removeCinderVolume(removedVolumeIndex);
}
}
}
@Override
public void onFailed(Guid cmdId, List<Guid> childCmdIds) {
super.onFailed(cmdId, childCmdIds);
ImagesHandler.updateImageStatus(getDiskId(), ImageStatus.ILLEGAL);
log.error("Failed extending disk. ID: {}", getDiskId());
updateAuditLog(AuditLogType.USER_EXTEND_DISK_SIZE_FAILURE, getCommand().getNewDiskSizeInGB());
getCommand().getParameters().setTaskGroupSuccess(false);
getCommand().endAction();
CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
}
private void extendImageSize() {
Guid diskImageId = getParameters().getBaseImage().getImageId();
long sizeInBytes = getParameters().getTopImage().getSize();
ExtendImageSizeParameters parameters =
new ExtendImageSizeParameters(diskImageId, sizeInBytes, true);
parameters.setStoragePoolId(getParameters().getBaseImage().getStoragePoolId());
parameters.setStorageDomainId(getParameters().getBaseImage().getStorageIds().get(0));
parameters.setImageGroupID(getParameters().getBaseImage().getId());
parameters.setParentCommand(VdcActionType.MergeExtend);
parameters.setParentParameters(getParameters());
CommandCoordinatorUtil.executeAsyncCommand(
VdcActionType.ExtendImageSize, parameters, cloneContextAndDetachFromParent());
log.info("Extending size of base volume {} to {} bytes", diskImageId, sizeInBytes);
}
/**
* The following method performs a removing of all cinder disks from vm. These is only DB
* operation
*/
private void removeCinderDisks(List<CinderDisk> cinderDisks) {
RemoveAllVmCinderDisksParameters removeParam =
new RemoveAllVmCinderDisksParameters(getVmTemplateId(), cinderDisks);
removeParam.setParentHasTasks(!getReturnValue().getVdsmTaskIdList().isEmpty());
Future<VdcReturnValueBase> future =
CommandCoordinatorUtil.executeAsyncCommand(
VdcActionType.RemoveAllVmCinderDisks,
withRootCommandInfo(removeParam),
cloneContextAndDetachFromParent(),
CINDERStorageHelper.getStorageEntities(cinderDisks));
try {
future.get().getActionReturnValue();
} catch (InterruptedException | ExecutionException e) {
log.error("Exception", e);
}
}
@Override
protected void executeCommand() {
VDSStatus statusBeforeUpgrade = getVds().getStatus();
if (statusBeforeUpgrade != VDSStatus.Maintenance) {
Future<VdcReturnValueBase> maintenanceCmd =
CommandCoordinatorUtil.executeAsyncCommand(
VdcActionType.MaintenanceNumberOfVdss, createMaintenanceParams(), cloneContext());
VdcReturnValueBase result;
try {
result = maintenanceCmd.get();
if (!result.getSucceeded()) {
propagateFailure(result);
return;
}
} catch (InterruptedException | ExecutionException e) {
log.error("Exception", e);
return;
}
}
setSucceeded(true);
}
/**
* The following method will perform a removing of all cinder disks from vm. These is only DB
* operation
*/
private Collection<CinderDisk> removeCinderDisks() {
Collection<CinderDisk> failedRemoveCinderDisks = null;
if (getParameters().isRemoveDisks()) {
List<CinderDisk> cinderDisks = getCinderDisks();
if (cinderDisks.isEmpty()) {
return Collections.emptyList();
}
RemoveAllVmCinderDisksParameters param =
new RemoveAllVmCinderDisksParameters(getVmId(), cinderDisks);
param.setEndProcedure(EndProcedure.COMMAND_MANAGED);
Future<VdcReturnValueBase> future =
CommandCoordinatorUtil.executeAsyncCommand(
VdcActionType.RemoveAllVmCinderDisks,
withRootCommandInfo(param),
cloneContextAndDetachFromParent());
try {
failedRemoveCinderDisks = future.get().getActionReturnValue();
} catch (InterruptedException | ExecutionException e) {
failedRemoveCinderDisks = cinderDisks;
log.error("Exception", e);
}
}
return failedRemoveCinderDisks;
}
@Override
public void storagePoolUpEvent(StoragePool storagePool) {
CommandCoordinatorUtil.addStoragePoolExistingTasks(storagePool);
}
private void endAction(Guid commandId, boolean succeeded) {
RemoveSnapshotCommand<RemoveSnapshotParameters> command = getCommand(commandId);
command.endAction();
CommandCoordinatorUtil.removeAllCommandsInHierarchy(commandId);
ExecutionHandler.endJob(command.getExecutionContext(), succeeded);
}
private RemoveVmPoolCommand<? extends VmPoolParametersBase> getCommand(Guid cmdId) {
return CommandCoordinatorUtil.retrieveCommand(cmdId);
}
private RemoveSnapshotCommand<RemoveSnapshotParameters> getCommand(Guid cmdId) {
return CommandCoordinatorUtil.retrieveCommand(cmdId);
}
private T getCommand(Guid cmdId) {
return CommandCoordinatorUtil.retrieveCommand(cmdId);
}
@Override
public void onFailed(Guid cmdId, List<Guid> childCmdIds) {
getCommand(cmdId).endAction();
CommandCoordinatorUtil.removeAllCommandsInHierarchy(cmdId);
}
public void proceedCommandExecution() {
// Steps are executed such that:
// a) all logic before the command runs is idempotent
// b) the command is the last action in the step
// This allows for recovery after a crash at any point during command execution.
log.debug("Proceeding with execution of RemoveSnapshotSingleDiskLiveCommand");
if (getParameters().getCommandStep() == null) {
getParameters().setCommandStep(getInitialMergeStepForImage(getParameters().getImageId()));
getParameters().setChildCommands(new HashMap<RemoveSnapshotSingleDiskLiveStep, Guid>());
}
// Upon recovery or after invoking a new child command, our map may be missing an entry
syncChildCommandList();
Guid currentChildId = getCurrentChildId();
VdcReturnValueBase vdcReturnValue = null;
if (currentChildId != null) {
switch (CommandCoordinatorUtil.getCommandStatus(currentChildId)) {
case ACTIVE:
case NOT_STARTED:
log.info(
"Waiting on Live Merge command step '{}' to complete",
getParameters().getCommandStep());
return;
case SUCCEEDED:
CommandEntity cmdEntity = CommandCoordinatorUtil.getCommandEntity(currentChildId);
if (cmdEntity.isCallbackEnabled() && !cmdEntity.isCallbackNotified()) {
log.info(
"Waiting on Live Merge command step '{}' to finalize",
getParameters().getCommandStep());
return;
}
vdcReturnValue = CommandCoordinatorUtil.getCommandReturnValue(currentChildId);
if (vdcReturnValue != null && vdcReturnValue.getSucceeded()) {
log.debug("Child command '{}' succeeded", getParameters().getCommandStep());
getParameters().setCommandStep(getParameters().getNextCommandStep());
break;
} else {
log.error(
"Child command '{}' failed: {}",
getParameters().getCommandStep(),
(vdcReturnValue != null
? vdcReturnValue.getExecuteFailedMessages()
: "null return value"));
setCommandStatus(CommandStatus.FAILED);
return;
}
case FAILED:
case FAILED_RESTARTED:
log.error("Failed child command status for step '{}'", getParameters().getCommandStep());
setCommandStatus(CommandStatus.FAILED);
return;
case UNKNOWN:
log.error("Unknown child command status for step '{}'", getParameters().getCommandStep());
setCommandStatus(CommandStatus.FAILED);
return;
}
}
log.info("Executing Live Merge command step '{}'", getParameters().getCommandStep());
Pair<VdcActionType, ? extends VdcActionParametersBase> nextCommand = null;
switch (getParameters().getCommandStep()) {
case EXTEND:
nextCommand = new Pair<>(VdcActionType.MergeExtend, buildMergeParameters());
getParameters().setNextCommandStep(RemoveSnapshotSingleDiskLiveStep.MERGE);
break;
case MERGE:
nextCommand = new Pair<>(VdcActionType.Merge, buildMergeParameters());
getParameters().setNextCommandStep(RemoveSnapshotSingleDiskLiveStep.MERGE_STATUS);
break;
case MERGE_STATUS:
getParameters().setMergeCommandComplete(true);
nextCommand = new Pair<>(VdcActionType.MergeStatus, buildMergeParameters());
getParameters().setNextCommandStep(RemoveSnapshotSingleDiskLiveStep.DESTROY_IMAGE);
break;
case DESTROY_IMAGE:
if (vdcReturnValue != null) {
getParameters()
.setMergeStatusReturnValue(
(MergeStatusReturnValue) vdcReturnValue.getActionReturnValue());
} else if (getParameters().getMergeStatusReturnValue() == null) {
// If the images were already merged, just add the orphaned image
getParameters().setMergeStatusReturnValue(synthesizeMergeStatusReturnValue());
}
nextCommand = new Pair<>(VdcActionType.DestroyImage, buildDestroyImageParameters());
getParameters().setNextCommandStep(RemoveSnapshotSingleDiskLiveStep.COMPLETE);
break;
case COMPLETE:
getParameters().setDestroyImageCommandComplete(true);
setCommandStatus(CommandStatus.SUCCEEDED);
break;
}
persistCommand(getParameters().getParentCommand(), true);
if (nextCommand != null) {
CommandCoordinatorUtil.executeAsyncCommand(
nextCommand.getFirst(), nextCommand.getSecond(), cloneContextAndDetachFromParent());
// Add the child, but wait, it's a race! child will start, task may spawn, get polled, and we
// won't have the child id
}
}
@Override
protected boolean validate() {
if (getVm() == null) {
return failValidation(EngineMessage.ACTION_TYPE_FAILED_VM_NOT_FOUND);
}
if (!canRunActionOnNonManagedVm()) {
return false;
}
if (getVm().isDeleteProtected()) {
return failValidation(EngineMessage.ACTION_TYPE_FAILED_DELETE_PROTECTION_ENABLED);
}
vmHandler.updateDisksFromDb(getVm());
getParameters()
.setUseCinderCommandCallback(
getParameters().isRemoveDisks() && !getCinderDisks().isEmpty());
if (!getParameters().isRemoveDisks() && !canRemoveVmWithDetachDisks()) {
return false;
}
switch (getVm().getStatus()) {
case Unassigned:
case Down:
case ImageIllegal:
case ImageLocked:
break;
case Suspended:
return failValidation(EngineMessage.VM_CANNOT_REMOVE_VM_WHEN_STATUS_IS_NOT_DOWN);
default:
return (getVm().isHostedEngine() && isInternalExecution())
|| failValidation(EngineMessage.ACTION_TYPE_FAILED_VM_IS_RUNNING);
}
if (getVm().getVmPoolId() != null) {
return failValidation(EngineMessage.ACTION_TYPE_FAILED_VM_ATTACHED_TO_POOL);
}
// enable to remove vms without images
SnapshotsValidator snapshotsValidator = new SnapshotsValidator();
if (!validate(snapshotsValidator.vmNotDuringSnapshot(getVmId()))) {
return false;
}
if (!getVm().getDiskMap().isEmpty()
&& !validate(new StoragePoolValidator(getStoragePool()).isUp())) {
return false;
}
Collection<Disk> vmDisks = getVm().getDiskMap().values();
List<DiskImage> vmImages =
DisksFilter.filterImageDisks(vmDisks, ONLY_NOT_SHAREABLE, ONLY_ACTIVE);
vmImages.addAll(DisksFilter.filterCinderDisks(vmDisks));
if (!vmImages.isEmpty()) {
Set<Guid> storageIds = ImagesHandler.getAllStorageIdsForImageIds(vmImages);
MultipleStorageDomainsValidator storageValidator =
new MultipleStorageDomainsValidator(getVm().getStoragePoolId(), storageIds);
if (!validate(storageValidator.allDomainsExistAndActive())) {
return false;
}
DiskImagesValidator diskImagesValidator = new DiskImagesValidator(vmImages);
if (!getParameters().getForce() && !validate(diskImagesValidator.diskImagesNotLocked())) {
return false;
}
}
// Handle VM status with ImageLocked
VmValidator vmValidator = new VmValidator(getVm());
ValidationResult vmLockedValidatorResult = vmValidator.vmNotLocked();
if (!vmLockedValidatorResult.isValid()) {
// without force remove, we can't remove the VM
if (!getParameters().getForce()) {
return failValidation(vmLockedValidatorResult.getMessages());
}
// If it is force, we cannot remove if there are task
if (CommandCoordinatorUtil.hasTasksByStoragePoolId(getVm().getStoragePoolId())) {
return failValidation(EngineMessage.VM_CANNOT_REMOVE_HAS_RUNNING_TASKS);
}
}
if (getParameters().isRemoveDisks()
&& !validate(vmValidator.vmNotHavingDeviceSnapshotsAttachedToOtherVms(false))) {
return false;
}
return true;
}