private static CommandEntity getCommandEntity(CommandCoordinator coco, Guid cmdId) {
CommandEntity cmdEntity = coco.getCommandEntity(cmdId);
if (cmdEntity == null) {
cmdEntity =
coco.createCommandEntity(cmdId, VdcActionType.Unknown, new VdcActionParametersBase());
}
return cmdEntity;
}
/**
* Call VDSCommand for each pool id fetched from poolsOfActiveTasks , and Initialize a map with
* each storage pool Id task statuses.
*
* @param poolsOfActiveTasks - Set of all the active tasks fetched from _tasks.
* @return poolsAsyncTaskMap - Map which contains tasks for each storage pool id.
*/
private Map<Guid, Map<Guid, AsyncTaskStatus>> getSPMsTasksStatuses(Set<Guid> poolsOfActiveTasks) {
Map<Guid, Map<Guid, AsyncTaskStatus>> poolsAsyncTaskMap = new HashMap<>();
// For each pool Id (SPM) ,add its tasks to the map.
for (Guid storagePoolID : poolsOfActiveTasks) {
try {
Map<Guid, AsyncTaskStatus> map = coco.getAllTasksStatuses(storagePoolID);
if (map != null) {
poolsAsyncTaskMap.put(storagePoolID, map);
}
} catch (RuntimeException e) {
if ((e instanceof EngineException)
&& (((EngineException) e).getErrorCode() == EngineError.VDS_NETWORK_ERROR)) {
log.debug(
"Get SPM task statuses: Calling Command {}VDSCommand, "
+ "with storagePoolId '{}') threw an exception.",
VDSCommandType.SPMGetAllTasksStatuses,
storagePoolID);
} else {
log.debug(
"Get SPM task statuses: Calling Command {}VDSCommand, "
+ "with storagePoolId '{}') threw an exception: {}",
VDSCommandType.SPMGetAllTasksStatuses,
storagePoolID,
e.getMessage());
}
log.debug("Exception", e);
}
}
return poolsAsyncTaskMap;
}
public void logAndFailPartiallySubmittedTaskOfCommand(final AsyncTask task, String message) {
log.info(
"Failing partially submitted task AsyncTaskType '{}': Task '{}' Parent Command '{}'",
task.getTaskType(),
task.getTaskId(),
task.getActionType());
task.getTaskParameters().setTaskGroupSuccess(false);
if (task.getActionType() == VdcActionType.Unknown) {
removeTaskFromDbByTaskId(task.getTaskId());
log.info(
"Not calling endAction for partially submitted task and AsyncTaskType '{}': Task '{}' Parent Command '{}'",
task.getTaskType(),
task.getTaskId(),
task.getActionType());
return;
}
log.info(
"Calling updateTask for partially submitted task and AsyncTaskType '{}': Task '{}' Parent Command"
+ " '{}' Parameters class '{}'",
task.getTaskType(),
task.getTaskId(),
task.getActionType());
AsyncTaskCreationInfo creationInfo =
new AsyncTaskCreationInfo(Guid.Empty, task.getTaskType(), task.getStoragePoolId());
SPMTask spmTask = coco.construct(creationInfo, task);
AsyncTaskStatus failureStatus = new AsyncTaskStatus();
failureStatus.setStatus(AsyncTaskStatusEnum.finished);
failureStatus.setResult(AsyncTaskResultEnum.failure);
failureStatus.setMessage(message);
spmTask.setState(AsyncTaskState.Ended);
spmTask.setLastTaskStatus(failureStatus);
spmTask.updateTask(failureStatus);
}
// NOTE - any change to the logic here and in the async tasks managed command requires inspection
// of
// CommandExecutor.handleUnmanagedCommands()
public void initAsyncTaskManager() {
tasksInDbAfterRestart = new ConcurrentHashMap<>();
Map<Guid, List<AsyncTask>> rootCommandIdToTasksMap =
groupTasksByRootCommandId(coco.getAllAsyncTasksFromDb());
int numberOfCommandsPartiallyExecuted = 0;
for (Entry<Guid, List<AsyncTask>> entry : rootCommandIdToTasksMap.entrySet()) {
if (isPartiallyExecutedCommand(rootCommandIdToTasksMap.get(entry.getKey()))) {
log.info("Root Command '{}' has partially executed task.", entry.getKey());
numberOfCommandsPartiallyExecuted++;
}
}
irsBrokerLatch = new CountDownLatch(numberOfCommandsPartiallyExecuted);
for (Entry<Guid, List<AsyncTask>> entry : rootCommandIdToTasksMap.entrySet()) {
if (isPartiallyExecutedCommand(rootCommandIdToTasksMap.get(entry.getKey()))) {
log.info("Root Command '{}' has partially executed tasks.", entry.getKey());
handlePartiallyExecuteTasksOfCommand(rootCommandIdToTasksMap.get(entry.getKey()));
}
for (AsyncTask task : entry.getValue()) {
if (!isPartiallyExecutedTask(task)) {
tasksInDbAfterRestart.putIfAbsent(task.getStoragePoolId(), new ArrayList<>());
tasksInDbAfterRestart.get(task.getStoragePoolId()).add(task);
}
}
}
try {
irsBrokerLatch.await();
log.info("Initialization of AsyncTaskManager completed successfully.");
} catch (InterruptedException e) {
}
}
/**
* Constructs a task based on creation info (task type and task parameters as retrieved from the
* vdsm). Use in order to construct tasks when service is initializing.
*
* @param coco Handle to command coordinator
* @param creationInfo The Asyc Task Creation info
*/
public static SPMAsyncTask construct(
CommandCoordinator coco, AsyncTaskCreationInfo creationInfo) {
AsyncTask asyncTask = coco.getByVdsmTaskId(creationInfo.getVdsmTaskId());
if (asyncTask == null || asyncTask.getActionParameters() == null) {
asyncTask =
new AsyncTask(
AsyncTaskResultEnum.success,
AsyncTaskStatusEnum.running,
Guid.Empty,
creationInfo.getVdsmTaskId(),
creationInfo.getStepId(),
creationInfo.getStoragePoolID(),
creationInfo.getTaskType(),
getCommandEntity(
coco, asyncTask == null ? Guid.newGuid() : asyncTask.getRootCommandId()),
getCommandEntity(
coco, asyncTask == null ? Guid.newGuid() : asyncTask.getCommandId()));
creationInfo.setTaskType(AsyncTaskType.unknown);
}
AsyncTaskParameters asyncTaskParams = new AsyncTaskParameters(creationInfo, asyncTask);
return construct(coco, creationInfo.getTaskType(), asyncTaskParams, true);
}
/**
* Retrieves from the specified storage pool the tasks that exist on it and adds them to the
* manager.
*
* @param sp the storage pool to retrieve running tasks from
*/
public void addStoragePoolExistingTasks(StoragePool sp) {
List<AsyncTaskCreationInfo> currPoolTasks = null;
try {
currPoolTasks = coco.getAllTasksInfo(sp.getId());
} catch (RuntimeException e) {
log.error(
"Getting existing tasks on Storage Pool '{}' failed: {}", sp.getName(), e.getMessage());
log.debug("Exception", e);
}
if (currPoolTasks != null && currPoolTasks.size() > 0) {
synchronized (this) {
final List<SPMTask> newlyAddedTasks = new ArrayList<>();
for (AsyncTaskCreationInfo creationInfo : currPoolTasks) {
creationInfo.setStoragePoolID(sp.getId());
if (!_tasks.containsKey(creationInfo.getVdsmTaskId())) {
try {
SPMTask task;
if (partiallyCompletedCommandTasks.containsKey(creationInfo.getVdsmTaskId())) {
AsyncTask asyncTaskInDb =
partiallyCompletedCommandTasks.get(creationInfo.getVdsmTaskId());
task = coco.construct(creationInfo, asyncTaskInDb);
if (task.getEntitiesMap() == null) {
task.setEntitiesMap(new HashMap<>());
}
partiallyCompletedCommandTasks.remove(task.getVdsmTaskId());
// mark it as a task of a partially completed command
// Will result in failure of the command
task.setPartiallyCompletedCommandTask(true);
} else {
task = coco.construct(creationInfo);
}
addTaskToManager(task);
newlyAddedTasks.add(task);
} catch (Exception e) {
log.error(
"Failed to load task of type '{}' with id '{}': {}.",
creationInfo.getTaskType(),
creationInfo.getVdsmTaskId(),
ExceptionUtils.getRootCauseMessage(e));
log.debug("Exception", e);
}
}
}
TransactionSupport.executeInNewTransaction(
() -> {
for (SPMTask task : newlyAddedTasks) {
AsyncTaskUtils.addOrUpdateTaskInDB(task);
}
return null;
});
for (SPMTask task : newlyAddedTasks) {
startPollingTask(task.getVdsmTaskId());
}
log.info(
"Discovered {} tasks on Storage Pool '{}', {} added to manager.",
currPoolTasks.size(),
sp.getName(),
newlyAddedTasks.size());
}
} else {
log.info("Discovered no tasks on Storage Pool '{}'", sp.getName());
}
List<AsyncTask> tasksInDForStoragePool = tasksInDbAfterRestart.get(sp.getId());
if (tasksInDForStoragePool != null) {
for (AsyncTask task : tasksInDForStoragePool) {
if (!_tasks.containsKey(task.getVdsmTaskId())) {
coco.removeByVdsmTaskId(task.getVdsmTaskId());
}
}
}
// Either the tasks were only in DB - so they were removed from db, or they are polled -
// in any case no need to hold them in the map that represents the tasksInDbAfterRestart
tasksInDbAfterRestart.remove(sp.getId());
}
public SPMTask createTask(AsyncTaskType taskType, AsyncTaskParameters taskParameters) {
return coco.construct(taskType, taskParameters, false);
}
public void logAndFailTaskOfCommandWithEmptyVdsmId(Guid asyncTaskId, String message) {
AsyncTask task = coco.getAsyncTaskFromDb(asyncTaskId);
if (task != null) {
logAndFailPartiallySubmittedTaskOfCommand(task, message);
}
}