/**
* We group the tasks by the root command id, so that we can identify the commands that were
* partially submitted to vdsm
*/
private Map<Guid, List<AsyncTask>> groupTasksByRootCommandId(List<AsyncTask> tasksInDB) {
Map<Guid, List<AsyncTask>> rootCommandIdToCommandsMap = new HashMap<>();
for (AsyncTask task : tasksInDB) {
MultiValueMapUtils.addToMap(task.getRootCommandId(), task, rootCommandIdToCommandsMap);
}
return rootCommandIdToCommandsMap;
}
// NOTE - any change to the logic here and in the async tasks managed command requires inspection
// of
// CommandExecutor.handleUnmanagedCommands()
public void initAsyncTaskManager() {
tasksInDbAfterRestart = new ConcurrentHashMap<>();
Map<Guid, List<AsyncTask>> rootCommandIdToTasksMap =
groupTasksByRootCommandId(coco.getAllAsyncTasksFromDb());
int numberOfCommandsPartiallyExecuted = 0;
for (Entry<Guid, List<AsyncTask>> entry : rootCommandIdToTasksMap.entrySet()) {
if (isPartiallyExecutedCommand(rootCommandIdToTasksMap.get(entry.getKey()))) {
log.info("Root Command '{}' has partially executed task.", entry.getKey());
numberOfCommandsPartiallyExecuted++;
}
}
irsBrokerLatch = new CountDownLatch(numberOfCommandsPartiallyExecuted);
for (Entry<Guid, List<AsyncTask>> entry : rootCommandIdToTasksMap.entrySet()) {
if (isPartiallyExecutedCommand(rootCommandIdToTasksMap.get(entry.getKey()))) {
log.info("Root Command '{}' has partially executed tasks.", entry.getKey());
handlePartiallyExecuteTasksOfCommand(rootCommandIdToTasksMap.get(entry.getKey()));
}
for (AsyncTask task : entry.getValue()) {
if (!isPartiallyExecutedTask(task)) {
tasksInDbAfterRestart.putIfAbsent(task.getStoragePoolId(), new ArrayList<>());
tasksInDbAfterRestart.get(task.getStoragePoolId()).add(task);
}
}
}
try {
irsBrokerLatch.await();
log.info("Initialization of AsyncTaskManager completed successfully.");
} catch (InterruptedException e) {
}
}
/**
* If a command is partially submitted to vdsm, the empty place holders can be removed from the
* database and we poll for the completion of tasks that were already submitted. Once the tasks
* that were submitted finish execution, they are marked to be failed by adding them to the
* partiallyCompletedCommandTasks list. If none of the tasks were submitted to vdsm, the empty
* place holders are deleted from the database and we endAction on the command with failure
*/
private void handlePartiallyExecutedTaskOfCommand(final AsyncTask task) {
if (hasEmptyVdsmId(task)) {
removeTaskFromDbByTaskId(task.getTaskId());
return;
}
partiallyCompletedCommandTasks.put(task.getVdsmTaskId(), task);
}
/**
* Constructs a task based on creation info (task type and task parameters as retrieved from the
* vdsm). Use in order to construct tasks when service is initializing.
*
* @param coco Handle to command coordinator
* @param creationInfo The Asyc Task Creation info
*/
public static SPMAsyncTask construct(
CommandCoordinator coco, AsyncTaskCreationInfo creationInfo) {
AsyncTask asyncTask = coco.getByVdsmTaskId(creationInfo.getVdsmTaskId());
if (asyncTask == null || asyncTask.getActionParameters() == null) {
asyncTask =
new AsyncTask(
AsyncTaskResultEnum.success,
AsyncTaskStatusEnum.running,
Guid.Empty,
creationInfo.getVdsmTaskId(),
creationInfo.getStepId(),
creationInfo.getStoragePoolID(),
creationInfo.getTaskType(),
getCommandEntity(
coco, asyncTask == null ? Guid.newGuid() : asyncTask.getRootCommandId()),
getCommandEntity(
coco, asyncTask == null ? Guid.newGuid() : asyncTask.getCommandId()));
creationInfo.setTaskType(AsyncTaskType.unknown);
}
AsyncTaskParameters asyncTaskParams = new AsyncTaskParameters(creationInfo, asyncTask);
return construct(coco, creationInfo.getTaskType(), asyncTaskParams, true);
}
/**
* Retrieves from the specified storage pool the tasks that exist on it and adds them to the
* manager.
*
* @param sp the storage pool to retrieve running tasks from
*/
public void addStoragePoolExistingTasks(StoragePool sp) {
List<AsyncTaskCreationInfo> currPoolTasks = null;
try {
currPoolTasks = coco.getAllTasksInfo(sp.getId());
} catch (RuntimeException e) {
log.error(
"Getting existing tasks on Storage Pool '{}' failed: {}", sp.getName(), e.getMessage());
log.debug("Exception", e);
}
if (currPoolTasks != null && currPoolTasks.size() > 0) {
synchronized (this) {
final List<SPMTask> newlyAddedTasks = new ArrayList<>();
for (AsyncTaskCreationInfo creationInfo : currPoolTasks) {
creationInfo.setStoragePoolID(sp.getId());
if (!_tasks.containsKey(creationInfo.getVdsmTaskId())) {
try {
SPMTask task;
if (partiallyCompletedCommandTasks.containsKey(creationInfo.getVdsmTaskId())) {
AsyncTask asyncTaskInDb =
partiallyCompletedCommandTasks.get(creationInfo.getVdsmTaskId());
task = coco.construct(creationInfo, asyncTaskInDb);
if (task.getEntitiesMap() == null) {
task.setEntitiesMap(new HashMap<>());
}
partiallyCompletedCommandTasks.remove(task.getVdsmTaskId());
// mark it as a task of a partially completed command
// Will result in failure of the command
task.setPartiallyCompletedCommandTask(true);
} else {
task = coco.construct(creationInfo);
}
addTaskToManager(task);
newlyAddedTasks.add(task);
} catch (Exception e) {
log.error(
"Failed to load task of type '{}' with id '{}': {}.",
creationInfo.getTaskType(),
creationInfo.getVdsmTaskId(),
ExceptionUtils.getRootCauseMessage(e));
log.debug("Exception", e);
}
}
}
TransactionSupport.executeInNewTransaction(
() -> {
for (SPMTask task : newlyAddedTasks) {
AsyncTaskUtils.addOrUpdateTaskInDB(task);
}
return null;
});
for (SPMTask task : newlyAddedTasks) {
startPollingTask(task.getVdsmTaskId());
}
log.info(
"Discovered {} tasks on Storage Pool '{}', {} added to manager.",
currPoolTasks.size(),
sp.getName(),
newlyAddedTasks.size());
}
} else {
log.info("Discovered no tasks on Storage Pool '{}'", sp.getName());
}
List<AsyncTask> tasksInDForStoragePool = tasksInDbAfterRestart.get(sp.getId());
if (tasksInDForStoragePool != null) {
for (AsyncTask task : tasksInDForStoragePool) {
if (!_tasks.containsKey(task.getVdsmTaskId())) {
coco.removeByVdsmTaskId(task.getVdsmTaskId());
}
}
}
// Either the tasks were only in DB - so they were removed from db, or they are polled -
// in any case no need to hold them in the map that represents the tasksInDbAfterRestart
tasksInDbAfterRestart.remove(sp.getId());
}
public void logAndFailPartiallySubmittedTaskOfCommand(final AsyncTask task, String message) {
log.info(
"Failing partially submitted task AsyncTaskType '{}': Task '{}' Parent Command '{}'",
task.getTaskType(),
task.getTaskId(),
task.getActionType());
task.getTaskParameters().setTaskGroupSuccess(false);
if (task.getActionType() == VdcActionType.Unknown) {
removeTaskFromDbByTaskId(task.getTaskId());
log.info(
"Not calling endAction for partially submitted task and AsyncTaskType '{}': Task '{}' Parent Command '{}'",
task.getTaskType(),
task.getTaskId(),
task.getActionType());
return;
}
log.info(
"Calling updateTask for partially submitted task and AsyncTaskType '{}': Task '{}' Parent Command"
+ " '{}' Parameters class '{}'",
task.getTaskType(),
task.getTaskId(),
task.getActionType());
AsyncTaskCreationInfo creationInfo =
new AsyncTaskCreationInfo(Guid.Empty, task.getTaskType(), task.getStoragePoolId());
SPMTask spmTask = coco.construct(creationInfo, task);
AsyncTaskStatus failureStatus = new AsyncTaskStatus();
failureStatus.setStatus(AsyncTaskStatusEnum.finished);
failureStatus.setResult(AsyncTaskResultEnum.failure);
failureStatus.setMessage(message);
spmTask.setState(AsyncTaskState.Ended);
spmTask.setLastTaskStatus(failureStatus);
spmTask.updateTask(failureStatus);
}
private static boolean isPartiallyExecutedTask(AsyncTask task) {
return !task.getChildCmdEntity().isExecuted();
}
private static boolean hasEmptyVdsmId(AsyncTask task) {
return Guid.Empty.equals(task.getVdsmTaskId());
}