private void cleanupTaskDueToDecomission(
final Set<String> requestIdsToReschedule,
final Set<SingularityTaskId> matchingTaskIds,
SingularityTask task,
SingularityMachineAbstraction<?> decommissioningObject) {
requestIdsToReschedule.add(task.getTaskRequest().getRequest().getId());
matchingTaskIds.add(task.getTaskId());
LOG.trace(
"Scheduling a cleanup task for {} due to decomissioning {}",
task.getTaskId(),
decommissioningObject);
taskManager.createTaskCleanup(
new SingularityTaskCleanup(
decommissioningObject.getCurrentState().getUser(),
TaskCleanupType.DECOMISSIONING,
System.currentTimeMillis(),
task.getTaskId(),
Optional.of(
String.format(
"%s %s is decomissioning",
decommissioningObject.getTypeName(), decommissioningObject.getName()))));
}
public void saveResult(
Optional<Integer> statusCode, Optional<String> responseBody, Optional<String> errorMessage) {
SingularityTaskHealthcheckResult result =
new SingularityTaskHealthcheckResult(
statusCode,
Optional.of(System.currentTimeMillis() - startTime),
startTime,
responseBody,
errorMessage,
task.getTaskId());
LOG.trace("Saving healthcheck result {}", result);
try {
taskManager.saveHealthcheckResult(result);
if (result.isFailed()) {
if (!taskManager.isActiveTask(task.getTaskId().getId())) {
LOG.trace("Task {} is not active, not re-enqueueing healthcheck", task.getTaskId());
return;
}
healthchecker.enqueueHealthcheck(task);
} else {
newTaskChecker.runNewTaskCheckImmediately(task);
}
} catch (Throwable t) {
LOG.error("Caught throwable while saving health check result {}, will re-enqueue", result, t);
exceptionNotifier.notify(t);
reEnqueueOrAbort(task);
}
}
@Override
public void onThrowable(Throwable t) {
LOG.trace("Exception while making health check for task {}", task.getTaskId(), t);
saveResult(
Optional.<Integer>absent(),
Optional.<String>absent(),
Optional.of(String.format("Healthcheck failed due to exception: %s", t.getMessage())));
}
private void reEnqueueOrAbort(SingularityTask task) {
try {
healthchecker.enqueueHealthcheck(task);
} catch (Throwable t) {
LOG.error(
"Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t);
exceptionNotifier.notify(t);
abort.abort(AbortReason.UNRECOVERABLE_ERROR);
}
}