public void saveResult(
Optional<Integer> statusCode, Optional<String> responseBody, Optional<String> errorMessage) {
SingularityTaskHealthcheckResult result =
new SingularityTaskHealthcheckResult(
statusCode,
Optional.of(System.currentTimeMillis() - startTime),
startTime,
responseBody,
errorMessage,
task.getTaskId());
LOG.trace("Saving healthcheck result {}", result);
try {
taskManager.saveHealthcheckResult(result);
if (result.isFailed()) {
if (!taskManager.isActiveTask(task.getTaskId().getId())) {
LOG.trace("Task {} is not active, not re-enqueueing healthcheck", task.getTaskId());
return;
}
healthchecker.enqueueHealthcheck(task);
} else {
newTaskChecker.runNewTaskCheckImmediately(task);
}
} catch (Throwable t) {
LOG.error("Caught throwable while saving health check result {}, will re-enqueue", result, t);
exceptionNotifier.notify(t);
reEnqueueOrAbort(task);
}
}
private void reEnqueueOrAbort(SingularityTask task) {
try {
healthchecker.enqueueHealthcheck(task);
} catch (Throwable t) {
LOG.error(
"Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t);
exceptionNotifier.notify(t);
abort.abort(AbortReason.UNRECOVERABLE_ERROR);
}
}
@Timed
public void checkStateAfterFinishedTask(
SingularityTaskId taskId, String slaveId, SingularitySchedulerStateCache stateCache) {
Optional<SingularitySlave> slave = slaveManager.getObject(slaveId);
if (!slave.isPresent()) {
final String message =
String.format("Couldn't find slave with id %s for task %s", slaveId, taskId);
LOG.warn(message);
exceptionNotifier.notify(
message, ImmutableMap.of("slaveId", slaveId, "taskId", taskId.toString()));
return;
}
if (slave.get().getCurrentState().getState() == MachineState.DECOMMISSIONING) {
if (!hasTaskLeftOnSlave(taskId, slaveId, stateCache)) {
slaveManager.changeState(
slave.get(), MachineState.DECOMMISSIONED, slave.get().getCurrentState().getUser());
}
}
Optional<SingularityRack> rack = rackManager.getObject(slave.get().getRackId());
if (!rack.isPresent()) {
final String message =
String.format(
"Couldn't find rack with id %s for task %s", slave.get().getRackId(), taskId);
LOG.warn(message);
exceptionNotifier.notify(
message, ImmutableMap.of("rackId", slave.get().getRackId(), "taskId", taskId.toString()));
return;
}
if (rack.get().getCurrentState().getState() == MachineState.DECOMMISSIONING) {
if (!hasTaskLeftOnRack(taskId, stateCache)) {
rackManager.changeState(
rack.get(), MachineState.DECOMMISSIONED, rack.get().getCurrentState().getUser());
}
}
}