public void saveResult( Optional<Integer> statusCode, Optional<String> responseBody, Optional<String> errorMessage) { SingularityTaskHealthcheckResult result = new SingularityTaskHealthcheckResult( statusCode, Optional.of(System.currentTimeMillis() - startTime), startTime, responseBody, errorMessage, task.getTaskId()); LOG.trace("Saving healthcheck result {}", result); try { taskManager.saveHealthcheckResult(result); if (result.isFailed()) { if (!taskManager.isActiveTask(task.getTaskId().getId())) { LOG.trace("Task {} is not active, not re-enqueueing healthcheck", task.getTaskId()); return; } healthchecker.enqueueHealthcheck(task); } else { newTaskChecker.runNewTaskCheckImmediately(task); } } catch (Throwable t) { LOG.error("Caught throwable while saving health check result {}, will re-enqueue", result, t); exceptionNotifier.notify(t); reEnqueueOrAbort(task); } }
private void reEnqueueOrAbort(SingularityTask task) { try { healthchecker.enqueueHealthcheck(task); } catch (Throwable t) { LOG.error( "Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t); exceptionNotifier.notify(t); abort.abort(AbortReason.UNRECOVERABLE_ERROR); } }
@Timed public void checkStateAfterFinishedTask( SingularityTaskId taskId, String slaveId, SingularitySchedulerStateCache stateCache) { Optional<SingularitySlave> slave = slaveManager.getObject(slaveId); if (!slave.isPresent()) { final String message = String.format("Couldn't find slave with id %s for task %s", slaveId, taskId); LOG.warn(message); exceptionNotifier.notify( message, ImmutableMap.of("slaveId", slaveId, "taskId", taskId.toString())); return; } if (slave.get().getCurrentState().getState() == MachineState.DECOMMISSIONING) { if (!hasTaskLeftOnSlave(taskId, slaveId, stateCache)) { slaveManager.changeState( slave.get(), MachineState.DECOMMISSIONED, slave.get().getCurrentState().getUser()); } } Optional<SingularityRack> rack = rackManager.getObject(slave.get().getRackId()); if (!rack.isPresent()) { final String message = String.format( "Couldn't find rack with id %s for task %s", slave.get().getRackId(), taskId); LOG.warn(message); exceptionNotifier.notify( message, ImmutableMap.of("rackId", slave.get().getRackId(), "taskId", taskId.toString())); return; } if (rack.get().getCurrentState().getState() == MachineState.DECOMMISSIONING) { if (!hasTaskLeftOnRack(taskId, stateCache)) { rackManager.changeState( rack.get(), MachineState.DECOMMISSIONED, rack.get().getCurrentState().getUser()); } } }