private boolean shouldScheduleTasks(
SingularityPendingRequest pendingRequest,
Optional<SingularityRequestWithState> maybeRequest) {
if (!isRequestActive(maybeRequest)) {
return false;
}
Optional<SingularityRequestDeployState> maybeRequestDeployState =
deployManager.getRequestDeployState(pendingRequest.getRequestId());
return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false);
}
private void checkActiveRequest(
SingularityRequestWithState requestWithState,
Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId,
final long timestamp) {
final SingularityRequest request = requestWithState.getRequest();
if (request.getRequestType() == RequestType.ON_DEMAND
|| request.getRequestType() == RequestType.RUN_ONCE) {
return; // There's no situation where we'd want to schedule an On Demand or Run Once request
// at startup, so don't even bother with them.
}
Optional<SingularityRequestDeployState> requestDeployState =
deployManager.getRequestDeployState(request.getId());
if (!requestDeployState.isPresent()
|| !requestDeployState.get().getActiveDeploy().isPresent()) {
LOG.debug("No active deploy for {} - not scheduling on startup", request.getId());
return;
}
final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId();
if (request.isScheduled()) {
SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId);
SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey);
if (pendingTaskId != null
&& pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) {
LOG.info(
"Not rescheduling {} because {} is newer than {}",
request.getId(),
pendingTaskId,
requestWithState.getTimestamp());
return;
}
}
requestManager.addToPendingQueue(
new SingularityPendingRequest(
request.getId(),
activeDeployId,
timestamp,
Optional.<String>absent(),
PendingType.STARTUP,
Optional.<Boolean>absent(),
Optional.<String>absent()));
}
private SingularityRequestDeployHolder getDeployHolder(String requestId) {
Optional<SingularityRequestDeployState> requestDeployState =
deployManager.getRequestDeployState(requestId);
Optional<SingularityDeploy> activeDeploy = Optional.absent();
Optional<SingularityDeploy> pendingDeploy = Optional.absent();
if (requestDeployState.isPresent()) {
if (requestDeployState.get().getActiveDeploy().isPresent()) {
activeDeploy =
deployManager.getDeploy(
requestId, requestDeployState.get().getActiveDeploy().get().getDeployId());
}
if (requestDeployState.get().getPendingDeploy().isPresent()) {
pendingDeploy =
deployManager.getDeploy(
requestId, requestDeployState.get().getPendingDeploy().get().getDeployId());
}
}
return new SingularityRequestDeployHolder(activeDeploy, pendingDeploy);
}
private Optional<PendingType> handleCompletedTaskWithStatistics(
Optional<SingularityTask> task,
SingularityTaskId taskId,
long timestamp,
ExtendedTaskState state,
SingularityDeployStatistics deployStatistics,
SingularityCreateResult taskHistoryUpdateCreateResult,
SingularitySchedulerStateCache stateCache) {
final Optional<SingularityRequestWithState> maybeRequestWithState =
requestManager.getRequest(taskId.getRequestId());
if (!isRequestActive(maybeRequestWithState)) {
LOG.warn(
"Not scheduling a new task, {} is {}",
taskId.getRequestId(),
SingularityRequestWithState.getRequestState(maybeRequestWithState));
return Optional.absent();
}
RequestState requestState = maybeRequestWithState.get().getState();
final SingularityRequest request = maybeRequestWithState.get().getRequest();
final Optional<SingularityRequestDeployState> requestDeployState =
deployManager.getRequestDeployState(request.getId());
if (!isDeployInUse(requestDeployState, taskId.getDeployId(), true)) {
LOG.debug(
"Task {} completed, but it didn't match active deploy state {} - ignoring",
taskId.getId(),
requestDeployState);
return Optional.absent();
}
if (taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED
&& requestState != RequestState.SYSTEM_COOLDOWN) {
mailer.sendTaskCompletedMail(task, taskId, request, state);
} else if (requestState == RequestState.SYSTEM_COOLDOWN) {
LOG.debug("Not sending a task completed email because task {} is in SYSTEM_COOLDOWN", taskId);
} else {
LOG.debug(
"Not sending a task completed email for task {} because Singularity already processed this update",
taskId);
}
if (!state.isSuccess()
&& taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED
&& cooldown.shouldEnterCooldown(
request, taskId, requestState, deployStatistics, timestamp)) {
LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId);
requestState = RequestState.SYSTEM_COOLDOWN;
requestManager.cooldown(request, System.currentTimeMillis());
mailer.sendRequestInCooldownMail(request);
}
PendingType pendingType = PendingType.TASK_DONE;
if (!state.isSuccess() && shouldRetryImmediately(request, deployStatistics)) {
LOG.debug("Retrying {} because {}", request.getId(), state);
pendingType = PendingType.RETRY;
} else if (!request.isAlwaysRunning()) {
return Optional.absent();
}
if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
// TODO send not cooldown anymore email
LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
requestState = RequestState.ACTIVE;
requestManager.exitCooldown(request, System.currentTimeMillis());
}
SingularityPendingRequest pendingRequest =
new SingularityPendingRequest(
request.getId(),
requestDeployState.get().getActiveDeploy().get().getDeployId(),
System.currentTimeMillis(),
pendingType);
scheduleTasks(
stateCache,
request,
requestState,
deployStatistics,
pendingRequest,
getMatchingTaskIds(stateCache, request, pendingRequest));
return Optional.of(pendingType);
}
