@POST
@Path("/request/{requestId}/exit-cooldown")
public SingularityRequestParent exitCooldown(
@PathParam("requestId") String requestId, @QueryParam("user") Optional<String> user) {
final SingularityRequestWithState requestWithState = fetchRequestWithState(requestId);
checkConflict(
requestWithState.getState() == RequestState.SYSTEM_COOLDOWN,
"Request %s is not in SYSTEM_COOLDOWN state, it is in %s",
requestId,
requestWithState.getState());
final Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);
final long now = System.currentTimeMillis();
requestManager.exitCooldown(requestWithState.getRequest(), now, user);
if (maybeDeployId.isPresent() && !requestWithState.getRequest().isOneOff()) {
requestManager.addToPendingQueue(
new SingularityPendingRequest(
requestId,
maybeDeployId.get(),
now,
user,
PendingType.IMMEDIATE,
Collections.<String>emptyList()));
}
return fillEntireRequest(requestWithState);
}
private String getAndCheckDeployId(String requestId) {
Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);
checkConflict(
maybeDeployId.isPresent(),
"Can not schedule/bounce a request (%s) with no deploy",
requestId);
return maybeDeployId.get();
}
private void checkReschedule(
SingularityRequest newRequest, Optional<SingularityRequest> maybeOldRequest, long timestamp) {
if (!maybeOldRequest.isPresent()) {
return;
}
if (shouldReschedule(newRequest, maybeOldRequest.get())) {
Optional<String> maybeDeployId = deployManager.getInUseDeployId(newRequest.getId());
if (maybeDeployId.isPresent()) {
requestManager.addToPendingQueue(
new SingularityPendingRequest(
newRequest.getId(), maybeDeployId.get(), timestamp, PendingType.UPDATED_REQUEST));
}
}
}
@POST
@Path("/request/{requestId}/unpause")
@ApiOperation(
value = "Unpause a Singularity Request, scheduling new tasks immediately",
response = SingularityRequestParent.class)
@ApiResponses({
@ApiResponse(code = 409, message = "Request is not paused"),
})
public SingularityRequestParent unpause(
@ApiParam("The request ID to unpause") @PathParam("requestId") String requestId,
@ApiParam("Username of the person requesting the unpause") @QueryParam("user")
Optional<String> user) {
SingularityRequestWithState requestWithState = fetchRequestWithState(requestId);
checkConflict(
requestWithState.getState() == RequestState.PAUSED,
"Request %s is not in PAUSED state, it is in %s",
requestId,
requestWithState.getState());
mailer.sendRequestUnpausedMail(requestWithState.getRequest(), user);
Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);
final long now = System.currentTimeMillis();
requestManager.unpause(requestWithState.getRequest(), now, user);
if (maybeDeployId.isPresent() && !requestWithState.getRequest().isOneOff()) {
requestManager.addToPendingQueue(
new SingularityPendingRequest(
requestId,
maybeDeployId.get(),
now,
user,
PendingType.UNPAUSED,
Collections.<String>emptyList()));
}
return fillEntireRequest(
new SingularityRequestWithState(requestWithState.getRequest(), RequestState.ACTIVE, now));
}
public void checkForDecomissions(SingularitySchedulerStateCache stateCache) {
final long start = System.currentTimeMillis();
final Set<String> requestIdsToReschedule = Sets.newHashSet();
final Set<SingularityTaskId> matchingTaskIds = Sets.newHashSet();
final Collection<SingularityTaskId> activeTaskIds = stateCache.getActiveTaskIds();
final Map<SingularitySlave, MachineState> slaves =
getDefaultMap(slaveManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));
for (SingularitySlave slave : slaves.keySet()) {
boolean foundTask = false;
for (SingularityTask activeTask : taskManager.getTasksOnSlave(activeTaskIds, slave)) {
cleanupTaskDueToDecomission(requestIdsToReschedule, matchingTaskIds, activeTask, slave);
foundTask = true;
}
if (!foundTask) {
slaves.put(slave, MachineState.DECOMMISSIONED);
}
}
final Map<SingularityRack, MachineState> racks =
getDefaultMap(rackManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));
for (SingularityRack rack : racks.keySet()) {
boolean foundTask = false;
for (SingularityTaskId activeTaskId : activeTaskIds) {
if (rack.getId().equals(activeTaskId.getRackId())) {
foundTask = true;
}
if (matchingTaskIds.contains(activeTaskId)) {
continue;
}
if (rack.getId().equals(activeTaskId.getRackId())) {
Optional<SingularityTask> maybeTask = taskManager.getTask(activeTaskId);
cleanupTaskDueToDecomission(
requestIdsToReschedule, matchingTaskIds, maybeTask.get(), rack);
}
}
if (!foundTask) {
racks.put(rack, MachineState.DECOMMISSIONED);
}
}
for (String requestId : requestIdsToReschedule) {
LOG.trace("Rescheduling request {} due to decomissions", requestId);
Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);
if (maybeDeployId.isPresent()) {
requestManager.addToPendingQueue(
new SingularityPendingRequest(
requestId, maybeDeployId.get(), start, PendingType.DECOMISSIONED_SLAVE_OR_RACK));
} else {
LOG.warn("Not rescheduling a request ({}) because of no active deploy", requestId);
}
}
changeState(slaves, slaveManager);
changeState(racks, rackManager);
if (slaves.isEmpty()
&& racks.isEmpty()
&& requestIdsToReschedule.isEmpty()
&& matchingTaskIds.isEmpty()) {
LOG.trace("Decomission check found nothing");
} else {
LOG.info(
"Found {} decomissioning slaves, {} decomissioning racks, rescheduling {} requests and scheduling {} tasks for cleanup in {}",
slaves.size(),
racks.size(),
requestIdsToReschedule.size(),
matchingTaskIds.size(),
JavaUtils.duration(start));
}
}