private List<SingularityTaskId> getMatchingTaskIds(
SingularitySchedulerStateCache stateCache,
SingularityRequest request,
SingularityPendingRequest pendingRequest) {
if (request.isLongRunning()) {
return SingularityTaskId.matchingAndNotIn(
stateCache.getActiveTaskIds(),
request.getId(),
pendingRequest.getDeployId(),
stateCache.getCleaningTasks());
} else {
return Lists.newArrayList(
Iterables.filter(
stateCache.getActiveTaskIds(), SingularityTaskId.matchingRequest(request.getId())));
}
}
public void handleCompletedTask(
Optional<SingularityTask> task,
SingularityTaskId taskId,
boolean wasActive,
long timestamp,
ExtendedTaskState state,
SingularityCreateResult taskHistoryUpdateCreateResult,
SingularitySchedulerStateCache stateCache) {
final SingularityDeployStatistics deployStatistics =
getDeployStatistics(taskId.getRequestId(), taskId.getDeployId());
if (wasActive) {
taskManager.deleteActiveTask(taskId.getId());
stateCache.getActiveTaskIds().remove(taskId);
}
if (!task.isPresent() || task.get().getTaskRequest().getRequest().isLoadBalanced()) {
taskManager.createLBCleanupTask(taskId);
}
final Optional<PendingType> scheduleResult =
handleCompletedTaskWithStatistics(
task,
taskId,
timestamp,
state,
deployStatistics,
taskHistoryUpdateCreateResult,
stateCache);
if (taskHistoryUpdateCreateResult == SingularityCreateResult.EXISTED) {
return;
}
updateDeployStatistics(deployStatistics, taskId, timestamp, state, scheduleResult);
}
private int scheduleTasks(
SingularitySchedulerStateCache stateCache,
SingularityRequest request,
RequestState state,
SingularityDeployStatistics deployStatistics,
SingularityPendingRequest pendingRequest,
List<SingularityTaskId> matchingTaskIds) {
deleteScheduledTasks(stateCache.getScheduledTasks(), pendingRequest);
final int numMissingInstances =
getNumMissingInstances(matchingTaskIds, request, pendingRequest);
LOG.debug(
"Missing {} instances of request {} (matching tasks: {}), pending request: {}",
numMissingInstances,
request.getId(),
matchingTaskIds,
pendingRequest);
if (numMissingInstances > 0) {
final List<SingularityPendingTask> scheduledTasks =
getScheduledTaskIds(
numMissingInstances,
matchingTaskIds,
request,
state,
deployStatistics,
pendingRequest.getDeployId(),
pendingRequest);
if (!scheduledTasks.isEmpty()) {
LOG.trace("Scheduling tasks: {}", scheduledTasks);
for (SingularityPendingTask scheduledTask : scheduledTasks) {
taskManager.savePendingTask(scheduledTask);
}
} else {
LOG.info(
"No new scheduled tasks found for {}, setting state to {}",
request.getId(),
RequestState.FINISHED);
requestManager.finish(request, System.currentTimeMillis());
}
} else if (numMissingInstances < 0) {
final long now = System.currentTimeMillis();
Collections.sort(
matchingTaskIds,
Collections.reverseOrder(
SingularityTaskId.INSTANCE_NO_COMPARATOR)); // clean the highest numbers
for (int i = 0; i < Math.abs(numMissingInstances); i++) {
final SingularityTaskId toCleanup = matchingTaskIds.get(i);
LOG.info(
"Cleaning up task {} due to new request {} - scaling down to {} instances",
toCleanup.getId(),
request.getId(),
request.getInstancesSafe());
taskManager.createTaskCleanup(
new SingularityTaskCleanup(
pendingRequest.getUser(),
TaskCleanupType.SCALING_DOWN,
now,
toCleanup,
Optional.<String>absent()));
}
}
return numMissingInstances;
}
public void checkForDecomissions(SingularitySchedulerStateCache stateCache) {
final long start = System.currentTimeMillis();
final Set<String> requestIdsToReschedule = Sets.newHashSet();
final Set<SingularityTaskId> matchingTaskIds = Sets.newHashSet();
final Collection<SingularityTaskId> activeTaskIds = stateCache.getActiveTaskIds();
final Map<SingularitySlave, MachineState> slaves =
getDefaultMap(slaveManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));
for (SingularitySlave slave : slaves.keySet()) {
boolean foundTask = false;
for (SingularityTask activeTask : taskManager.getTasksOnSlave(activeTaskIds, slave)) {
cleanupTaskDueToDecomission(requestIdsToReschedule, matchingTaskIds, activeTask, slave);
foundTask = true;
}
if (!foundTask) {
slaves.put(slave, MachineState.DECOMMISSIONED);
}
}
final Map<SingularityRack, MachineState> racks =
getDefaultMap(rackManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));
for (SingularityRack rack : racks.keySet()) {
boolean foundTask = false;
for (SingularityTaskId activeTaskId : activeTaskIds) {
if (rack.getId().equals(activeTaskId.getRackId())) {
foundTask = true;
}
if (matchingTaskIds.contains(activeTaskId)) {
continue;
}
if (rack.getId().equals(activeTaskId.getRackId())) {
Optional<SingularityTask> maybeTask = taskManager.getTask(activeTaskId);
cleanupTaskDueToDecomission(
requestIdsToReschedule, matchingTaskIds, maybeTask.get(), rack);
}
}
if (!foundTask) {
racks.put(rack, MachineState.DECOMMISSIONED);
}
}
for (String requestId : requestIdsToReschedule) {
LOG.trace("Rescheduling request {} due to decomissions", requestId);
Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);
if (maybeDeployId.isPresent()) {
requestManager.addToPendingQueue(
new SingularityPendingRequest(
requestId, maybeDeployId.get(), start, PendingType.DECOMISSIONED_SLAVE_OR_RACK));
} else {
LOG.warn("Not rescheduling a request ({}) because of no active deploy", requestId);
}
}
changeState(slaves, slaveManager);
changeState(racks, rackManager);
if (slaves.isEmpty()
&& racks.isEmpty()
&& requestIdsToReschedule.isEmpty()
&& matchingTaskIds.isEmpty()) {
LOG.trace("Decomission check found nothing");
} else {
LOG.info(
"Found {} decomissioning slaves, {} decomissioning racks, rescheduling {} requests and scheduling {} tasks for cleanup in {}",
slaves.size(),
racks.size(),
requestIdsToReschedule.size(),
matchingTaskIds.size(),
JavaUtils.duration(start));
}
}
