private boolean shouldScheduleTasks(
SingularityPendingRequest pendingRequest,
Optional<SingularityRequestWithState> maybeRequest) {
if (!isRequestActive(maybeRequest)) {
return false;
}
Optional<SingularityRequestDeployState> maybeRequestDeployState =
deployManager.getRequestDeployState(pendingRequest.getRequestId());
return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false);
}
private List<SingularityPendingTask> getScheduledTaskIds(
int numMissingInstances,
List<SingularityTaskId> matchingTaskIds,
SingularityRequest request,
RequestState state,
SingularityDeployStatistics deployStatistics,
String deployId,
SingularityPendingRequest pendingRequest) {
final Optional<Long> nextRunAt =
getNextRunAt(request, state, deployStatistics, pendingRequest.getPendingType());
if (!nextRunAt.isPresent()) {
return Collections.emptyList();
}
final Set<Integer> inuseInstanceNumbers =
Sets.newHashSetWithExpectedSize(matchingTaskIds.size());
for (SingularityTaskId matchingTaskId : matchingTaskIds) {
inuseInstanceNumbers.add(matchingTaskId.getInstanceNo());
}
final List<SingularityPendingTask> newTasks =
Lists.newArrayListWithCapacity(numMissingInstances);
int nextInstanceNumber = 1;
for (int i = 0; i < numMissingInstances; i++) {
while (inuseInstanceNumbers.contains(nextInstanceNumber)) {
nextInstanceNumber++;
}
newTasks.add(
new SingularityPendingTask(
new SingularityPendingTaskId(
request.getId(),
deployId,
nextRunAt.get(),
nextInstanceNumber,
pendingRequest.getPendingType(),
pendingRequest.getTimestamp()),
pendingRequest.getCmdLineArgsList(),
pendingRequest.getUser()));
nextInstanceNumber++;
}
return newTasks;
}
private void deleteScheduledTasks(
final Collection<SingularityPendingTask> scheduledTasks,
SingularityPendingRequest pendingRequest) {
for (SingularityPendingTask task :
Iterables.filter(
scheduledTasks,
Predicates.and(
SingularityPendingTask.matchingRequest(pendingRequest.getRequestId()),
SingularityPendingTask.matchingDeploy(pendingRequest.getDeployId())))) {
LOG.debug(
"Deleting pending task {} in order to reschedule {}",
task.getPendingTaskId().getId(),
pendingRequest);
taskManager.deletePendingTask(task.getPendingTaskId());
}
}
private int getNumMissingInstances(
List<SingularityTaskId> matchingTaskIds,
SingularityRequest request,
SingularityPendingRequest pendingRequest) {
if (request.isOneOff() && pendingRequest.getPendingType() == PendingType.ONEOFF) {
return 1;
}
final int numInstances = request.getInstancesSafe();
return numInstances - matchingTaskIds.size();
}
private List<SingularityTaskId> getMatchingTaskIds(
SingularitySchedulerStateCache stateCache,
SingularityRequest request,
SingularityPendingRequest pendingRequest) {
if (request.isLongRunning()) {
return SingularityTaskId.matchingAndNotIn(
stateCache.getActiveTaskIds(),
request.getId(),
pendingRequest.getDeployId(),
stateCache.getCleaningTasks());
} else {
return Lists.newArrayList(
Iterables.filter(
stateCache.getActiveTaskIds(), SingularityTaskId.matchingRequest(request.getId())));
}
}
private int scheduleTasks(
SingularitySchedulerStateCache stateCache,
SingularityRequest request,
RequestState state,
SingularityDeployStatistics deployStatistics,
SingularityPendingRequest pendingRequest,
List<SingularityTaskId> matchingTaskIds) {
deleteScheduledTasks(stateCache.getScheduledTasks(), pendingRequest);
final int numMissingInstances =
getNumMissingInstances(matchingTaskIds, request, pendingRequest);
LOG.debug(
"Missing {} instances of request {} (matching tasks: {}), pending request: {}",
numMissingInstances,
request.getId(),
matchingTaskIds,
pendingRequest);
if (numMissingInstances > 0) {
final List<SingularityPendingTask> scheduledTasks =
getScheduledTaskIds(
numMissingInstances,
matchingTaskIds,
request,
state,
deployStatistics,
pendingRequest.getDeployId(),
pendingRequest);
if (!scheduledTasks.isEmpty()) {
LOG.trace("Scheduling tasks: {}", scheduledTasks);
for (SingularityPendingTask scheduledTask : scheduledTasks) {
taskManager.savePendingTask(scheduledTask);
}
} else {
LOG.info(
"No new scheduled tasks found for {}, setting state to {}",
request.getId(),
RequestState.FINISHED);
requestManager.finish(request, System.currentTimeMillis());
}
} else if (numMissingInstances < 0) {
final long now = System.currentTimeMillis();
Collections.sort(
matchingTaskIds,
Collections.reverseOrder(
SingularityTaskId.INSTANCE_NO_COMPARATOR)); // clean the highest numbers
for (int i = 0; i < Math.abs(numMissingInstances); i++) {
final SingularityTaskId toCleanup = matchingTaskIds.get(i);
LOG.info(
"Cleaning up task {} due to new request {} - scaling down to {} instances",
toCleanup.getId(),
request.getId(),
request.getInstancesSafe());
taskManager.createTaskCleanup(
new SingularityTaskCleanup(
pendingRequest.getUser(),
TaskCleanupType.SCALING_DOWN,
now,
toCleanup,
Optional.<String>absent()));
}
}
return numMissingInstances;
}
public void drainPendingQueue(final SingularitySchedulerStateCache stateCache) {
final long start = System.currentTimeMillis();
final ImmutableList<SingularityPendingRequest> pendingRequests =
ImmutableList.copyOf(requestManager.getPendingRequests());
if (pendingRequests.isEmpty()) {
LOG.trace("Pending queue was empty");
return;
}
LOG.info("Pending queue had {} requests", pendingRequests.size());
int totalNewScheduledTasks = 0;
int heldForScheduledActiveTask = 0;
int obsoleteRequests = 0;
for (SingularityPendingRequest pendingRequest : pendingRequests) {
Optional<SingularityRequestWithState> maybeRequest =
requestManager.getRequest(pendingRequest.getRequestId());
if (shouldScheduleTasks(pendingRequest, maybeRequest)) {
final List<SingularityTaskId> matchingTaskIds =
getMatchingTaskIds(stateCache, maybeRequest.get().getRequest(), pendingRequest);
final SingularityDeployStatistics deployStatistics =
getDeployStatistics(pendingRequest.getRequestId(), pendingRequest.getDeployId());
final RequestState requestState = checkCooldown(maybeRequest.get(), deployStatistics);
int numScheduledTasks =
scheduleTasks(
stateCache,
maybeRequest.get().getRequest(),
requestState,
deployStatistics,
pendingRequest,
matchingTaskIds);
if (numScheduledTasks == 0
&& !matchingTaskIds.isEmpty()
&& maybeRequest.get().getRequest().isScheduled()
&& pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) {
LOG.trace(
"Holding pending request {} because it is scheduled and has an active task",
pendingRequest);
heldForScheduledActiveTask++;
continue;
}
LOG.debug(
"Pending request {} resulted in {} new scheduled tasks",
pendingRequest,
numScheduledTasks);
totalNewScheduledTasks += numScheduledTasks;
} else {
LOG.debug(
"Pending request {} was obsolete (request {})",
pendingRequest,
SingularityRequestWithState.getRequestState(maybeRequest));
obsoleteRequests++;
}
requestManager.deletePendingRequest(pendingRequest);
}
LOG.info(
"Scheduled {} new tasks ({} obsolete requests, {} held) in {}",
totalNewScheduledTasks,
obsoleteRequests,
heldForScheduledActiveTask,
JavaUtils.duration(start));
}