private List<SingularityTaskId> getMatchingTaskIds( SingularitySchedulerStateCache stateCache, SingularityRequest request, SingularityPendingRequest pendingRequest) { if (request.isLongRunning()) { return SingularityTaskId.matchingAndNotIn( stateCache.getActiveTaskIds(), request.getId(), pendingRequest.getDeployId(), stateCache.getCleaningTasks()); } else { return Lists.newArrayList( Iterables.filter( stateCache.getActiveTaskIds(), SingularityTaskId.matchingRequest(request.getId()))); } }
public void handleCompletedTask( Optional<SingularityTask> task, SingularityTaskId taskId, boolean wasActive, long timestamp, ExtendedTaskState state, SingularityCreateResult taskHistoryUpdateCreateResult, SingularitySchedulerStateCache stateCache) { final SingularityDeployStatistics deployStatistics = getDeployStatistics(taskId.getRequestId(), taskId.getDeployId()); if (wasActive) { taskManager.deleteActiveTask(taskId.getId()); stateCache.getActiveTaskIds().remove(taskId); } if (!task.isPresent() || task.get().getTaskRequest().getRequest().isLoadBalanced()) { taskManager.createLBCleanupTask(taskId); } final Optional<PendingType> scheduleResult = handleCompletedTaskWithStatistics( task, taskId, timestamp, state, deployStatistics, taskHistoryUpdateCreateResult, stateCache); if (taskHistoryUpdateCreateResult == SingularityCreateResult.EXISTED) { return; } updateDeployStatistics(deployStatistics, taskId, timestamp, state, scheduleResult); }
private int scheduleTasks( SingularitySchedulerStateCache stateCache, SingularityRequest request, RequestState state, SingularityDeployStatistics deployStatistics, SingularityPendingRequest pendingRequest, List<SingularityTaskId> matchingTaskIds) { deleteScheduledTasks(stateCache.getScheduledTasks(), pendingRequest); final int numMissingInstances = getNumMissingInstances(matchingTaskIds, request, pendingRequest); LOG.debug( "Missing {} instances of request {} (matching tasks: {}), pending request: {}", numMissingInstances, request.getId(), matchingTaskIds, pendingRequest); if (numMissingInstances > 0) { final List<SingularityPendingTask> scheduledTasks = getScheduledTaskIds( numMissingInstances, matchingTaskIds, request, state, deployStatistics, pendingRequest.getDeployId(), pendingRequest); if (!scheduledTasks.isEmpty()) { LOG.trace("Scheduling tasks: {}", scheduledTasks); for (SingularityPendingTask scheduledTask : scheduledTasks) { taskManager.savePendingTask(scheduledTask); } } else { LOG.info( "No new scheduled tasks found for {}, setting state to {}", request.getId(), RequestState.FINISHED); requestManager.finish(request, System.currentTimeMillis()); } } else if (numMissingInstances < 0) { final long now = System.currentTimeMillis(); Collections.sort( matchingTaskIds, Collections.reverseOrder( SingularityTaskId.INSTANCE_NO_COMPARATOR)); // clean the highest numbers for (int i = 0; i < Math.abs(numMissingInstances); i++) { final SingularityTaskId toCleanup = matchingTaskIds.get(i); LOG.info( "Cleaning up task {} due to new request {} - scaling down to {} instances", toCleanup.getId(), request.getId(), request.getInstancesSafe()); taskManager.createTaskCleanup( new SingularityTaskCleanup( pendingRequest.getUser(), TaskCleanupType.SCALING_DOWN, now, toCleanup, Optional.<String>absent())); } } return numMissingInstances; }
public void checkForDecomissions(SingularitySchedulerStateCache stateCache) { final long start = System.currentTimeMillis(); final Set<String> requestIdsToReschedule = Sets.newHashSet(); final Set<SingularityTaskId> matchingTaskIds = Sets.newHashSet(); final Collection<SingularityTaskId> activeTaskIds = stateCache.getActiveTaskIds(); final Map<SingularitySlave, MachineState> slaves = getDefaultMap(slaveManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION)); for (SingularitySlave slave : slaves.keySet()) { boolean foundTask = false; for (SingularityTask activeTask : taskManager.getTasksOnSlave(activeTaskIds, slave)) { cleanupTaskDueToDecomission(requestIdsToReschedule, matchingTaskIds, activeTask, slave); foundTask = true; } if (!foundTask) { slaves.put(slave, MachineState.DECOMMISSIONED); } } final Map<SingularityRack, MachineState> racks = getDefaultMap(rackManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION)); for (SingularityRack rack : racks.keySet()) { boolean foundTask = false; for (SingularityTaskId activeTaskId : activeTaskIds) { if (rack.getId().equals(activeTaskId.getRackId())) { foundTask = true; } if (matchingTaskIds.contains(activeTaskId)) { continue; } if (rack.getId().equals(activeTaskId.getRackId())) { Optional<SingularityTask> maybeTask = taskManager.getTask(activeTaskId); cleanupTaskDueToDecomission( requestIdsToReschedule, matchingTaskIds, maybeTask.get(), rack); } } if (!foundTask) { racks.put(rack, MachineState.DECOMMISSIONED); } } for (String requestId : requestIdsToReschedule) { LOG.trace("Rescheduling request {} due to decomissions", requestId); Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId); if (maybeDeployId.isPresent()) { requestManager.addToPendingQueue( new SingularityPendingRequest( requestId, maybeDeployId.get(), start, PendingType.DECOMISSIONED_SLAVE_OR_RACK)); } else { LOG.warn("Not rescheduling a request ({}) because of no active deploy", requestId); } } changeState(slaves, slaveManager); changeState(racks, rackManager); if (slaves.isEmpty() && racks.isEmpty() && requestIdsToReschedule.isEmpty() && matchingTaskIds.isEmpty()) { LOG.trace("Decomission check found nothing"); } else { LOG.info( "Found {} decomissioning slaves, {} decomissioning racks, rescheduling {} requests and scheduling {} tasks for cleanup in {}", slaves.size(), racks.size(), requestIdsToReschedule.size(), matchingTaskIds.size(), JavaUtils.duration(start)); } }