private List<SingularityRequestParent> getRequestsWithDeployState(
      Iterable<SingularityRequestWithState> requests) {
    List<String> requestIds = Lists.newArrayList();
    for (SingularityRequestWithState requestWithState : requests) {
      requestIds.add(requestWithState.getRequest().getId());
    }

    List<SingularityRequestParent> parents = Lists.newArrayListWithCapacity(requestIds.size());

    Map<String, SingularityRequestDeployState> deployStates =
        deployManager.getRequestDeployStatesByRequestIds(requestIds);

    for (SingularityRequestWithState requestWithState : requests) {
      Optional<SingularityRequestDeployState> deployState =
          Optional.fromNullable(deployStates.get(requestWithState.getRequest().getId()));
      parents.add(
          new SingularityRequestParent(
              requestWithState.getRequest(),
              requestWithState.getState(),
              deployState,
              Optional.<SingularityDeploy>absent(),
              Optional.<SingularityDeploy>absent(),
              Optional.<SingularityPendingDeploy>absent()));
    }

    return parents;
  }
  @POST
  @Path("/request/{requestId}/exit-cooldown")
  public SingularityRequestParent exitCooldown(
      @PathParam("requestId") String requestId, @QueryParam("user") Optional<String> user) {
    final SingularityRequestWithState requestWithState = fetchRequestWithState(requestId);

    checkConflict(
        requestWithState.getState() == RequestState.SYSTEM_COOLDOWN,
        "Request %s is not in SYSTEM_COOLDOWN state, it is in %s",
        requestId,
        requestWithState.getState());

    final Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);

    final long now = System.currentTimeMillis();

    requestManager.exitCooldown(requestWithState.getRequest(), now, user);

    if (maybeDeployId.isPresent() && !requestWithState.getRequest().isOneOff()) {
      requestManager.addToPendingQueue(
          new SingularityPendingRequest(
              requestId,
              maybeDeployId.get(),
              now,
              user,
              PendingType.IMMEDIATE,
              Collections.<String>emptyList()));
    }

    return fillEntireRequest(requestWithState);
  }
  private SingularityDeployStatistics getDeployStatistics(String requestId, String deployId) {
    final Optional<SingularityDeployStatistics> maybeDeployStatistics =
        deployManager.getDeployStatistics(requestId, deployId);

    if (maybeDeployStatistics.isPresent()) {
      return maybeDeployStatistics.get();
    }

    return new SingularityDeployStatisticsBuilder(requestId, deployId).build();
  }
  private String getAndCheckDeployId(String requestId) {
    Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);

    checkConflict(
        maybeDeployId.isPresent(),
        "Can not schedule/bounce a request (%s) with no deploy",
        requestId);

    return maybeDeployId.get();
  }
  private void enqueueHealthAndNewTaskChecks() {
    final long start = System.currentTimeMillis();

    final List<SingularityTask> activeTasks = taskManager.getActiveTasks();
    final Map<SingularityTaskId, SingularityTask> activeTaskMap =
        Maps.uniqueIndex(activeTasks, SingularityTaskIdHolder.getTaskIdFunction());

    final Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates =
        taskManager.getTaskHistoryUpdates(activeTaskMap.keySet());

    final Map<SingularityDeployKey, SingularityPendingDeploy> pendingDeploys =
        Maps.uniqueIndex(
            deployManager.getPendingDeploys(), SingularityDeployKey.FROM_PENDING_TO_DEPLOY_KEY);
    final Map<String, SingularityRequestWithState> idToRequest =
        Maps.uniqueIndex(
            requestManager.getRequests(), SingularityRequestWithState.REQUEST_STATE_TO_REQUEST_ID);

    requestManager.getActiveRequests();
    int enqueuedNewTaskChecks = 0;
    int enqueuedHealthchecks = 0;

    for (Map.Entry<SingularityTaskId, SingularityTask> entry : activeTaskMap.entrySet()) {
      SingularityTaskId taskId = entry.getKey();
      SingularityTask task = entry.getValue();
      SimplifiedTaskState simplifiedTaskState =
          SingularityTaskHistoryUpdate.getCurrentState(taskUpdates.get(taskId));

      if (simplifiedTaskState != SimplifiedTaskState.DONE) {
        SingularityDeployKey deployKey =
            new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId());
        Optional<SingularityPendingDeploy> pendingDeploy =
            Optional.fromNullable(pendingDeploys.get(deployKey));
        Optional<SingularityRequestWithState> request =
            Optional.fromNullable(idToRequest.get(taskId.getRequestId()));

        if (!pendingDeploy.isPresent()) {
          newTaskChecker.enqueueNewTaskCheck(task, request, healthchecker);
          enqueuedNewTaskChecks++;
        }
        if (simplifiedTaskState == SimplifiedTaskState.RUNNING) {
          if (healthchecker.enqueueHealthcheck(task, pendingDeploy, request)) {
            enqueuedHealthchecks++;
          }
        }
      }
    }

    LOG.info(
        "Enqueued {} health checks and {} new task checks (out of {} active tasks) in {}",
        enqueuedHealthchecks,
        enqueuedNewTaskChecks,
        activeTasks.size(),
        JavaUtils.duration(start));
  }
  private boolean shouldScheduleTasks(
      SingularityPendingRequest pendingRequest,
      Optional<SingularityRequestWithState> maybeRequest) {
    if (!isRequestActive(maybeRequest)) {
      return false;
    }

    Optional<SingularityRequestDeployState> maybeRequestDeployState =
        deployManager.getRequestDeployState(pendingRequest.getRequestId());

    return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false);
  }
  private SingularityRequestDeployHolder getDeployHolder(String requestId) {
    Optional<SingularityRequestDeployState> requestDeployState =
        deployManager.getRequestDeployState(requestId);

    Optional<SingularityDeploy> activeDeploy = Optional.absent();
    Optional<SingularityDeploy> pendingDeploy = Optional.absent();

    if (requestDeployState.isPresent()) {
      if (requestDeployState.get().getActiveDeploy().isPresent()) {
        activeDeploy =
            deployManager.getDeploy(
                requestId, requestDeployState.get().getActiveDeploy().get().getDeployId());
      }
      if (requestDeployState.get().getPendingDeploy().isPresent()) {
        pendingDeploy =
            deployManager.getDeploy(
                requestId, requestDeployState.get().getPendingDeploy().get().getDeployId());
      }
    }

    return new SingularityRequestDeployHolder(activeDeploy, pendingDeploy);
  }
  private void checkActiveRequest(
      SingularityRequestWithState requestWithState,
      Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId,
      final long timestamp) {
    final SingularityRequest request = requestWithState.getRequest();

    if (request.getRequestType() == RequestType.ON_DEMAND
        || request.getRequestType() == RequestType.RUN_ONCE) {
      return; // There's no situation where we'd want to schedule an On Demand or Run Once request
              // at startup, so don't even bother with them.
    }

    Optional<SingularityRequestDeployState> requestDeployState =
        deployManager.getRequestDeployState(request.getId());

    if (!requestDeployState.isPresent()
        || !requestDeployState.get().getActiveDeploy().isPresent()) {
      LOG.debug("No active deploy for {} - not scheduling on startup", request.getId());
      return;
    }

    final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId();

    if (request.isScheduled()) {
      SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId);
      SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey);

      if (pendingTaskId != null
          && pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) {
        LOG.info(
            "Not rescheduling {} because {} is newer than {}",
            request.getId(),
            pendingTaskId,
            requestWithState.getTimestamp());
        return;
      }
    }

    requestManager.addToPendingQueue(
        new SingularityPendingRequest(
            request.getId(),
            activeDeployId,
            timestamp,
            Optional.<String>absent(),
            PendingType.STARTUP,
            Optional.<Boolean>absent(),
            Optional.<String>absent()));
  }
  private void checkReschedule(
      SingularityRequest newRequest, Optional<SingularityRequest> maybeOldRequest, long timestamp) {
    if (!maybeOldRequest.isPresent()) {
      return;
    }

    if (shouldReschedule(newRequest, maybeOldRequest.get())) {
      Optional<String> maybeDeployId = deployManager.getInUseDeployId(newRequest.getId());

      if (maybeDeployId.isPresent()) {
        requestManager.addToPendingQueue(
            new SingularityPendingRequest(
                newRequest.getId(), maybeDeployId.get(), timestamp, PendingType.UPDATED_REQUEST));
      }
    }
  }
  @POST
  @Path("/request/{requestId}/unpause")
  @ApiOperation(
      value = "Unpause a Singularity Request, scheduling new tasks immediately",
      response = SingularityRequestParent.class)
  @ApiResponses({
    @ApiResponse(code = 409, message = "Request is not paused"),
  })
  public SingularityRequestParent unpause(
      @ApiParam("The request ID to unpause") @PathParam("requestId") String requestId,
      @ApiParam("Username of the person requesting the unpause") @QueryParam("user")
          Optional<String> user) {
    SingularityRequestWithState requestWithState = fetchRequestWithState(requestId);

    checkConflict(
        requestWithState.getState() == RequestState.PAUSED,
        "Request %s is not in PAUSED state, it is in %s",
        requestId,
        requestWithState.getState());

    mailer.sendRequestUnpausedMail(requestWithState.getRequest(), user);

    Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);

    final long now = System.currentTimeMillis();

    requestManager.unpause(requestWithState.getRequest(), now, user);

    if (maybeDeployId.isPresent() && !requestWithState.getRequest().isOneOff()) {
      requestManager.addToPendingQueue(
          new SingularityPendingRequest(
              requestId,
              maybeDeployId.get(),
              now,
              user,
              PendingType.UNPAUSED,
              Collections.<String>emptyList()));
    }

    return fillEntireRequest(
        new SingularityRequestWithState(requestWithState.getRequest(), RequestState.ACTIVE, now));
  }
  private List<SingularityTaskRequest> checkForStaleScheduledTasks(
      List<SingularityPendingTask> pendingTasks, List<SingularityTaskRequest> taskRequests) {
    final Set<String> foundPendingTaskId = Sets.newHashSetWithExpectedSize(taskRequests.size());
    final Set<String> requestIds = Sets.newHashSetWithExpectedSize(taskRequests.size());

    for (SingularityTaskRequest taskRequest : taskRequests) {
      foundPendingTaskId.add(taskRequest.getPendingTask().getPendingTaskId().getId());
      requestIds.add(taskRequest.getRequest().getId());
    }

    for (SingularityPendingTask pendingTask : pendingTasks) {
      if (!foundPendingTaskId.contains(pendingTask.getPendingTaskId().getId())) {
        LOG.info("Removing stale pending task {}", pendingTask.getPendingTaskId());
        taskManager.deletePendingTask(pendingTask.getPendingTaskId());
      }
    }

    // TODO this check isn't necessary if we keep track better during deploys
    final Map<String, SingularityRequestDeployState> deployStates =
        deployManager.getRequestDeployStatesByRequestIds(requestIds);
    final List<SingularityTaskRequest> taskRequestsWithValidDeploys =
        Lists.newArrayListWithCapacity(taskRequests.size());

    for (SingularityTaskRequest taskRequest : taskRequests) {
      SingularityRequestDeployState requestDeployState =
          deployStates.get(taskRequest.getRequest().getId());

      if (!matchesDeploy(requestDeployState, taskRequest)) {
        LOG.info(
            "Removing stale pending task {} because the deployId did not match active/pending deploys {}",
            taskRequest.getPendingTask().getPendingTaskId(),
            requestDeployState);
        taskManager.deletePendingTask(taskRequest.getPendingTask().getPendingTaskId());
      } else {
        taskRequestsWithValidDeploys.add(taskRequest);
      }
    }

    return taskRequestsWithValidDeploys;
  }
  private void updateDeployStatistics(
      SingularityDeployStatistics deployStatistics,
      SingularityTaskId taskId,
      long timestamp,
      ExtendedTaskState state,
      Optional<PendingType> scheduleResult) {
    SingularityDeployStatisticsBuilder bldr = deployStatistics.toBuilder();

    if (bldr.getAverageRuntimeMillis().isPresent()) {
      long newAvgRuntimeMillis =
          (bldr.getAverageRuntimeMillis().get() * bldr.getNumTasks()
                  + (timestamp - taskId.getStartedAt()))
              / (bldr.getNumTasks() + 1);

      bldr.setAverageRuntimeMillis(Optional.of(newAvgRuntimeMillis));
    } else {
      bldr.setAverageRuntimeMillis(Optional.of(timestamp - taskId.getStartedAt()));
    }

    bldr.setNumTasks(bldr.getNumTasks() + 1);

    if (!bldr.getLastFinishAt().isPresent() || timestamp > bldr.getLastFinishAt().get()) {
      bldr.setLastFinishAt(Optional.of(timestamp));
      bldr.setLastTaskState(Optional.of(state));
    }

    final ListMultimap<Integer, Long> instanceSequentialFailureTimestamps =
        bldr.getInstanceSequentialFailureTimestamps();
    final List<Long> sequentialFailureTimestamps =
        instanceSequentialFailureTimestamps.get(taskId.getInstanceNo());

    if (!state.isSuccess()) {
      if (SingularityTaskHistoryUpdate.getUpdate(
              taskManager.getTaskHistoryUpdates(taskId), ExtendedTaskState.TASK_CLEANING)
          .isPresent()) {
        LOG.debug("{} failed with {} after cleaning - ignoring it for cooldown", taskId, state);
      } else {

        if (sequentialFailureTimestamps.size() < configuration.getCooldownAfterFailures()) {
          sequentialFailureTimestamps.add(timestamp);
        } else if (timestamp > sequentialFailureTimestamps.get(0)) {
          sequentialFailureTimestamps.set(0, timestamp);
        }

        Collections.sort(sequentialFailureTimestamps);
      }
    } else {
      bldr.setNumSuccess(bldr.getNumSuccess() + 1);
      sequentialFailureTimestamps.clear();
    }

    if (scheduleResult.isPresent() && scheduleResult.get() == PendingType.RETRY) {
      bldr.setNumSequentialRetries(bldr.getNumSequentialRetries() + 1);
    } else {
      bldr.setNumSequentialRetries(0);
    }

    final SingularityDeployStatistics newStatistics = bldr.build();

    LOG.trace("Saving new deploy statistics {}", newStatistics);

    deployManager.saveDeployStatistics(newStatistics);
  }
  private Optional<PendingType> handleCompletedTaskWithStatistics(
      Optional<SingularityTask> task,
      SingularityTaskId taskId,
      long timestamp,
      ExtendedTaskState state,
      SingularityDeployStatistics deployStatistics,
      SingularityCreateResult taskHistoryUpdateCreateResult,
      SingularitySchedulerStateCache stateCache) {
    final Optional<SingularityRequestWithState> maybeRequestWithState =
        requestManager.getRequest(taskId.getRequestId());

    if (!isRequestActive(maybeRequestWithState)) {
      LOG.warn(
          "Not scheduling a new task, {} is {}",
          taskId.getRequestId(),
          SingularityRequestWithState.getRequestState(maybeRequestWithState));
      return Optional.absent();
    }

    RequestState requestState = maybeRequestWithState.get().getState();
    final SingularityRequest request = maybeRequestWithState.get().getRequest();

    final Optional<SingularityRequestDeployState> requestDeployState =
        deployManager.getRequestDeployState(request.getId());

    if (!isDeployInUse(requestDeployState, taskId.getDeployId(), true)) {
      LOG.debug(
          "Task {} completed, but it didn't match active deploy state {} - ignoring",
          taskId.getId(),
          requestDeployState);
      return Optional.absent();
    }

    if (taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED
        && requestState != RequestState.SYSTEM_COOLDOWN) {
      mailer.sendTaskCompletedMail(task, taskId, request, state);
    } else if (requestState == RequestState.SYSTEM_COOLDOWN) {
      LOG.debug("Not sending a task completed email because task {} is in SYSTEM_COOLDOWN", taskId);
    } else {
      LOG.debug(
          "Not sending a task completed email for task {} because Singularity already processed this update",
          taskId);
    }

    if (!state.isSuccess()
        && taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED
        && cooldown.shouldEnterCooldown(
            request, taskId, requestState, deployStatistics, timestamp)) {
      LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId);
      requestState = RequestState.SYSTEM_COOLDOWN;
      requestManager.cooldown(request, System.currentTimeMillis());
      mailer.sendRequestInCooldownMail(request);
    }

    PendingType pendingType = PendingType.TASK_DONE;

    if (!state.isSuccess() && shouldRetryImmediately(request, deployStatistics)) {
      LOG.debug("Retrying {} because {}", request.getId(), state);
      pendingType = PendingType.RETRY;
    } else if (!request.isAlwaysRunning()) {
      return Optional.absent();
    }

    if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
      // TODO send not cooldown anymore email
      LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
      requestState = RequestState.ACTIVE;
      requestManager.exitCooldown(request, System.currentTimeMillis());
    }

    SingularityPendingRequest pendingRequest =
        new SingularityPendingRequest(
            request.getId(),
            requestDeployState.get().getActiveDeploy().get().getDeployId(),
            System.currentTimeMillis(),
            pendingType);

    scheduleTasks(
        stateCache,
        request,
        requestState,
        deployStatistics,
        pendingRequest,
        getMatchingTaskIds(stateCache, request, pendingRequest));

    return Optional.of(pendingType);
  }
  public void checkForDecomissions(SingularitySchedulerStateCache stateCache) {
    final long start = System.currentTimeMillis();

    final Set<String> requestIdsToReschedule = Sets.newHashSet();
    final Set<SingularityTaskId> matchingTaskIds = Sets.newHashSet();

    final Collection<SingularityTaskId> activeTaskIds = stateCache.getActiveTaskIds();

    final Map<SingularitySlave, MachineState> slaves =
        getDefaultMap(slaveManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));

    for (SingularitySlave slave : slaves.keySet()) {
      boolean foundTask = false;

      for (SingularityTask activeTask : taskManager.getTasksOnSlave(activeTaskIds, slave)) {
        cleanupTaskDueToDecomission(requestIdsToReschedule, matchingTaskIds, activeTask, slave);
        foundTask = true;
      }

      if (!foundTask) {
        slaves.put(slave, MachineState.DECOMMISSIONED);
      }
    }

    final Map<SingularityRack, MachineState> racks =
        getDefaultMap(rackManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));

    for (SingularityRack rack : racks.keySet()) {
      boolean foundTask = false;

      for (SingularityTaskId activeTaskId : activeTaskIds) {
        if (rack.getId().equals(activeTaskId.getRackId())) {
          foundTask = true;
        }

        if (matchingTaskIds.contains(activeTaskId)) {
          continue;
        }

        if (rack.getId().equals(activeTaskId.getRackId())) {
          Optional<SingularityTask> maybeTask = taskManager.getTask(activeTaskId);
          cleanupTaskDueToDecomission(
              requestIdsToReschedule, matchingTaskIds, maybeTask.get(), rack);
        }
      }

      if (!foundTask) {
        racks.put(rack, MachineState.DECOMMISSIONED);
      }
    }

    for (String requestId : requestIdsToReschedule) {
      LOG.trace("Rescheduling request {} due to decomissions", requestId);

      Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);

      if (maybeDeployId.isPresent()) {
        requestManager.addToPendingQueue(
            new SingularityPendingRequest(
                requestId, maybeDeployId.get(), start, PendingType.DECOMISSIONED_SLAVE_OR_RACK));
      } else {
        LOG.warn("Not rescheduling a request ({}) because of no active deploy", requestId);
      }
    }

    changeState(slaves, slaveManager);
    changeState(racks, rackManager);

    if (slaves.isEmpty()
        && racks.isEmpty()
        && requestIdsToReschedule.isEmpty()
        && matchingTaskIds.isEmpty()) {
      LOG.trace("Decomission check found nothing");
    } else {
      LOG.info(
          "Found {} decomissioning slaves, {} decomissioning racks, rescheduling {} requests and scheduling {} tasks for cleanup in {}",
          slaves.size(),
          racks.size(),
          requestIdsToReschedule.size(),
          matchingTaskIds.size(),
          JavaUtils.duration(start));
    }
  }