private boolean shouldScheduleTasks(
      SingularityPendingRequest pendingRequest,
      Optional<SingularityRequestWithState> maybeRequest) {
    if (!isRequestActive(maybeRequest)) {
      return false;
    }

    Optional<SingularityRequestDeployState> maybeRequestDeployState =
        deployManager.getRequestDeployState(pendingRequest.getRequestId());

    return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false);
  }
Пример #2
0
  private void checkActiveRequest(
      SingularityRequestWithState requestWithState,
      Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId,
      final long timestamp) {
    final SingularityRequest request = requestWithState.getRequest();

    if (request.getRequestType() == RequestType.ON_DEMAND
        || request.getRequestType() == RequestType.RUN_ONCE) {
      return; // There's no situation where we'd want to schedule an On Demand or Run Once request
              // at startup, so don't even bother with them.
    }

    Optional<SingularityRequestDeployState> requestDeployState =
        deployManager.getRequestDeployState(request.getId());

    if (!requestDeployState.isPresent()
        || !requestDeployState.get().getActiveDeploy().isPresent()) {
      LOG.debug("No active deploy for {} - not scheduling on startup", request.getId());
      return;
    }

    final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId();

    if (request.isScheduled()) {
      SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId);
      SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey);

      if (pendingTaskId != null
          && pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) {
        LOG.info(
            "Not rescheduling {} because {} is newer than {}",
            request.getId(),
            pendingTaskId,
            requestWithState.getTimestamp());
        return;
      }
    }

    requestManager.addToPendingQueue(
        new SingularityPendingRequest(
            request.getId(),
            activeDeployId,
            timestamp,
            Optional.<String>absent(),
            PendingType.STARTUP,
            Optional.<Boolean>absent(),
            Optional.<String>absent()));
  }
Пример #3
0
  private SingularityRequestDeployHolder getDeployHolder(String requestId) {
    Optional<SingularityRequestDeployState> requestDeployState =
        deployManager.getRequestDeployState(requestId);

    Optional<SingularityDeploy> activeDeploy = Optional.absent();
    Optional<SingularityDeploy> pendingDeploy = Optional.absent();

    if (requestDeployState.isPresent()) {
      if (requestDeployState.get().getActiveDeploy().isPresent()) {
        activeDeploy =
            deployManager.getDeploy(
                requestId, requestDeployState.get().getActiveDeploy().get().getDeployId());
      }
      if (requestDeployState.get().getPendingDeploy().isPresent()) {
        pendingDeploy =
            deployManager.getDeploy(
                requestId, requestDeployState.get().getPendingDeploy().get().getDeployId());
      }
    }

    return new SingularityRequestDeployHolder(activeDeploy, pendingDeploy);
  }
  private Optional<PendingType> handleCompletedTaskWithStatistics(
      Optional<SingularityTask> task,
      SingularityTaskId taskId,
      long timestamp,
      ExtendedTaskState state,
      SingularityDeployStatistics deployStatistics,
      SingularityCreateResult taskHistoryUpdateCreateResult,
      SingularitySchedulerStateCache stateCache) {
    final Optional<SingularityRequestWithState> maybeRequestWithState =
        requestManager.getRequest(taskId.getRequestId());

    if (!isRequestActive(maybeRequestWithState)) {
      LOG.warn(
          "Not scheduling a new task, {} is {}",
          taskId.getRequestId(),
          SingularityRequestWithState.getRequestState(maybeRequestWithState));
      return Optional.absent();
    }

    RequestState requestState = maybeRequestWithState.get().getState();
    final SingularityRequest request = maybeRequestWithState.get().getRequest();

    final Optional<SingularityRequestDeployState> requestDeployState =
        deployManager.getRequestDeployState(request.getId());

    if (!isDeployInUse(requestDeployState, taskId.getDeployId(), true)) {
      LOG.debug(
          "Task {} completed, but it didn't match active deploy state {} - ignoring",
          taskId.getId(),
          requestDeployState);
      return Optional.absent();
    }

    if (taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED
        && requestState != RequestState.SYSTEM_COOLDOWN) {
      mailer.sendTaskCompletedMail(task, taskId, request, state);
    } else if (requestState == RequestState.SYSTEM_COOLDOWN) {
      LOG.debug("Not sending a task completed email because task {} is in SYSTEM_COOLDOWN", taskId);
    } else {
      LOG.debug(
          "Not sending a task completed email for task {} because Singularity already processed this update",
          taskId);
    }

    if (!state.isSuccess()
        && taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED
        && cooldown.shouldEnterCooldown(
            request, taskId, requestState, deployStatistics, timestamp)) {
      LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId);
      requestState = RequestState.SYSTEM_COOLDOWN;
      requestManager.cooldown(request, System.currentTimeMillis());
      mailer.sendRequestInCooldownMail(request);
    }

    PendingType pendingType = PendingType.TASK_DONE;

    if (!state.isSuccess() && shouldRetryImmediately(request, deployStatistics)) {
      LOG.debug("Retrying {} because {}", request.getId(), state);
      pendingType = PendingType.RETRY;
    } else if (!request.isAlwaysRunning()) {
      return Optional.absent();
    }

    if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
      // TODO send not cooldown anymore email
      LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
      requestState = RequestState.ACTIVE;
      requestManager.exitCooldown(request, System.currentTimeMillis());
    }

    SingularityPendingRequest pendingRequest =
        new SingularityPendingRequest(
            request.getId(),
            requestDeployState.get().getActiveDeploy().get().getDeployId(),
            System.currentTimeMillis(),
            pendingType);

    scheduleTasks(
        stateCache,
        request,
        requestState,
        deployStatistics,
        pendingRequest,
        getMatchingTaskIds(stateCache, request, pendingRequest));

    return Optional.of(pendingType);
  }