Example #1
0
  @Override
  public ILock acquireLock(final ILockKey lockKey, final String user) throws LockException {
    return storage.write(
        storeProvider -> {
          LockStore.Mutable lockStore = storeProvider.getLockStore();
          Optional<ILock> existingLock = lockStore.fetchLock(lockKey);

          if (existingLock.isPresent()) {
            throw new LockException(
                String.format(
                    "Operation for: %s is already in progress. Started at: %s. Current owner: %s.",
                    formatLockKey(lockKey),
                    new Date(existingLock.get().getTimestampMs()).toString(),
                    existingLock.get().getUser()));
          }

          ILock lock =
              ILock.build(
                  new Lock()
                      .setKey(lockKey.newBuilder())
                      .setToken(tokenGenerator.createNew().toString())
                      .setTimestampMs(clock.nowMillis())
                      .setUser(user));

          lockStore.saveLock(lock);
          return lock;
        });
  }
 synchronized PulseState pulseAndGet(IJobUpdateKey key) {
   PulseState state = pulseStates.get(key);
   if (state != null) {
     state =
         pulseStates.put(
             key,
             new PulseState(state.getStatus(), state.getPulseTimeoutMs(), clock.nowMillis()));
   }
   return state;
 }
 boolean isBlocked(Clock clock) {
   return clock.nowMillis() - lastPulseMs >= pulseTimeoutMs;
 }
  private void evaluateUpdater(
      final MutableStoreProvider storeProvider,
      final UpdateFactory.Update update,
      IJobUpdateSummary summary,
      Map<Integer, Optional<IScheduledTask>> changedInstance)
      throws UpdateStateException {

    JobUpdateStatus updaterStatus = summary.getState().getStatus();
    final IJobUpdateKey key = summary.getKey();

    JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
    if (!updateStore.getLockToken(key).isPresent()) {
      recordAndChangeJobUpdateStatus(
          storeProvider, key, newEvent(ERROR).setMessage(LOST_LOCK_MESSAGE));
      return;
    }

    IJobUpdateInstructions instructions = updateStore.fetchJobUpdateInstructions(key).get();
    if (isCoordinatedAndPulseExpired(key, instructions)) {
      // Move coordinated update into awaiting pulse state.
      JobUpdateStatus blockedStatus = getBlockedState(summary.getState().getStatus());
      changeUpdateStatus(
          storeProvider, summary, newEvent(blockedStatus).setMessage(PULSE_TIMEOUT_MESSAGE));
      return;
    }

    InstanceStateProvider<Integer, Optional<IScheduledTask>> stateProvider =
        instanceId -> getActiveInstance(storeProvider.getTaskStore(), key.getJob(), instanceId);

    EvaluationResult<Integer> result = update.getUpdater().evaluate(changedInstance, stateProvider);

    LOG.info(key + " evaluation result: " + result);

    for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
      Iterable<InstanceUpdateStatus> statusChanges;

      int instanceId = entry.getKey();
      List<IJobInstanceUpdateEvent> savedEvents = updateStore.fetchInstanceEvents(key, instanceId);

      Set<JobUpdateAction> savedActions =
          FluentIterable.from(savedEvents).transform(EVENT_TO_ACTION).toSet();

      // Don't bother persisting a sequence of status changes that represents an instance that
      // was immediately recognized as being healthy and in the desired state.
      if (entry.getValue().getStatusChanges().equals(NOOP_INSTANCE_UPDATE)
          && savedEvents.isEmpty()) {

        LOG.info("Suppressing no-op update for instance " + instanceId);
        statusChanges = ImmutableSet.of();
      } else {
        statusChanges = entry.getValue().getStatusChanges();
      }

      for (InstanceUpdateStatus statusChange : statusChanges) {
        JobUpdateAction action = STATE_MAP.get(Pair.of(statusChange, updaterStatus));
        requireNonNull(action);

        // A given instance update action may only be issued once during the update lifecycle.
        // Suppress duplicate events due to pause/resume operations.
        if (savedActions.contains(action)) {
          LOG.info(
              String.format(
                  "Suppressing duplicate update %s for instance %s.", action, instanceId));
        } else {
          IJobInstanceUpdateEvent event =
              IJobInstanceUpdateEvent.build(
                  new JobInstanceUpdateEvent()
                      .setInstanceId(instanceId)
                      .setTimestampMs(clock.nowMillis())
                      .setAction(action));
          updateStore.saveJobInstanceUpdateEvent(summary.getKey(), event);
        }
      }
    }

    OneWayStatus status = result.getStatus();
    if (status == SUCCEEDED || status == OneWayStatus.FAILED) {
      if (SideEffect.hasActions(result.getSideEffects().values())) {
        throw new IllegalArgumentException(
            "A terminal state should not specify actions: " + result);
      }

      JobUpdateEvent event = new JobUpdateEvent();
      if (status == SUCCEEDED) {
        event.setStatus(update.getSuccessStatus());
      } else {
        event.setStatus(update.getFailureStatus());
        // Generate a transition message based on one (arbitrary) instance in the group that pushed
        // the update over the failure threshold (in all likelihood this group is of size 1).
        // This is done as a rough cut to aid in diagnosing a failed update, as generating a
        // complete summary would likely be of dubious value.
        for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
          Optional<Failure> failure = entry.getValue().getFailure();
          if (failure.isPresent()) {
            event.setMessage(failureMessage(entry.getKey(), failure.get()));
            break;
          }
        }
      }
      changeUpdateStatus(storeProvider, summary, event);
    } else {
      LOG.info("Executing side-effects for update of " + key + ": " + result.getSideEffects());
      for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
        IInstanceKey instance = InstanceKeys.from(key.getJob(), entry.getKey());

        Optional<InstanceAction> action = entry.getValue().getAction();
        if (action.isPresent()) {
          Optional<InstanceActionHandler> handler = action.get().getHandler();
          if (handler.isPresent()) {
            Optional<Amount<Long, Time>> reevaluateDelay =
                handler
                    .get()
                    .getReevaluationDelay(
                        instance, instructions, storeProvider, stateManager, updaterStatus);
            if (reevaluateDelay.isPresent()) {
              executor.schedule(
                  getDeferredEvaluator(instance, key),
                  reevaluateDelay.get().getValue(),
                  reevaluateDelay.get().getUnit().getTimeUnit());
            }
          }
        }
      }
    }
  }
  private void changeJobUpdateStatus(
      MutableStoreProvider storeProvider,
      IJobUpdateKey key,
      JobUpdateEvent proposedEvent,
      boolean recordChange)
      throws UpdateStateException {

    JobUpdateStatus status;
    boolean record;

    JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
    Optional<String> updateLock = updateStore.getLockToken(key);
    if (updateLock.isPresent()) {
      status = proposedEvent.getStatus();
      record = recordChange;
    } else {
      LOG.severe("Update " + key + " does not have a lock");
      status = ERROR;
      record = true;
    }

    LOG.info(String.format("Update %s is now in state %s", key, status));
    if (record) {
      updateStore.saveJobUpdateEvent(
          key,
          IJobUpdateEvent.build(proposedEvent.setTimestampMs(clock.nowMillis()).setStatus(status)));
    }

    if (TERMINAL_STATES.contains(status)) {
      if (updateLock.isPresent()) {
        lockManager.releaseLock(
            ILock.build(
                new Lock()
                    .setKey(LockKey.job(key.getJob().newBuilder()))
                    .setToken(updateLock.get())));
      }

      pulseHandler.remove(key);
    } else {
      pulseHandler.updatePulseStatus(key, status);
    }

    MonitorAction action = JobUpdateStateMachine.getActionForStatus(status);
    IJobKey job = key.getJob();
    if (action == STOP_WATCHING) {
      updates.remove(job);
    } else if (action == ROLL_FORWARD || action == ROLL_BACK) {
      if (action == ROLL_BACK) {
        updates.remove(job);
      } else {
        checkState(!updates.containsKey(job), "Updater already exists for " + job);
      }

      IJobUpdate jobUpdate = updateStore.fetchJobUpdate(key).get();
      UpdateFactory.Update update;
      try {
        update = updateFactory.newUpdate(jobUpdate.getInstructions(), action == ROLL_FORWARD);
      } catch (RuntimeException e) {
        LOG.log(Level.WARNING, "Uncaught exception: " + e, e);
        changeJobUpdateStatus(
            storeProvider,
            key,
            newEvent(ERROR).setMessage("Internal scheduler error: " + e.getMessage()),
            true);
        return;
      }
      updates.put(job, update);
      evaluateUpdater(storeProvider, update, jobUpdate.getSummary(), ImmutableMap.of());
    }
  }