@Override
public ILock acquireLock(final ILockKey lockKey, final String user) throws LockException {
return storage.write(
storeProvider -> {
LockStore.Mutable lockStore = storeProvider.getLockStore();
Optional<ILock> existingLock = lockStore.fetchLock(lockKey);
if (existingLock.isPresent()) {
throw new LockException(
String.format(
"Operation for: %s is already in progress. Started at: %s. Current owner: %s.",
formatLockKey(lockKey),
new Date(existingLock.get().getTimestampMs()).toString(),
existingLock.get().getUser()));
}
ILock lock =
ILock.build(
new Lock()
.setKey(lockKey.newBuilder())
.setToken(tokenGenerator.createNew().toString())
.setTimestampMs(clock.nowMillis())
.setUser(user));
lockStore.saveLock(lock);
return lock;
});
}
synchronized PulseState pulseAndGet(IJobUpdateKey key) {
PulseState state = pulseStates.get(key);
if (state != null) {
state =
pulseStates.put(
key,
new PulseState(state.getStatus(), state.getPulseTimeoutMs(), clock.nowMillis()));
}
return state;
}
boolean isBlocked(Clock clock) {
return clock.nowMillis() - lastPulseMs >= pulseTimeoutMs;
}
private void evaluateUpdater(
final MutableStoreProvider storeProvider,
final UpdateFactory.Update update,
IJobUpdateSummary summary,
Map<Integer, Optional<IScheduledTask>> changedInstance)
throws UpdateStateException {
JobUpdateStatus updaterStatus = summary.getState().getStatus();
final IJobUpdateKey key = summary.getKey();
JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
if (!updateStore.getLockToken(key).isPresent()) {
recordAndChangeJobUpdateStatus(
storeProvider, key, newEvent(ERROR).setMessage(LOST_LOCK_MESSAGE));
return;
}
IJobUpdateInstructions instructions = updateStore.fetchJobUpdateInstructions(key).get();
if (isCoordinatedAndPulseExpired(key, instructions)) {
// Move coordinated update into awaiting pulse state.
JobUpdateStatus blockedStatus = getBlockedState(summary.getState().getStatus());
changeUpdateStatus(
storeProvider, summary, newEvent(blockedStatus).setMessage(PULSE_TIMEOUT_MESSAGE));
return;
}
InstanceStateProvider<Integer, Optional<IScheduledTask>> stateProvider =
instanceId -> getActiveInstance(storeProvider.getTaskStore(), key.getJob(), instanceId);
EvaluationResult<Integer> result = update.getUpdater().evaluate(changedInstance, stateProvider);
LOG.info(key + " evaluation result: " + result);
for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
Iterable<InstanceUpdateStatus> statusChanges;
int instanceId = entry.getKey();
List<IJobInstanceUpdateEvent> savedEvents = updateStore.fetchInstanceEvents(key, instanceId);
Set<JobUpdateAction> savedActions =
FluentIterable.from(savedEvents).transform(EVENT_TO_ACTION).toSet();
// Don't bother persisting a sequence of status changes that represents an instance that
// was immediately recognized as being healthy and in the desired state.
if (entry.getValue().getStatusChanges().equals(NOOP_INSTANCE_UPDATE)
&& savedEvents.isEmpty()) {
LOG.info("Suppressing no-op update for instance " + instanceId);
statusChanges = ImmutableSet.of();
} else {
statusChanges = entry.getValue().getStatusChanges();
}
for (InstanceUpdateStatus statusChange : statusChanges) {
JobUpdateAction action = STATE_MAP.get(Pair.of(statusChange, updaterStatus));
requireNonNull(action);
// A given instance update action may only be issued once during the update lifecycle.
// Suppress duplicate events due to pause/resume operations.
if (savedActions.contains(action)) {
LOG.info(
String.format(
"Suppressing duplicate update %s for instance %s.", action, instanceId));
} else {
IJobInstanceUpdateEvent event =
IJobInstanceUpdateEvent.build(
new JobInstanceUpdateEvent()
.setInstanceId(instanceId)
.setTimestampMs(clock.nowMillis())
.setAction(action));
updateStore.saveJobInstanceUpdateEvent(summary.getKey(), event);
}
}
}
OneWayStatus status = result.getStatus();
if (status == SUCCEEDED || status == OneWayStatus.FAILED) {
if (SideEffect.hasActions(result.getSideEffects().values())) {
throw new IllegalArgumentException(
"A terminal state should not specify actions: " + result);
}
JobUpdateEvent event = new JobUpdateEvent();
if (status == SUCCEEDED) {
event.setStatus(update.getSuccessStatus());
} else {
event.setStatus(update.getFailureStatus());
// Generate a transition message based on one (arbitrary) instance in the group that pushed
// the update over the failure threshold (in all likelihood this group is of size 1).
// This is done as a rough cut to aid in diagnosing a failed update, as generating a
// complete summary would likely be of dubious value.
for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
Optional<Failure> failure = entry.getValue().getFailure();
if (failure.isPresent()) {
event.setMessage(failureMessage(entry.getKey(), failure.get()));
break;
}
}
}
changeUpdateStatus(storeProvider, summary, event);
} else {
LOG.info("Executing side-effects for update of " + key + ": " + result.getSideEffects());
for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
IInstanceKey instance = InstanceKeys.from(key.getJob(), entry.getKey());
Optional<InstanceAction> action = entry.getValue().getAction();
if (action.isPresent()) {
Optional<InstanceActionHandler> handler = action.get().getHandler();
if (handler.isPresent()) {
Optional<Amount<Long, Time>> reevaluateDelay =
handler
.get()
.getReevaluationDelay(
instance, instructions, storeProvider, stateManager, updaterStatus);
if (reevaluateDelay.isPresent()) {
executor.schedule(
getDeferredEvaluator(instance, key),
reevaluateDelay.get().getValue(),
reevaluateDelay.get().getUnit().getTimeUnit());
}
}
}
}
}
}
private void changeJobUpdateStatus(
MutableStoreProvider storeProvider,
IJobUpdateKey key,
JobUpdateEvent proposedEvent,
boolean recordChange)
throws UpdateStateException {
JobUpdateStatus status;
boolean record;
JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
Optional<String> updateLock = updateStore.getLockToken(key);
if (updateLock.isPresent()) {
status = proposedEvent.getStatus();
record = recordChange;
} else {
LOG.severe("Update " + key + " does not have a lock");
status = ERROR;
record = true;
}
LOG.info(String.format("Update %s is now in state %s", key, status));
if (record) {
updateStore.saveJobUpdateEvent(
key,
IJobUpdateEvent.build(proposedEvent.setTimestampMs(clock.nowMillis()).setStatus(status)));
}
if (TERMINAL_STATES.contains(status)) {
if (updateLock.isPresent()) {
lockManager.releaseLock(
ILock.build(
new Lock()
.setKey(LockKey.job(key.getJob().newBuilder()))
.setToken(updateLock.get())));
}
pulseHandler.remove(key);
} else {
pulseHandler.updatePulseStatus(key, status);
}
MonitorAction action = JobUpdateStateMachine.getActionForStatus(status);
IJobKey job = key.getJob();
if (action == STOP_WATCHING) {
updates.remove(job);
} else if (action == ROLL_FORWARD || action == ROLL_BACK) {
if (action == ROLL_BACK) {
updates.remove(job);
} else {
checkState(!updates.containsKey(job), "Updater already exists for " + job);
}
IJobUpdate jobUpdate = updateStore.fetchJobUpdate(key).get();
UpdateFactory.Update update;
try {
update = updateFactory.newUpdate(jobUpdate.getInstructions(), action == ROLL_FORWARD);
} catch (RuntimeException e) {
LOG.log(Level.WARNING, "Uncaught exception: " + e, e);
changeJobUpdateStatus(
storeProvider,
key,
newEvent(ERROR).setMessage("Internal scheduler error: " + e.getMessage()),
true);
return;
}
updates.put(job, update);
evaluateUpdater(storeProvider, update, jobUpdate.getSummary(), ImmutableMap.of());
}
}