private void instanceChanged(final IInstanceKey instance, final Optional<IScheduledTask> state) {
storage.write(
(NoResult.Quiet)
storeProvider -> {
IJobKey job = instance.getJobKey();
UpdateFactory.Update update = updates.get(job);
if (update != null) {
if (update.getUpdater().containsInstance(instance.getInstanceId())) {
LOG.info("Forwarding task change for " + InstanceKeys.toString(instance));
try {
evaluateUpdater(
storeProvider,
update,
getOnlyMatch(storeProvider.getJobUpdateStore(), queryActiveByJob(job)),
ImmutableMap.of(instance.getInstanceId(), state));
} catch (UpdateStateException e) {
throw Throwables.propagate(e);
}
} else {
LOG.info(
"Instance "
+ instance
+ " is not part of active update for "
+ JobKeys.canonicalString(job));
}
}
});
}
private void evaluateUpdater(
final MutableStoreProvider storeProvider,
final UpdateFactory.Update update,
IJobUpdateSummary summary,
Map<Integer, Optional<IScheduledTask>> changedInstance)
throws UpdateStateException {
JobUpdateStatus updaterStatus = summary.getState().getStatus();
final IJobUpdateKey key = summary.getKey();
JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
if (!updateStore.getLockToken(key).isPresent()) {
recordAndChangeJobUpdateStatus(
storeProvider, key, newEvent(ERROR).setMessage(LOST_LOCK_MESSAGE));
return;
}
IJobUpdateInstructions instructions = updateStore.fetchJobUpdateInstructions(key).get();
if (isCoordinatedAndPulseExpired(key, instructions)) {
// Move coordinated update into awaiting pulse state.
JobUpdateStatus blockedStatus = getBlockedState(summary.getState().getStatus());
changeUpdateStatus(
storeProvider, summary, newEvent(blockedStatus).setMessage(PULSE_TIMEOUT_MESSAGE));
return;
}
InstanceStateProvider<Integer, Optional<IScheduledTask>> stateProvider =
instanceId -> getActiveInstance(storeProvider.getTaskStore(), key.getJob(), instanceId);
EvaluationResult<Integer> result = update.getUpdater().evaluate(changedInstance, stateProvider);
LOG.info(key + " evaluation result: " + result);
for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
Iterable<InstanceUpdateStatus> statusChanges;
int instanceId = entry.getKey();
List<IJobInstanceUpdateEvent> savedEvents = updateStore.fetchInstanceEvents(key, instanceId);
Set<JobUpdateAction> savedActions =
FluentIterable.from(savedEvents).transform(EVENT_TO_ACTION).toSet();
// Don't bother persisting a sequence of status changes that represents an instance that
// was immediately recognized as being healthy and in the desired state.
if (entry.getValue().getStatusChanges().equals(NOOP_INSTANCE_UPDATE)
&& savedEvents.isEmpty()) {
LOG.info("Suppressing no-op update for instance " + instanceId);
statusChanges = ImmutableSet.of();
} else {
statusChanges = entry.getValue().getStatusChanges();
}
for (InstanceUpdateStatus statusChange : statusChanges) {
JobUpdateAction action = STATE_MAP.get(Pair.of(statusChange, updaterStatus));
requireNonNull(action);
// A given instance update action may only be issued once during the update lifecycle.
// Suppress duplicate events due to pause/resume operations.
if (savedActions.contains(action)) {
LOG.info(
String.format(
"Suppressing duplicate update %s for instance %s.", action, instanceId));
} else {
IJobInstanceUpdateEvent event =
IJobInstanceUpdateEvent.build(
new JobInstanceUpdateEvent()
.setInstanceId(instanceId)
.setTimestampMs(clock.nowMillis())
.setAction(action));
updateStore.saveJobInstanceUpdateEvent(summary.getKey(), event);
}
}
}
OneWayStatus status = result.getStatus();
if (status == SUCCEEDED || status == OneWayStatus.FAILED) {
if (SideEffect.hasActions(result.getSideEffects().values())) {
throw new IllegalArgumentException(
"A terminal state should not specify actions: " + result);
}
JobUpdateEvent event = new JobUpdateEvent();
if (status == SUCCEEDED) {
event.setStatus(update.getSuccessStatus());
} else {
event.setStatus(update.getFailureStatus());
// Generate a transition message based on one (arbitrary) instance in the group that pushed
// the update over the failure threshold (in all likelihood this group is of size 1).
// This is done as a rough cut to aid in diagnosing a failed update, as generating a
// complete summary would likely be of dubious value.
for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
Optional<Failure> failure = entry.getValue().getFailure();
if (failure.isPresent()) {
event.setMessage(failureMessage(entry.getKey(), failure.get()));
break;
}
}
}
changeUpdateStatus(storeProvider, summary, event);
} else {
LOG.info("Executing side-effects for update of " + key + ": " + result.getSideEffects());
for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
IInstanceKey instance = InstanceKeys.from(key.getJob(), entry.getKey());
Optional<InstanceAction> action = entry.getValue().getAction();
if (action.isPresent()) {
Optional<InstanceActionHandler> handler = action.get().getHandler();
if (handler.isPresent()) {
Optional<Amount<Long, Time>> reevaluateDelay =
handler
.get()
.getReevaluationDelay(
instance, instructions, storeProvider, stateManager, updaterStatus);
if (reevaluateDelay.isPresent()) {
executor.schedule(
getDeferredEvaluator(instance, key),
reevaluateDelay.get().getValue(),
reevaluateDelay.get().getUnit().getTimeUnit());
}
}
}
}
}
}