public String getDescription() {
Long lastUpTime = connectionLastUp.get();
Long lastDownTime = connectionLastDown.get();
Duration serviceFailedStabilizationDelay = getConnectionFailedStabilizationDelay();
Duration serviceRecoveredStabilizationDelay = getConnectionRecoveredStabilizationDelay();
return String.format(
"endpoint=%s; connected=%s; timeNow=%s; lastUp=%s; lastDown=%s; lastPublished=%s; "
+ "currentFailurePeriod=%s; currentRecoveryPeriod=%s",
getConfig(ENDPOINT),
connected,
Time.makeDateString(System.currentTimeMillis()),
(lastUpTime != null ? Time.makeDateString(lastUpTime) : "<never>"),
(lastDownTime != null ? Time.makeDateString(lastDownTime) : "<never>"),
lastPublished,
(currentFailureStartTime != null ? getTimeStringSince(currentFailureStartTime) : "<none>")
+ " (stabilization "
+ makeTimeStringRounded(serviceFailedStabilizationDelay)
+ ")",
(currentRecoveryStartTime != null
? getTimeStringSince(currentRecoveryStartTime)
: "<none>")
+ " (stabilization "
+ makeTimeStringRounded(serviceRecoveredStabilizationDelay)
+ ")");
}
@Override
public void start(Collection<? extends Location> locations) {
super.start(locations);
connectSensors();
Time.sleep(getConfig(DELAY_BEFORE_ADVERTISING_CLUSTER));
// FIXME: add a quorum to tolerate failed nodes before setting on fire.
Optional<Entity> anyNode =
Iterables.tryFind(
getMembers(),
new Predicate<Entity>() {
@Override
public boolean apply(@Nullable Entity entity) {
return (entity instanceof RiakNode
&& hasMemberJoinedCluster(entity)
&& entity.getAttribute(RiakNode.SERVICE_UP));
}
});
if (anyNode.isPresent()) {
log.info(
"Planning and Committing cluster changes on node: {}, cluster: {}",
anyNode.get().getId(),
getId());
Entities.invokeEffector(this, anyNode.get(), RiakNode.COMMIT_RIAK_CLUSTER);
setAttribute(IS_CLUSTER_INIT, true);
} else {
log.warn("No Riak Nodes are found on the cluster: {}. Initialization Failed", getId());
setAttribute(SERVICE_STATE, Lifecycle.ON_FIRE);
}
}
@Test
public void testSimple() throws InterruptedException, ExecutionException {
Callable<String> mainJob =
new Callable<String>() {
public String call() {
log.info("main job - " + Tasks.current());
messages.add("main");
DynamicTasks.queue(sayTask("world"));
return "bye";
}
};
DynamicSequentialTask<String> t = new DynamicSequentialTask<String>(mainJob);
// this should be added before anything added when the task is invoked
t.queue(sayTask("hello"));
Assert.assertEquals(messages, Lists.newArrayList());
Assert.assertEquals(t.isBegun(), false);
Assert.assertEquals(Iterables.size(t.getChildren()), 1);
ec.submit(t);
Assert.assertEquals(t.isSubmitted(), true);
Assert.assertEquals(t.getUnchecked(Duration.ONE_SECOND), "bye");
long elapsed = t.getEndTimeUtc() - t.getSubmitTimeUtc();
Assert.assertTrue(
elapsed < 1000,
"elapsed time should have been less than 1s but was " + Time.makeTimeString(elapsed, true));
Assert.assertEquals(Iterables.size(t.getChildren()), 2);
Assert.assertEquals(messages.size(), 3, "expected 3 entries, but had " + messages);
// either main or hello can be first, but world should be last
Assert.assertEquals(messages.get(2), "world");
}
@Test
public void testCancelled() throws InterruptedException, ExecutionException {
Task<List<?>> t =
Tasks.sequential(sayTask("1"), sayTask("2a", Duration.THIRTY_SECONDS, "2b"), sayTask("3"));
ec.submit(t);
synchronized (messages) {
while (messages.size() <= 1) messages.wait();
}
Assert.assertEquals(messages, Arrays.asList("1", "2a"));
Time.sleep(Duration.millis(50));
t.cancel(true);
Assert.assertTrue(t.isDone());
// 2 should get cancelled, and invoke the cancellation semaphore
// 3 should get cancelled and not run at all
Assert.assertEquals(messages, Arrays.asList("1", "2a"));
// Need to ensure that 2 has been started; race where we might cancel it before its run method
// is even begun. Hence doing "2a; pause; 2b" where nothing is interruptable before pause.
Assert.assertTrue(cancellations.tryAcquire(10, TimeUnit.SECONDS));
Iterator<Task<?>> ci = ((HasTaskChildren) t).getChildren().iterator();
Assert.assertEquals(ci.next().get(), "1");
Task<?> task2 = ci.next();
Assert.assertTrue(task2.isBegun());
Assert.assertTrue(task2.isDone());
Assert.assertTrue(task2.isCancelled());
Task<?> task3 = ci.next();
Assert.assertFalse(task3.isBegun());
Assert.assertTrue(task2.isDone());
Assert.assertTrue(task2.isCancelled());
// but we do _not_ get a mutex from task3 as it does not run (is not interrupted)
Assert.assertEquals(cancellations.availablePermits(), 0);
}
protected String getExplanation(Lifecycle state) {
Duration serviceFailedStabilizationDelay = getConfig(ENTITY_FAILED_STABILIZATION_DELAY);
Duration serviceRecoveredStabilizationDelay = getConfig(ENTITY_RECOVERED_STABILIZATION_DELAY);
return String.format(
"location=%s; status=%s; lastPublished=%s; timeNow=%s; "
+ "currentFailurePeriod=%s; currentRecoveryPeriod=%s",
entity.getLocations(),
(state != null ? state : "<unreported>"),
lastPublished,
Time.makeDateString(System.currentTimeMillis()),
(currentFailureStartTime != null ? getTimeStringSince(currentFailureStartTime) : "<none>")
+ " (stabilization "
+ Time.makeTimeStringRounded(serviceFailedStabilizationDelay)
+ ")",
(currentRecoveryStartTime != null ? getTimeStringSince(currentRecoveryStartTime) : "<none>")
+ " (stabilization "
+ Time.makeTimeStringRounded(serviceRecoveredStabilizationDelay)
+ ")");
}
/** Setup renderer hints for the MXBean attributes. */
public static void init() {
if (initialized.get()) return;
synchronized (initialized) {
if (initialized.get()) return;
RendererHints.register(
UsesJavaMXBeans.USED_HEAP_MEMORY, RendererHints.displayValue(ByteSizeStrings.metric()));
RendererHints.register(
UsesJavaMXBeans.INIT_HEAP_MEMORY, RendererHints.displayValue(ByteSizeStrings.metric()));
RendererHints.register(
UsesJavaMXBeans.MAX_HEAP_MEMORY, RendererHints.displayValue(ByteSizeStrings.metric()));
RendererHints.register(
UsesJavaMXBeans.COMMITTED_HEAP_MEMORY,
RendererHints.displayValue(ByteSizeStrings.metric()));
RendererHints.register(
UsesJavaMXBeans.NON_HEAP_MEMORY_USAGE,
RendererHints.displayValue(ByteSizeStrings.metric()));
RendererHints.register(
UsesJavaMXBeans.TOTAL_PHYSICAL_MEMORY_SIZE,
RendererHints.displayValue(ByteSizeStrings.metric()));
RendererHints.register(
UsesJavaMXBeans.FREE_PHYSICAL_MEMORY_SIZE,
RendererHints.displayValue(ByteSizeStrings.metric()));
RendererHints.register(
UsesJavaMXBeans.START_TIME, RendererHints.displayValue(Time.toDateString()));
RendererHints.register(
UsesJavaMXBeans.UP_TIME, RendererHints.displayValue(Duration.millisToStringRounded()));
RendererHints.register(
UsesJavaMXBeans.PROCESS_CPU_TIME,
RendererHints.displayValue(Duration.millisToStringRounded()));
RendererHints.register(
UsesJavaMXBeans.PROCESS_CPU_TIME_FRACTION_LAST,
RendererHints.displayValue(MathFunctions.percent(4)));
RendererHints.register(
UsesJavaMXBeans.PROCESS_CPU_TIME_FRACTION_IN_WINDOW,
RendererHints.displayValue(MathFunctions.percent(4)));
initialized.set(true);
}
}
/** Setup renderer hints. */
public static void init() {
if (initialized.getAndSet(true)) return;
final Function<Double, Long> longValue =
new Function<Double, Long>() {
@Override
public Long apply(@Nullable Double input) {
if (input == null) return null;
return input.longValue();
}
};
RendererHints.register(UPTIME, RendererHints.displayValue(Time.toTimeStringRounded()));
RendererHints.register(CPU_USAGE, RendererHints.displayValue(MathFunctions.percent(2)));
RendererHints.register(AVERAGE_CPU_USAGE, RendererHints.displayValue(MathFunctions.percent(2)));
RendererHints.register(
FREE_MEMORY,
RendererHints.displayValue(
Functionals.chain(MathFunctions.times(1000L), ByteSizeStrings.metric())));
RendererHints.register(
TOTAL_MEMORY,
RendererHints.displayValue(
Functionals.chain(MathFunctions.times(1000L), ByteSizeStrings.metric())));
RendererHints.register(
USED_MEMORY,
RendererHints.displayValue(
Functionals.chain(MathFunctions.times(1000L), ByteSizeStrings.metric())));
RendererHints.register(
USED_MEMORY_DELTA_PER_SECOND_LAST,
RendererHints.displayValue(Functionals.chain(longValue, ByteSizeStrings.metric())));
RendererHints.register(
USED_MEMORY_DELTA_PER_SECOND_IN_WINDOW,
RendererHints.displayValue(Functionals.chain(longValue, ByteSizeStrings.metric())));
}
private String getTimeStringSince(Long time) {
return time == null ? null : Time.makeTimeStringRounded(System.currentTimeMillis() - time);
}
@Override
protected void setActualState(Lifecycle state) {
long now = System.currentTimeMillis();
synchronized (mutex) {
if (state == Lifecycle.ON_FIRE) {
if (lastPublished == LastPublished.FAILED) {
if (currentRecoveryStartTime != null) {
if (LOG.isDebugEnabled())
LOG.debug(
"{} health-check for {}, component was recovering, now failing: {}",
new Object[] {this, entity, getExplanation(state)});
currentRecoveryStartTime = null;
publishEntityRecoveredTime = null;
} else {
if (LOG.isTraceEnabled())
LOG.trace(
"{} health-check for {}, component still failed: {}",
new Object[] {this, entity, getExplanation(state)});
}
} else {
if (firstUpTime == null && getConfig(ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP)) {
// suppress; won't publish
} else if (currentFailureStartTime == null) {
if (LOG.isDebugEnabled())
LOG.debug(
"{} health-check for {}, component now failing: {}",
new Object[] {this, entity, getExplanation(state)});
currentFailureStartTime = now;
publishEntityFailedTime =
currentFailureStartTime
+ getConfig(ENTITY_FAILED_STABILIZATION_DELAY).toMilliseconds();
} else {
if (LOG.isTraceEnabled())
LOG.trace(
"{} health-check for {}, component continuing failing: {}",
new Object[] {this, entity, getExplanation(state)});
}
}
if (setEntityOnFireTime == null) {
setEntityOnFireTime =
now + getConfig(SERVICE_ON_FIRE_STABILIZATION_DELAY).toMilliseconds();
}
currentRecoveryStartTime = null;
publishEntityRecoveredTime = null;
} else if (state == Lifecycle.RUNNING) {
if (lastPublished == LastPublished.FAILED) {
if (currentRecoveryStartTime == null) {
if (LOG.isDebugEnabled())
LOG.debug(
"{} health-check for {}, component now recovering: {}",
new Object[] {this, entity, getExplanation(state)});
currentRecoveryStartTime = now;
publishEntityRecoveredTime =
currentRecoveryStartTime
+ getConfig(ENTITY_RECOVERED_STABILIZATION_DELAY).toMilliseconds();
} else {
if (LOG.isTraceEnabled())
LOG.trace(
"{} health-check for {}, component continuing recovering: {}",
new Object[] {this, entity, getExplanation(state)});
}
} else {
if (currentFailureStartTime != null) {
if (LOG.isDebugEnabled())
LOG.debug(
"{} health-check for {}, component was failing, now healthy: {}",
new Object[] {this, entity, getExplanation(state)});
} else {
if (LOG.isTraceEnabled())
LOG.trace(
"{} health-check for {}, component still healthy: {}",
new Object[] {this, entity, getExplanation(state)});
}
}
currentFailureStartTime = null;
publishEntityFailedTime = null;
setEntityOnFireTime = null;
} else {
if (LOG.isTraceEnabled())
LOG.trace(
"{} health-check for {}, in unconfirmed sate: {}",
new Object[] {this, entity, getExplanation(state)});
}
long recomputeIn = Long.MAX_VALUE; // For whether to call recomputeAfterDelay
if (publishEntityFailedTime != null) {
long delayBeforeCheck = publishEntityFailedTime - now;
if (delayBeforeCheck <= 0) {
if (LOG.isDebugEnabled())
LOG.debug(
"{} publishing failed (state={}; currentFailureStartTime={}; now={}",
new Object[] {
this,
state,
Time.makeDateString(currentFailureStartTime),
Time.makeDateString(now)
});
publishEntityFailedTime = null;
lastPublished = LastPublished.FAILED;
entity.emit(
HASensors.ENTITY_FAILED,
new HASensors.FailureDescriptor(entity, getFailureDescription(now)));
} else {
recomputeIn = Math.min(recomputeIn, delayBeforeCheck);
}
} else if (publishEntityRecoveredTime != null) {
long delayBeforeCheck = publishEntityRecoveredTime - now;
if (delayBeforeCheck <= 0) {
if (LOG.isDebugEnabled())
LOG.debug(
"{} publishing recovered (state={}; currentRecoveryStartTime={}; now={}",
new Object[] {
this,
state,
Time.makeDateString(currentRecoveryStartTime),
Time.makeDateString(now)
});
publishEntityRecoveredTime = null;
lastPublished = LastPublished.RECOVERED;
entity.emit(HASensors.ENTITY_RECOVERED, new HASensors.FailureDescriptor(entity, null));
} else {
recomputeIn = Math.min(recomputeIn, delayBeforeCheck);
}
}
if (setEntityOnFireTime != null) {
long delayBeforeCheck = setEntityOnFireTime - now;
if (delayBeforeCheck <= 0) {
if (LOG.isDebugEnabled())
LOG.debug(
"{} setting on-fire, now that deferred period has passed (state={})",
new Object[] {this, state});
setEntityOnFireTime = null;
super.setActualState(state);
} else {
recomputeIn = Math.min(recomputeIn, delayBeforeCheck);
}
} else {
super.setActualState(state);
}
if (recomputeIn < Long.MAX_VALUE) {
recomputeAfterDelay(recomputeIn);
}
}
}