@Test
public void durationOverflow() {
Backoff b = new Backoff();
b.setMin(100);
b.setMax(10000);
b.setJitter(1.0);
for (int i = 0; i < 100; i++) {
long duration = b.duration();
assertTrue(100 <= duration && duration <= 10000);
}
}
@Test
public void durationShouldIncreaseTheBackoff() {
Backoff b = new Backoff();
assertTrue(100 == b.duration());
assertTrue(200 == b.duration());
assertTrue(400 == b.duration());
assertTrue(800 == b.duration());
b.reset();
assertTrue(100 == b.duration());
assertTrue(200 == b.duration());
}
public void startRequest() {
// before scheduling a new request clear the error timer
// we consider a request to be "new" if there are no current failures
if (backoff.getFailureCount() == 0) {
requestSucceeded();
}
}
public void requestFailed(Throwable reason) throws PrestoException {
// cancellation is not a failure
if (reason instanceof CancellationException) {
return;
}
if (reason instanceof RejectedExecutionException) {
throw new PrestoException(REMOTE_TASK_ERROR, reason);
}
// log failure message
if (isExpectedError(reason)) {
// don't print a stack for a known errors
log.warn("Error " + jobDescription + " %s: %s: %s", taskId, reason.getMessage(), taskUri);
} else {
log.warn(reason, "Error " + jobDescription + " %s: %s", taskId, taskUri);
}
// remember the first 10 errors
if (errorsSinceLastSuccess.size() < 10) {
errorsSinceLastSuccess.add(reason);
}
// fail the task, if we have more than X failures in a row and more than Y seconds have passed
// since the last request
if (backoff.failure()) {
// it is weird to mark the task failed locally and then cancel the remote task, but there is
// no way to tell a remote task that it is failed
PrestoException exception =
new PrestoException(
TOO_MANY_REQUESTS_FAILED,
format(
"%s (%s %s - %s failures, time since last success %s)",
WORKER_NODE_ERROR,
jobDescription,
taskUri,
backoff.getFailureCount(),
backoff.getTimeSinceLastSuccess().convertTo(SECONDS)));
errorsSinceLastSuccess.forEach(exception::addSuppressed);
throw exception;
}
}
public ListenableFuture<?> acquireRequestPermit() {
long delayNanos = backoff.getBackoffDelayNanos();
if (delayNanos == 0) {
return Futures.immediateFuture(null);
}
ListenableFutureTask<Object> futureTask = ListenableFutureTask.create(() -> null);
scheduledExecutor.schedule(futureTask, delayNanos, NANOSECONDS);
return futureTask;
}
public void requestSucceeded() {
backoff.success();
errorsSinceLastSuccess.clear();
}