private synchronized void scheduleNextRequest() {
// stopped or done?
TaskInfo taskInfo = HttpRemoteTask.this.taskInfo.get();
if (!running || taskInfo.getState().isDone()) {
return;
}
// outstanding request?
if (future != null && !future.isDone()) {
// this should never happen
log.error("Can not reschedule update because an update is already running");
return;
}
// if throttled due to error, asynchronously wait for timeout and try again
ListenableFuture<?> errorRateLimit = getErrorTracker.acquireRequestPermit();
if (!errorRateLimit.isDone()) {
errorRateLimit.addListener(this::scheduleNextRequest, executor);
return;
}
Request request =
prepareGet()
.setUri(uriBuilderFrom(taskInfo.getSelf()).addParameter("summarize").build())
.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.JSON_UTF_8.toString())
.setHeader(PrestoHeaders.PRESTO_CURRENT_STATE, taskInfo.getState().toString())
.setHeader(PrestoHeaders.PRESTO_MAX_WAIT, refreshMaxWait.toString())
.build();
getErrorTracker.startRequest();
future = httpClient.executeAsync(request, createFullJsonResponseHandler(taskInfoCodec));
Futures.addCallback(
future, new SimpleHttpResponseHandler<>(this, request.getUri()), executor);
}
/** Move the task directly to the failed state */
private void failTask(Throwable cause) {
TaskInfo taskInfo = getTaskInfo();
if (!taskInfo.getState().isDone()) {
log.debug(cause, "Remote task failed: %s", taskInfo.getSelf());
}
updateTaskInfo(
new TaskInfo(
taskInfo.getTaskId(),
taskInfo.getNodeInstanceId(),
TaskInfo.MAX_VERSION,
TaskState.FAILED,
taskInfo.getSelf(),
taskInfo.getLastHeartbeat(),
taskInfo.getOutputBuffers(),
taskInfo.getNoMoreSplits(),
taskInfo.getStats(),
ImmutableList.of(toFailure(cause))));
}
@Override
public synchronized void abort() {
try (SetThreadName ignored = new SetThreadName("HttpRemoteTask-%s", taskId)) {
// clear pending splits to free memory
fireSplitCountChanged(-pendingSourceSplitCount);
pendingSplits.clear();
pendingSourceSplitCount = 0;
// cancel pending request
if (currentRequest != null) {
currentRequest.cancel(true);
currentRequest = null;
currentRequestStartNanos = 0;
}
// mark task as canceled (if not already done)
TaskInfo taskInfo = getTaskInfo();
URI uri = taskInfo.getSelf();
updateTaskInfo(
new TaskInfo(
taskInfo.getTaskId(),
taskInfo.getNodeInstanceId(),
TaskInfo.MAX_VERSION,
TaskState.ABORTED,
uri,
taskInfo.getLastHeartbeat(),
taskInfo.getOutputBuffers(),
taskInfo.getNoMoreSplits(),
taskInfo.getStats(),
ImmutableList.<ExecutionFailureInfo>of()));
// send abort to task and ignore response
Request request =
prepareDelete().setUri(uriBuilderFrom(uri).addParameter("summarize").build()).build();
scheduleAsyncCleanupRequest(new Backoff(MAX_CLEANUP_RETRY_TIME), request, "abort");
}
}
private synchronized void updateTaskInfo(TaskInfo newValue, List<TaskSource> sources) {
if (newValue.getState().isDone()) {
// splits can be huge so clear the list
pendingSplits.clear();
fireSplitCountChanged(-pendingSourceSplitCount);
pendingSourceSplitCount = 0;
}
int oldPartitionedSplitCount = getPartitionedSplitCount();
// change to new value if old value is not changed and new value has a newer version
AtomicBoolean workerRestarted = new AtomicBoolean();
boolean updated =
taskInfo.setIf(
newValue,
oldValue -> {
// did the worker restart
if (oldValue.getNodeInstanceId().isPresent()
&& !oldValue.getNodeInstanceId().equals(newValue.getNodeInstanceId())) {
workerRestarted.set(true);
return false;
}
if (oldValue.getState().isDone()) {
// never update if the task has reached a terminal state
return false;
}
if (newValue.getVersion() < oldValue.getVersion()) {
// don't update to an older version (same version is ok)
return false;
}
return true;
});
if (workerRestarted.get()) {
PrestoException exception =
new PrestoException(
WORKER_RESTARTED, format("%s (%s)", WORKER_RESTARTED_ERROR, newValue.getSelf()));
failTask(exception);
abort();
}
// remove acknowledged splits, which frees memory
for (TaskSource source : sources) {
PlanNodeId planNodeId = source.getPlanNodeId();
int removed = 0;
for (ScheduledSplit split : source.getSplits()) {
if (pendingSplits.remove(planNodeId, split)) {
removed++;
}
}
if (planNodeId.equals(planFragment.getPartitionedSource())) {
pendingSourceSplitCount -= removed;
}
}
if (updated) {
if (getTaskInfo().getState().isDone()) {
fireSplitCountChanged(-oldPartitionedSplitCount);
} else {
fireSplitCountChanged(getPartitionedSplitCount() - oldPartitionedSplitCount);
}
}
}
