@Test
public void testBufferNotCloseOnFail() throws Exception {
outputBuffers =
outputBuffers
.withBuffer("out", new UnpartitionedPagePartitionFunction())
.withNoMoreBufferIds();
taskExecution.addResultQueue(outputBuffers);
BufferResult bufferResult =
taskExecution.getResults(
"out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(0, TimeUnit.MILLISECONDS));
assertFalse(bufferResult.isBufferClosed());
bufferResult =
taskExecution.getResults(
"out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(0, TimeUnit.MILLISECONDS));
assertFalse(bufferResult.isBufferClosed());
taskExecution.fail(new Exception("test"));
assertEquals(taskExecution.getTaskInfo(false).getState(), TaskState.FAILED);
// buffer will not be closed by fail event. event is async so wait for 500 MS for event to fire
bufferResult =
taskExecution.getResults(
"out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(500, TimeUnit.MILLISECONDS));
assertFalse(bufferResult.isBufferClosed());
bufferResult =
taskExecution.getResults(
"out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(500, TimeUnit.MILLISECONDS));
assertFalse(bufferResult.isBufferClosed());
}
@Test
public void testBufferCloseOnFinish() throws Exception {
outputBuffers =
outputBuffers
.withBuffer("out", new UnpartitionedPagePartitionFunction())
.withNoMoreBufferIds();
taskExecution.addResultQueue(outputBuffers);
BufferResult bufferResult =
taskExecution.getResults(
"out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(0, TimeUnit.MILLISECONDS));
assertFalse(bufferResult.isBufferClosed());
bufferResult =
taskExecution.getResults(
"out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(0, TimeUnit.MILLISECONDS));
assertFalse(bufferResult.isBufferClosed());
taskExecution.addSources(
ImmutableList.of(new TaskSource(tableScanNodeId, ImmutableSet.<ScheduledSplit>of(), true)));
assertEquals(taskExecution.getTaskInfo(false).getState(), TaskState.FINISHED);
// buffer will be closed by cancel event (wait for 500 MS for event to fire)
bufferResult =
taskExecution.getResults(
"out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(500, TimeUnit.MILLISECONDS));
assertTrue(bufferResult.isBufferClosed());
bufferResult =
taskExecution.getResults(
"out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(500, TimeUnit.MILLISECONDS));
assertTrue(bufferResult.isBufferClosed());
}
@Override
public synchronized void setOutputBuffers(OutputBuffers newOutputBuffers) {
if (getTaskInfo().getState().isDone()) {
return;
}
if (newOutputBuffers.getVersion() > outputBuffers.get().getVersion()) {
outputBuffers.set(newOutputBuffers);
needsUpdate.set(true);
scheduleUpdate();
}
}
public HttpRemoteTask(
Session session,
TaskId taskId,
String nodeId,
URI location,
PlanFragment planFragment,
Multimap<PlanNodeId, Split> initialSplits,
OutputBuffers outputBuffers,
HttpClient httpClient,
Executor executor,
ScheduledExecutorService errorScheduledExecutor,
Duration minErrorDuration,
Duration refreshMaxWait,
JsonCodec<TaskInfo> taskInfoCodec,
JsonCodec<TaskUpdateRequest> taskUpdateRequestCodec,
SplitCountChangeListener splitCountChangeListener) {
requireNonNull(session, "session is null");
requireNonNull(taskId, "taskId is null");
requireNonNull(nodeId, "nodeId is null");
requireNonNull(location, "location is null");
requireNonNull(planFragment, "planFragment1 is null");
requireNonNull(outputBuffers, "outputBuffers is null");
requireNonNull(httpClient, "httpClient is null");
requireNonNull(executor, "executor is null");
requireNonNull(taskInfoCodec, "taskInfoCodec is null");
requireNonNull(taskUpdateRequestCodec, "taskUpdateRequestCodec is null");
requireNonNull(splitCountChangeListener, "splitCountChangeListener is null");
try (SetThreadName ignored = new SetThreadName("HttpRemoteTask-%s", taskId)) {
this.taskId = taskId;
this.session = session;
this.nodeId = nodeId;
this.planFragment = planFragment;
this.outputBuffers.set(outputBuffers);
this.httpClient = httpClient;
this.executor = executor;
this.errorScheduledExecutor = errorScheduledExecutor;
this.taskInfoCodec = taskInfoCodec;
this.taskUpdateRequestCodec = taskUpdateRequestCodec;
this.updateErrorTracker =
new RequestErrorTracker(
taskId, location, minErrorDuration, errorScheduledExecutor, "updating task");
this.getErrorTracker =
new RequestErrorTracker(
taskId, location, minErrorDuration, errorScheduledExecutor, "getting info for task");
this.splitCountChangeListener = splitCountChangeListener;
for (Entry<PlanNodeId, Split> entry :
requireNonNull(initialSplits, "initialSplits is null").entries()) {
ScheduledSplit scheduledSplit =
new ScheduledSplit(nextSplitId.getAndIncrement(), entry.getValue());
pendingSplits.put(entry.getKey(), scheduledSplit);
}
if (initialSplits.containsKey(planFragment.getPartitionedSource())) {
pendingSourceSplitCount = initialSplits.get(planFragment.getPartitionedSource()).size();
fireSplitCountChanged(pendingSourceSplitCount);
}
List<BufferInfo> bufferStates =
outputBuffers
.getBuffers()
.keySet()
.stream()
.map(outputId -> new BufferInfo(outputId, false, 0, 0, PageBufferInfo.empty()))
.collect(toImmutableList());
TaskStats taskStats = new TaskStats(DateTime.now(), null);
taskInfo =
new StateMachine<>(
"task " + taskId,
executor,
new TaskInfo(
taskId,
Optional.empty(),
TaskInfo.MIN_VERSION,
TaskState.PLANNED,
location,
DateTime.now(),
new SharedBufferInfo(BufferState.OPEN, true, true, 0, 0, 0, 0, bufferStates),
ImmutableSet.<PlanNodeId>of(),
taskStats,
ImmutableList.<ExecutionFailureInfo>of()));
continuousTaskInfoFetcher = new ContinuousTaskInfoFetcher(refreshMaxWait);
}
}