Ejemplo n.º 1
0
  @Test
  public void testBufferNotCloseOnFail() throws Exception {
    outputBuffers =
        outputBuffers
            .withBuffer("out", new UnpartitionedPagePartitionFunction())
            .withNoMoreBufferIds();
    taskExecution.addResultQueue(outputBuffers);

    BufferResult bufferResult =
        taskExecution.getResults(
            "out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(0, TimeUnit.MILLISECONDS));
    assertFalse(bufferResult.isBufferClosed());

    bufferResult =
        taskExecution.getResults(
            "out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(0, TimeUnit.MILLISECONDS));
    assertFalse(bufferResult.isBufferClosed());

    taskExecution.fail(new Exception("test"));
    assertEquals(taskExecution.getTaskInfo(false).getState(), TaskState.FAILED);

    // buffer will not be closed by fail event.  event is async so wait for 500 MS for event to fire
    bufferResult =
        taskExecution.getResults(
            "out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(500, TimeUnit.MILLISECONDS));
    assertFalse(bufferResult.isBufferClosed());

    bufferResult =
        taskExecution.getResults(
            "out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(500, TimeUnit.MILLISECONDS));
    assertFalse(bufferResult.isBufferClosed());
  }
Ejemplo n.º 2
0
  @Test
  public void testBufferCloseOnFinish() throws Exception {
    outputBuffers =
        outputBuffers
            .withBuffer("out", new UnpartitionedPagePartitionFunction())
            .withNoMoreBufferIds();
    taskExecution.addResultQueue(outputBuffers);

    BufferResult bufferResult =
        taskExecution.getResults(
            "out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(0, TimeUnit.MILLISECONDS));
    assertFalse(bufferResult.isBufferClosed());

    bufferResult =
        taskExecution.getResults(
            "out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(0, TimeUnit.MILLISECONDS));
    assertFalse(bufferResult.isBufferClosed());

    taskExecution.addSources(
        ImmutableList.of(new TaskSource(tableScanNodeId, ImmutableSet.<ScheduledSplit>of(), true)));
    assertEquals(taskExecution.getTaskInfo(false).getState(), TaskState.FINISHED);

    // buffer will be closed by cancel event (wait for 500 MS for event to fire)
    bufferResult =
        taskExecution.getResults(
            "out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(500, TimeUnit.MILLISECONDS));
    assertTrue(bufferResult.isBufferClosed());

    bufferResult =
        taskExecution.getResults(
            "out", 0, new DataSize(1, Unit.MEGABYTE), new Duration(500, TimeUnit.MILLISECONDS));
    assertTrue(bufferResult.isBufferClosed());
  }
Ejemplo n.º 3
0
  @Override
  public synchronized void setOutputBuffers(OutputBuffers newOutputBuffers) {
    if (getTaskInfo().getState().isDone()) {
      return;
    }

    if (newOutputBuffers.getVersion() > outputBuffers.get().getVersion()) {
      outputBuffers.set(newOutputBuffers);
      needsUpdate.set(true);
      scheduleUpdate();
    }
  }
Ejemplo n.º 4
0
  public HttpRemoteTask(
      Session session,
      TaskId taskId,
      String nodeId,
      URI location,
      PlanFragment planFragment,
      Multimap<PlanNodeId, Split> initialSplits,
      OutputBuffers outputBuffers,
      HttpClient httpClient,
      Executor executor,
      ScheduledExecutorService errorScheduledExecutor,
      Duration minErrorDuration,
      Duration refreshMaxWait,
      JsonCodec<TaskInfo> taskInfoCodec,
      JsonCodec<TaskUpdateRequest> taskUpdateRequestCodec,
      SplitCountChangeListener splitCountChangeListener) {
    requireNonNull(session, "session is null");
    requireNonNull(taskId, "taskId is null");
    requireNonNull(nodeId, "nodeId is null");
    requireNonNull(location, "location is null");
    requireNonNull(planFragment, "planFragment1 is null");
    requireNonNull(outputBuffers, "outputBuffers is null");
    requireNonNull(httpClient, "httpClient is null");
    requireNonNull(executor, "executor is null");
    requireNonNull(taskInfoCodec, "taskInfoCodec is null");
    requireNonNull(taskUpdateRequestCodec, "taskUpdateRequestCodec is null");
    requireNonNull(splitCountChangeListener, "splitCountChangeListener is null");

    try (SetThreadName ignored = new SetThreadName("HttpRemoteTask-%s", taskId)) {
      this.taskId = taskId;
      this.session = session;
      this.nodeId = nodeId;
      this.planFragment = planFragment;
      this.outputBuffers.set(outputBuffers);
      this.httpClient = httpClient;
      this.executor = executor;
      this.errorScheduledExecutor = errorScheduledExecutor;
      this.taskInfoCodec = taskInfoCodec;
      this.taskUpdateRequestCodec = taskUpdateRequestCodec;
      this.updateErrorTracker =
          new RequestErrorTracker(
              taskId, location, minErrorDuration, errorScheduledExecutor, "updating task");
      this.getErrorTracker =
          new RequestErrorTracker(
              taskId, location, minErrorDuration, errorScheduledExecutor, "getting info for task");
      this.splitCountChangeListener = splitCountChangeListener;

      for (Entry<PlanNodeId, Split> entry :
          requireNonNull(initialSplits, "initialSplits is null").entries()) {
        ScheduledSplit scheduledSplit =
            new ScheduledSplit(nextSplitId.getAndIncrement(), entry.getValue());
        pendingSplits.put(entry.getKey(), scheduledSplit);
      }
      if (initialSplits.containsKey(planFragment.getPartitionedSource())) {
        pendingSourceSplitCount = initialSplits.get(planFragment.getPartitionedSource()).size();
        fireSplitCountChanged(pendingSourceSplitCount);
      }

      List<BufferInfo> bufferStates =
          outputBuffers
              .getBuffers()
              .keySet()
              .stream()
              .map(outputId -> new BufferInfo(outputId, false, 0, 0, PageBufferInfo.empty()))
              .collect(toImmutableList());

      TaskStats taskStats = new TaskStats(DateTime.now(), null);

      taskInfo =
          new StateMachine<>(
              "task " + taskId,
              executor,
              new TaskInfo(
                  taskId,
                  Optional.empty(),
                  TaskInfo.MIN_VERSION,
                  TaskState.PLANNED,
                  location,
                  DateTime.now(),
                  new SharedBufferInfo(BufferState.OPEN, true, true, 0, 0, 0, 0, bufferStates),
                  ImmutableSet.<PlanNodeId>of(),
                  taskStats,
                  ImmutableList.<ExecutionFailureInfo>of()));

      continuousTaskInfoFetcher = new ContinuousTaskInfoFetcher(refreshMaxWait);
    }
  }