Exemplo n.º 1
0
  @Test
  public void testBufferCloseOnFinish() throws Exception {
    SqlTask sqlTask = createInitialTask();

    OutputBuffers outputBuffers =
        INITIAL_EMPTY_OUTPUT_BUFFERS
            .withBuffer("out", new UnpartitionedPagePartitionFunction())
            .withNoMoreBufferIds();
    updateTask(sqlTask, EMPTY_SOURCES, outputBuffers);

    ListenableFuture<BufferResult> bufferResult =
        sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
    assertFalse(bufferResult.isDone());

    // finish the task by closing the sources (no splits will ever be added)
    updateTask(
        sqlTask,
        ImmutableList.of(
            new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.<ScheduledSplit>of(), true)),
        outputBuffers);
    assertEquals(sqlTask.getTaskInfo().getState(), TaskState.FINISHED);

    // buffer will be closed by cancel event (wait for 500 MS for event to fire)
    assertTrue(bufferResult.get(200, MILLISECONDS).isBufferClosed());
    assertEquals(sqlTask.getTaskInfo().getOutputBuffers().getState(), BufferState.FINISHED);

    // verify the buffer is closed
    bufferResult = sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
    assertTrue(bufferResult.isDone());
    assertTrue(bufferResult.get().isBufferClosed());
  }
Exemplo n.º 2
0
 @Test
 public void pingPong() throws Exception {
   connect();
   Utils.rollMockClock(0);
   // No ping pong happened yet.
   assertEquals(Long.MAX_VALUE, peer.getLastPingTime());
   assertEquals(Long.MAX_VALUE, peer.getPingTime());
   ListenableFuture<Long> future = peer.ping();
   assertEquals(Long.MAX_VALUE, peer.getLastPingTime());
   assertEquals(Long.MAX_VALUE, peer.getPingTime());
   assertFalse(future.isDone());
   Ping pingMsg = (Ping) outbound(writeTarget);
   Utils.rollMockClock(5);
   // The pong is returned.
   inbound(writeTarget, new Pong(pingMsg.getNonce()));
   pingAndWait(writeTarget);
   assertTrue(future.isDone());
   long elapsed = future.get();
   assertTrue("" + elapsed, elapsed > 1000);
   assertEquals(elapsed, peer.getLastPingTime());
   assertEquals(elapsed, peer.getPingTime());
   // Do it again and make sure it affects the average.
   future = peer.ping();
   pingMsg = (Ping) outbound(writeTarget);
   Utils.rollMockClock(50);
   inbound(writeTarget, new Pong(pingMsg.getNonce()));
   elapsed = future.get();
   assertEquals(elapsed, peer.getLastPingTime());
   assertEquals(7250, peer.getPingTime());
 }
 @Test
 public void fourPeers() throws Exception {
   InboundMessageQueuer[] channels = {
     connectPeer(1), connectPeer(2), connectPeer(3), connectPeer(4)
   };
   Transaction tx = new Transaction(params);
   TransactionBroadcast broadcast = new TransactionBroadcast(peerGroup, tx);
   ListenableFuture<Transaction> future = broadcast.broadcast();
   assertFalse(future.isDone());
   // We expect two peers to receive a tx message, and at least one of the others must announce for
   // the future to
   // complete successfully.
   Message[] messages = {
     (Message) outbound(channels[0]),
     (Message) outbound(channels[1]),
     (Message) outbound(channels[2]),
     (Message) outbound(channels[3])
   };
   // 0 and 3 are randomly selected to receive the broadcast.
   assertEquals(tx, messages[0]);
   assertEquals(tx, messages[3]);
   assertNull(messages[1]);
   assertNull(messages[2]);
   Threading.waitForUserCode();
   assertFalse(future.isDone());
   inbound(channels[1], InventoryMessage.with(tx));
   pingAndWait(channels[1]);
   Threading.waitForUserCode();
   assertTrue(future.isDone());
 }
Exemplo n.º 4
0
    public void checkAsyncCallReturnValues() throws ExecutionException, InterruptedException {
      // All these async calls include a delay, so none she be finished at first
      assertFalse(getBeforeFuture.isDone());
      assertFalse(getAfterFuture.isDone());
      assertFalse(putFuture.isDone());

      // Calls are timed to complete in order, but Verify that we can still wait on the
      // futures out of order
      assertEquals(getBeforeFuture.get(), "default");
      assertEquals(getAfterFuture.get(), "testValue");
      putFuture.get();
    }
  private void cancelFuture(final NamespaceImplData implDatum) {
    synchronized (implDatum.enabled) {
      final CountDownLatch latch = new CountDownLatch(1);
      final ListenableFuture<?> future = implDatum.future;
      Futures.addCallback(
          future,
          new FutureCallback<Object>() {
            @Override
            public void onSuccess(Object result) {
              latch.countDown();
            }

            @Override
            public void onFailure(Throwable t) {
              // Expect CancellationException
              latch.countDown();
              if (!(t instanceof CancellationException)) {
                log.error(t, "Error in namespace [%s]", implDatum.name);
              }
            }
          });
      if (!future.isDone()
          && !future.cancel(
              true)) { // Interrupt to make sure we don't pollute stuff after we've already cleaned
                       // up
        throw new ISE("Future for namespace [%s] was not able to be canceled", implDatum.name);
      }
      try {
        latch.await();
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw Throwables.propagate(e);
      }
    }
  }
Exemplo n.º 6
0
    private synchronized void scheduleNextRequest() {
      // stopped or done?
      TaskInfo taskInfo = HttpRemoteTask.this.taskInfo.get();
      if (!running || taskInfo.getState().isDone()) {
        return;
      }

      // outstanding request?
      if (future != null && !future.isDone()) {
        // this should never happen
        log.error("Can not reschedule update because an update is already running");
        return;
      }

      // if throttled due to error, asynchronously wait for timeout and try again
      ListenableFuture<?> errorRateLimit = getErrorTracker.acquireRequestPermit();
      if (!errorRateLimit.isDone()) {
        errorRateLimit.addListener(this::scheduleNextRequest, executor);
        return;
      }

      Request request =
          prepareGet()
              .setUri(uriBuilderFrom(taskInfo.getSelf()).addParameter("summarize").build())
              .setHeader(HttpHeaders.CONTENT_TYPE, MediaType.JSON_UTF_8.toString())
              .setHeader(PrestoHeaders.PRESTO_CURRENT_STATE, taskInfo.getState().toString())
              .setHeader(PrestoHeaders.PRESTO_MAX_WAIT, refreshMaxWait.toString())
              .build();

      getErrorTracker.startRequest();

      future = httpClient.executeAsync(request, createFullJsonResponseHandler(taskInfoCodec));
      Futures.addCallback(
          future, new SimpleHttpResponseHandler<>(this, request.getUri()), executor);
    }
Exemplo n.º 7
0
  @Test
  public void testBufferNotCloseOnFail() throws Exception {
    SqlTask sqlTask = createInitialTask();

    updateTask(
        sqlTask,
        EMPTY_SOURCES,
        INITIAL_EMPTY_OUTPUT_BUFFERS.withBuffer("out", new UnpartitionedPagePartitionFunction()));

    ListenableFuture<BufferResult> bufferResult =
        sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
    assertFalse(bufferResult.isDone());

    TaskState taskState = sqlTask.getTaskInfo().getState();
    sqlTask.failed(new Exception("test"));
    assertEquals(
        sqlTask.getTaskInfo(taskState).get(200, MILLISECONDS).getState(), TaskState.FAILED);

    // buffer will not be closed by fail event.  event is async so wait a bit for event to fire
    try {
      assertTrue(bufferResult.get(200, MILLISECONDS).isBufferClosed());
      fail("expected TimeoutException");
    } catch (TimeoutException expected) {
    }
    assertFalse(sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE)).isDone());
  }
  @Test(enabled = false)
  public void weCanCancelTasks(NodeMetadata node) throws InterruptedException, ExecutionException {
    ListenableFuture<ExecResponse> future =
        client.submitScriptOnNode(node.getId(), "sleep 300", nameTask("sleeper").runAsRoot(false));
    ExecResponse response = null;
    try {
      response = future.get(1, TimeUnit.MILLISECONDS);
      fail(node.getId() + ": " + response);
    } catch (TimeoutException e) {
      assert !future.isDone();
      response =
          client.runScriptOnNode(
              node.getId(), "/tmp/init-sleeper status", wrapInInitScript(false).runAsRoot(false));
      assert !response.getOutput().trim().equals("") : node.getId() + ": " + response;
      future.cancel(true);
      response =
          client.runScriptOnNode(
              node.getId(), "/tmp/init-sleeper status", wrapInInitScript(false).runAsRoot(false));
      assert response.getOutput().trim().equals("") : node.getId() + ": " + response;
      try {
        future.get();
        fail(future.toString());
      } catch (CancellationException e1) {

      }
    }
  }
 @Override
 public ListenableFuture<?> isBlocked() {
   if (blocked != NOT_BLOCKED && blocked.isDone()) {
     blocked = NOT_BLOCKED;
   }
   return blocked;
 }
Exemplo n.º 10
0
 @Override
 public ListenableFuture<?> isBlocked() {
   ListenableFuture<?> blocked = exchangeClient.isBlocked();
   if (blocked.isDone()) {
     return NOT_BLOCKED;
   }
   return blocked;
 }
 @Override
 public ListenableFuture<?> isBlocked() {
   ListenableFuture<?> blocked = inMemoryExchange.waitForWriting();
   if (blocked.isDone()) {
     return NOT_BLOCKED;
   }
   return blocked;
 }
Exemplo n.º 12
0
  private synchronized void scheduleUpdate() {
    // don't update if the task hasn't been started yet or if it is already finished
    if (!needsUpdate.get() || taskInfo.get().getState().isDone()) {
      return;
    }

    // if we have an old request outstanding, cancel it
    if (currentRequest != null
        && Duration.nanosSince(currentRequestStartNanos).compareTo(new Duration(2, SECONDS)) >= 0) {
      needsUpdate.set(true);
      currentRequest.cancel(true);
      currentRequest = null;
      currentRequestStartNanos = 0;
    }

    // if there is a request already running, wait for it to complete
    if (this.currentRequest != null && !this.currentRequest.isDone()) {
      return;
    }

    // if throttled due to error, asynchronously wait for timeout and try again
    ListenableFuture<?> errorRateLimit = updateErrorTracker.acquireRequestPermit();
    if (!errorRateLimit.isDone()) {
      errorRateLimit.addListener(this::scheduleUpdate, executor);
      return;
    }

    List<TaskSource> sources = getSources();
    TaskUpdateRequest updateRequest =
        new TaskUpdateRequest(
            session.toSessionRepresentation(), planFragment, sources, outputBuffers.get());

    Request request =
        preparePost()
            .setUri(uriBuilderFrom(taskInfo.get().getSelf()).addParameter("summarize").build())
            .setHeader(HttpHeaders.CONTENT_TYPE, MediaType.JSON_UTF_8.toString())
            .setBodyGenerator(jsonBodyGenerator(taskUpdateRequestCodec, updateRequest))
            .build();

    updateErrorTracker.startRequest();

    ListenableFuture<JsonResponse<TaskInfo>> future =
        httpClient.executeAsync(request, createFullJsonResponseHandler(taskInfoCodec));
    currentRequest = future;
    currentRequestStartNanos = System.nanoTime();

    // The needsUpdate flag needs to be set to false BEFORE adding the Future callback since
    // callback might change the flag value
    // and does so without grabbing the instance lock.
    needsUpdate.set(false);

    Futures.addCallback(
        future,
        new SimpleHttpResponseHandler<>(new UpdateResponseHandler(sources), request.getUri()),
        executor);
  }
Exemplo n.º 13
0
  @Test
  public void testFireOnceMajoritySuccess() {

    SettableFuture<Boolean> f1 = SettableFuture.create();
    SettableFuture<Boolean> f2 = SettableFuture.create();
    SettableFuture<Boolean> f3 = SettableFuture.create();

    List<SettableFuture<Boolean>> responses = Lists.newArrayList(f1, f2, f3);

    ListenableFuture<Boolean> collector = majorityResponse(responses, Identity);

    f1.set(Boolean.TRUE);
    assertFalse(collector.isDone());

    f2.set(Boolean.TRUE);
    assertTrue(collector.isDone());

    assertTrue(Futures.getUnchecked(collector));
  }
Exemplo n.º 14
0
  @Test
  public void testFireOnceMajorityFailed1() {

    SettableFuture<Boolean> f1 = SettableFuture.create();
    SettableFuture<Boolean> f2 = SettableFuture.create();

    List<ListenableFuture<Boolean>> responses =
        Lists.newArrayList(f1, f2, Futures.<Boolean>immediateFuture(Boolean.TRUE));

    ListenableFuture<Boolean> collector = majorityResponse(responses, Identity);

    f1.setException(new Exception());
    assertFalse(collector.isDone());

    f2.setException(new Exception());
    assertTrue(collector.isDone());

    assertFalse(Futures.getUnchecked(collector));
  }
Exemplo n.º 15
0
  @Test
  public void testBufferCloseOnCancel() throws Exception {
    SqlTask sqlTask = createInitialTask();

    updateTask(
        sqlTask,
        EMPTY_SOURCES,
        INITIAL_EMPTY_OUTPUT_BUFFERS.withBuffer("out", new UnpartitionedPagePartitionFunction()));

    ListenableFuture<BufferResult> bufferResult =
        sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
    assertFalse(bufferResult.isDone());

    sqlTask.cancel();
    assertEquals(sqlTask.getTaskInfo().getState(), TaskState.CANCELED);

    // buffer will be closed by cancel event.. the event is async so wait a bit for event to
    // propagate
    assertTrue(bufferResult.get(200, MILLISECONDS).isBufferClosed());

    bufferResult = sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
    assertTrue(bufferResult.isDone());
    assertTrue(bufferResult.get().isBufferClosed());
  }
Exemplo n.º 16
0
  public ListenableFuture<?> processFor(Duration duration) {
    checkLockNotHeld("Can not process for a duration while holding the driver lock");

    checkNotNull(duration, "duration is null");

    long maxRuntime = duration.roundTo(TimeUnit.NANOSECONDS);

    long start = System.nanoTime();
    do {
      ListenableFuture<?> future = process();
      if (!future.isDone()) {
        return future;
      }
    } while (System.nanoTime() - start < maxRuntime && !isFinished());

    return NOT_BLOCKED;
  }
Exemplo n.º 17
0
  @Test
  public void testEmptyPath() throws Exception {
    ImmutableMap<String, List<FileStatus>> paths =
        ImmutableMap.<String, List<FileStatus>>builder()
            .put("/", ImmutableList.<FileStatus>of())
            .build();

    AsyncRecursiveWalker walker =
        new AsyncRecursiveWalker(createMockFileSystem(paths), MoreExecutors.sameThreadExecutor());

    MockFileStatusCallback callback = new MockFileStatusCallback();
    ListenableFuture<Void> listenableFuture = walker.beginWalk(new Path("/"), callback);

    Assert.assertTrue(listenableFuture.isDone());
    Assert.assertTrue(callback.getProcessedFiles().isEmpty());

    // Should not have an exception
    listenableFuture.get();
  }
    public ListenableFuture<?> flush(boolean force) {
      // add all full pages to output buffer
      List<ListenableFuture<?>> blockedFutures = new ArrayList<>();
      for (int partition = 0; partition < pageBuilders.size(); partition++) {
        PageBuilder partitionPageBuilder = pageBuilders.get(partition);
        if (!partitionPageBuilder.isEmpty() && (force || partitionPageBuilder.isFull())) {
          Page pagePartition = partitionPageBuilder.build();
          partitionPageBuilder.reset();

          blockedFutures.add(outputBuffer.enqueue(partition, pagePartition));
          pagesAdded.incrementAndGet();
          rowsAdded.addAndGet(pagePartition.getPositionCount());
        }
      }
      ListenableFuture<?> future = Futures.allAsList(blockedFutures);
      if (future.isDone()) {
        return NOT_BLOCKED;
      }
      return future;
    }
Exemplo n.º 19
0
  @Test
  public void
      loadBuildTypeList_callback_registers_project_to_BuildTypeManager_and_dispatch_it_on_event_bus()
          throws Exception {
    // Setup
    final BuildTypeList buildTypelist = new BuildTypeList();
    buildTypelist.addBuildType(new BuildType("bt1", "btName", "btProjectName", "btProjectId"));
    buildTypelist.addBuildType(new BuildType("bt2", "btName", "btProjectName", "btProjectId"));

    when(_mockRequestController.sendRequest(getApiVersion(), "buildTypes", BuildTypeList.class))
        .thenReturn(Futures.immediateFuture(buildTypelist));
    // Exercise
    final ListenableFuture<Void> ackFuture = _apiController.loadBuildTypeList();
    // Verify
    assertThat(_buildTypeManager.getBuildTypes().size(), is(2));
    assertThat(_buildTypeManager.getBuildTypes().get(0).getId(), is("bt1"));
    assertThat(_buildTypeManager.getBuildTypes().get(1).getId(), is("bt2"));
    assertThat(_dispatchedObjects, hasItem(_buildTypeManager));
    assertThat(ackFuture.isDone(), is(true));
  }
Exemplo n.º 20
0
  @Test
  public void
      loadProjectList_callback_registers_project_to_ProjectManager_and_dispatch_it_on_event_bus()
          throws Exception {
    // Setup
    final ProjectList projectList = new ProjectList();
    projectList.addProject(new Project("pId1", "pName", "pParentId"));
    projectList.addProject(new Project("pId2", "pName", "pParentId"));

    when(_mockRequestController.sendRequest(getApiVersion(), "projects", ProjectList.class))
        .thenReturn(Futures.immediateFuture(projectList));
    // Exercise
    final ListenableFuture<Void> ackFuture = _apiController.loadProjectList();
    // Verify
    assertThat(_projectManager.getProjects().size(), is(2));
    assertThat(_projectManager.getProjects().get(0).getId(), is("pId1"));
    assertThat(_projectManager.getProjects().get(1).getId(), is("pId2"));
    assertThat(_dispatchedObjects, hasItem(_projectManager));
    assertThat(ackFuture.isDone(), is(true));
  }
Exemplo n.º 21
0
  @Test
  public void testHiddenFiles() throws Exception {
    ImmutableMap<String, List<FileStatus>> paths =
        ImmutableMap.<String, List<FileStatus>>builder()
            .put(
                "/",
                ImmutableList.of(
                    fileStatus("/.a", true),
                    fileStatus("/_b", true),
                    fileStatus("/c", true),
                    fileStatus("/file1", false),
                    fileStatus("/_file2", false),
                    fileStatus("/.file3", false)))
            .put(
                "/.a",
                ImmutableList.of(fileStatus("/.a/file4", false), fileStatus("/.a/file5", false)))
            .put(
                "/_b",
                ImmutableList.of(fileStatus("/_b/file6", false), fileStatus("/_b/file7", false)))
            .put(
                "/c",
                ImmutableList.of(
                    fileStatus("/c/file8", false),
                    fileStatus("/c/.file9", false),
                    fileStatus("/c/_file10", false)))
            .build();

    AsyncRecursiveWalker walker =
        new AsyncRecursiveWalker(createMockFileSystem(paths), MoreExecutors.sameThreadExecutor());

    MockFileStatusCallback callback = new MockFileStatusCallback();
    ListenableFuture<Void> listenableFuture = walker.beginWalk(new Path("/"), callback);

    Assert.assertTrue(listenableFuture.isDone());
    Assert.assertEquals(
        ImmutableSet.copyOf(callback.getProcessedFiles()), ImmutableSet.of("/file1", "/c/file8"));

    // Should not have an exception
    listenableFuture.get();
  }
Exemplo n.º 22
0
  @Test
  public void testNormalFinish() {
    List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    ValuesOperator source =
        new ValuesOperator(
            driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"),
            types,
            rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());

    Operator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS));
    assertTrue(blocked.isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
Exemplo n.º 23
0
  @Test
  public void testNormalFinish() {
    ValuesOperator source =
        new ValuesOperator(
            driverContext.addOperatorContext(0, "values"),
            rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
                .addSequencePage(10, 20, 30, 40)
                .build());

    MaterializingOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS));
    assertTrue(blocked.isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
  // For testing purposes this is protected
  protected <T extends ExtractionNamespace> ListenableFuture<?> schedule(
      final String id,
      final T namespace,
      final ExtractionNamespaceCacheFactory<T> factory,
      final Runnable postRunnable,
      final String cacheId) {
    log.debug("Trying to update namespace [%s]", id);
    final NamespaceImplData implDatum = implData.get(id);
    if (implDatum != null) {
      synchronized (implDatum.enabled) {
        if (implDatum.enabled.get()) {
          // We also check at the end of the function, but fail fast here
          throw new IAE(
              "Namespace [%s] already exists! Leaving prior running", namespace.toString());
        }
      }
    }
    final long updateMs = namespace.getPollMs();
    final CountDownLatch startLatch = new CountDownLatch(1);

    final Runnable command =
        new Runnable() {
          @Override
          public void run() {
            try {
              startLatch.await(); // wait for "election" to leadership or cancellation
              if (!Thread.currentThread().isInterrupted()) {
                final Map<String, String> cache = getCacheMap(cacheId);
                final String preVersion = lastVersion.get(id);
                final Callable<String> runnable =
                    factory.getCachePopulator(id, namespace, preVersion, cache);

                tasksStarted.incrementAndGet();
                final String newVersion = runnable.call();
                if (preVersion != null && preVersion.equals(newVersion)) {
                  throw new CancellationException(
                      String.format("Version `%s` already exists", preVersion));
                }
                if (newVersion != null) {
                  lastVersion.put(id, newVersion);
                }
                postRunnable.run();
                log.debug("Namespace [%s] successfully updated", id);
              }
            } catch (Throwable t) {
              delete(cacheId);
              if (t instanceof CancellationException) {
                log.debug(t, "Namespace [%s] cancelled", id);
              } else {
                log.error(t, "Failed update namespace [%s]", namespace);
              }
              if (Thread.currentThread().isInterrupted()) {
                throw Throwables.propagate(t);
              }
            }
          }
        };

    ListenableFuture<?> future;
    try {
      if (updateMs > 0) {
        future =
            listeningScheduledExecutorService.scheduleAtFixedRate(
                command, 0, updateMs, TimeUnit.MILLISECONDS);
      } else {
        future = listeningScheduledExecutorService.schedule(command, 0, TimeUnit.MILLISECONDS);
      }

      final NamespaceImplData me = new NamespaceImplData(future, namespace, id);
      final NamespaceImplData other = implData.putIfAbsent(id, me);
      if (other != null) {
        if (!future.isDone() && !future.cancel(true)) {
          log.warn("Unable to cancel future for namespace[%s] on race loss", id);
        }
        throw new IAE("Namespace [%s] already exists! Leaving prior running", namespace);
      } else {
        if (!me.enabled.compareAndSet(false, true)) {
          log.wtf("How did someone enable this before ME?");
        }
        log.debug("I own namespace [%s]", id);
        return future;
      }
    } finally {
      startLatch.countDown();
    }
  }
Exemplo n.º 25
0
    @Override
    public void run() {
      try (SetThreadName runnerName = new SetThreadName("SplitRunner-%s", runnerId)) {
        while (!closed && !Thread.currentThread().isInterrupted()) {
          // select next worker
          final PrioritizedSplitRunner split;
          try {
            split = pendingSplits.take();
            if (split.updatePriorityLevel()) {
              // priority level changed, return split to queue for re-prioritization
              pendingSplits.put(split);
              continue;
            }
          } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            return;
          }

          try (SetThreadName splitName = new SetThreadName(split.toString())) {
            runningSplits.add(split);

            boolean finished;
            ListenableFuture<?> blocked;
            try {
              split.initializeIfNecessary();
              blocked = split.process();
              finished = split.isFinished();
            } finally {
              runningSplits.remove(split);
            }

            if (finished) {
              log.debug("%s is finished", split);
              splitFinished(split);
            } else {
              if (blocked.isDone()) {
                pendingSplits.put(split);
              } else {
                blockedSplits.add(split);
                blocked.addListener(
                    new Runnable() {
                      @Override
                      public void run() {
                        blockedSplits.remove(split);
                        split.updatePriorityLevel();
                        pendingSplits.put(split);
                      }
                    },
                    executor);
              }
            }
          } catch (Throwable t) {
            log.error(t, "Error processing %s", split);
            splitFinished(split);
          }
        }
      } finally {
        // unless we have been closed, we need to replace this thread
        if (!closed) {
          addRunnerThread();
        }
      }
    }
Exemplo n.º 26
0
  public ListenableFuture<?> process() {
    checkLockNotHeld("Can not process while holding the driver lock");

    try (DriverLockResult lockResult =
        tryLockAndProcessPendingStateChanges(100, TimeUnit.MILLISECONDS)) {
      try {
        if (!lockResult.wasAcquired()) {
          // this is unlikely to happen unless the driver is being
          // destroyed and in that case the caller should notice notice
          // this state change by calling isFinished
          return NOT_BLOCKED;
        }

        driverContext.start();

        if (!newSources.isEmpty()) {
          processNewSources();
        }

        for (int i = 0; i < operators.size() - 1 && !driverContext.isDone(); i++) {
          // check if current operator is blocked
          Operator current = operators.get(i);
          ListenableFuture<?> blocked = current.isBlocked();
          if (!blocked.isDone()) {
            current.getOperatorContext().recordBlocked(blocked);
            return blocked;
          }

          // check if next operator is blocked
          Operator next = operators.get(i + 1);
          blocked = next.isBlocked();
          if (!blocked.isDone()) {
            next.getOperatorContext().recordBlocked(blocked);
            return blocked;
          }

          // if current operator is finished...
          if (current.isFinished()) {
            // let next operator know there will be no more data
            next.getOperatorContext().startIntervalTimer();
            next.finish();
            next.getOperatorContext().recordFinish();
          } else {
            // if next operator needs input...
            if (next.needsInput()) {
              // get an output page from current operator
              current.getOperatorContext().startIntervalTimer();
              Page page = current.getOutput();
              current.getOperatorContext().recordGetOutput(page);

              // if we got an output page, add it to the next operator
              if (page != null) {
                next.getOperatorContext().startIntervalTimer();
                next.addInput(page);
                next.getOperatorContext().recordAddInput(page);
              }
            }
          }
        }
        return NOT_BLOCKED;
      } catch (Throwable t) {
        driverContext.failed(t);
        throw t;
      }
    }
  }