Exemplo n.º 1
0
  @Test
  public void testBrokenOperatorProcessWhileClosing() throws Exception {
    BrokenOperator brokenOperator =
        new BrokenOperator(
            driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), true);
    final Driver driver =
        new Driver(driverContext, brokenOperator, createSinkOperator(brokenOperator));

    assertSame(driver.getDriverContext(), driverContext);

    // block thread in operator close
    Future<Boolean> driverClose =
        executor.submit(
            new Callable<Boolean>() {
              @Override
              public Boolean call() throws Exception {
                driver.close();
                return true;
              }
            });
    brokenOperator.waitForLocked();

    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertTrue(driver.isFinished());

    brokenOperator.unlock();

    assertTrue(driverClose.get());
  }
Exemplo n.º 2
0
    @Override
    public synchronized ListenableFuture<?> processFor(Duration duration) {
      if (driver == null) {
        driver = driverSplitRunnerFactory.createDriver(partitionedSplit);
      }

      return driver.processFor(duration);
    }
Exemplo n.º 3
0
  @Test
  public void testAddSourceFinish() {
    PlanNodeId sourceId = new PlanNodeId("source");
    TableScanOperator source =
        new TableScanOperator(
            driverContext.addOperatorContext(99, "values"),
            sourceId,
            new DataStreamProvider() {
              @Override
              public Operator createNewDataStream(
                  OperatorContext operatorContext, Split split, List<ColumnHandle> columns) {
                return new ValuesOperator(
                    driverContext.addOperatorContext(0, "values"),
                    rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
                        .addSequencePage(10, 20, 30, 40)
                        .build());
              }
            },
            ImmutableList.of(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG),
            ImmutableList.<ColumnHandle>of());

    MaterializingOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    // todo TableScanOperator should be blocked until split is set
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.updateSource(
        new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, new MockSplit())), true));

    assertFalse(driver.isFinished());
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
Exemplo n.º 4
0
  @Test
  public void testAddSourceFinish() {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    TableScanOperator source =
        new TableScanOperator(
            driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"),
            sourceId,
            new PageSourceProvider() {
              @Override
              public ConnectorPageSource createPageSource(
                  Session session, Split split, List<ColumnHandle> columns) {
                return new FixedPageSource(
                    rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());
              }
            },
            types,
            ImmutableList.<ColumnHandle>of());

    PageConsumerOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.updateSource(
        new TaskSource(
            sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));

    assertFalse(driver.isFinished());
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
Exemplo n.º 5
0
    @Override
    public ListenableFuture<?> processFor(Duration duration) {
      Driver driver;
      synchronized (this) {
        // if close() was called before we get here, there's not point in even creating the driver
        if (closed) {
          return Futures.immediateFuture(null);
        }

        if (this.driver == null) {
          this.driver = driverSplitRunnerFactory.createDriver(driverContext, partitionedSplit);
        }

        driver = this.driver;
      }

      return driver.processFor(duration);
    }
Exemplo n.º 6
0
  @Test
  public void testNormalFinish() {
    List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    ValuesOperator source =
        new ValuesOperator(
            driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"),
            types,
            rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());

    Operator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS));
    assertTrue(blocked.isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
Exemplo n.º 7
0
  @Test
  public void testNormalFinish() {
    ValuesOperator source =
        new ValuesOperator(
            driverContext.addOperatorContext(0, "values"),
            rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
                .addSequencePage(10, 20, 30, 40)
                .build());

    MaterializingOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS));
    assertTrue(blocked.isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
Exemplo n.º 8
0
  @Test
  public void testBrokenOperatorAddSource() throws Exception {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    // create a table scan operator that does not block, which will cause the driver loop to busy
    // wait
    TableScanOperator source =
        new NotBlockedTableScanOperator(
            driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"),
            sourceId,
            new PageSourceProvider() {
              @Override
              public ConnectorPageSource createPageSource(
                  Session session, Split split, List<ColumnHandle> columns) {
                return new FixedPageSource(
                    rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());
              }
            },
            types,
            ImmutableList.<ColumnHandle>of());

    BrokenOperator brokenOperator =
        new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
    final Driver driver = new Driver(driverContext, source, brokenOperator);

    // block thread in operator processing
    Future<Boolean> driverProcessFor =
        executor.submit(
            new Callable<Boolean>() {
              @Override
              public Boolean call() throws Exception {
                return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
              }
            });
    brokenOperator.waitForLocked();

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.updateSource(
        new TaskSource(
            sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));

    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.close();
    assertTrue(driver.isFinished());

    try {
      driverProcessFor.get(1, TimeUnit.SECONDS);
      fail("Expected InterruptedException");
    } catch (ExecutionException e) {
      checkArgument(
          getRootCause(e) instanceof InterruptedException,
          "Expected root cause exception to be an instance of InterruptedException");
    }
  }
Exemplo n.º 9
0
  @Test
  public void testBrokenOperatorAddSource() throws Exception {
    PlanNodeId sourceId = new PlanNodeId("source");
    TableScanOperator source =
        new TableScanOperator(
            driverContext.addOperatorContext(99, "values"),
            sourceId,
            new DataStreamProvider() {
              @Override
              public Operator createNewDataStream(
                  OperatorContext operatorContext, Split split, List<ColumnHandle> columns) {
                return new ValuesOperator(
                    driverContext.addOperatorContext(0, "values"),
                    rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
                        .addSequencePage(10, 20, 30, 40)
                        .build());
              }
            },
            ImmutableList.of(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG),
            ImmutableList.<ColumnHandle>of());

    BrokenOperator brokenOperator =
        new BrokenOperator(driverContext.addOperatorContext(0, "source"));
    final Driver driver = new Driver(driverContext, source, brokenOperator);

    // block thread in operator processing
    Future<Boolean> driverProcessFor =
        executor.submit(
            new Callable<Boolean>() {
              @Override
              public Boolean call() throws Exception {
                return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
              }
            });
    brokenOperator.waitForLocked();

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    // todo TableScanOperator should be blocked until split is set
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.updateSource(
        new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, new MockSplit())), true));

    assertFalse(driver.isFinished());
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.close();
    assertTrue(driver.isFinished());

    try {
      driverProcessFor.get(1, TimeUnit.SECONDS);
      fail("Expected InterruptedException");
    } catch (ExecutionException e) {
      checkArgument(
          getRootCause(e) instanceof InterruptedException,
          "Expected root cause exception to be an instance of InterruptedException");
    }
  }