예제 #1
0
파일: TestDriver.java 프로젝트: cdma/presto
  @Test
  public void testAbruptFinish() {
    List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    ValuesOperator source =
        new ValuesOperator(
            driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"),
            types,
            rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());

    PageConsumerOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    driver.close();
    assertTrue(driver.isFinished());

    // finish is only called in normal operations
    assertFalse(source.isFinished());
    assertFalse(sink.isFinished());

    // close is always called (values operator doesn't have a closed state)
    assertTrue(sink.isClosed());
  }
예제 #2
0
파일: TestDriver.java 프로젝트: cdma/presto
  @Test
  public void testBrokenOperatorProcessWhileClosing() throws Exception {
    BrokenOperator brokenOperator =
        new BrokenOperator(
            driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), true);
    final Driver driver =
        new Driver(driverContext, brokenOperator, createSinkOperator(brokenOperator));

    assertSame(driver.getDriverContext(), driverContext);

    // block thread in operator close
    Future<Boolean> driverClose =
        executor.submit(
            new Callable<Boolean>() {
              @Override
              public Boolean call() throws Exception {
                driver.close();
                return true;
              }
            });
    brokenOperator.waitForLocked();

    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertTrue(driver.isFinished());

    brokenOperator.unlock();

    assertTrue(driverClose.get());
  }
예제 #3
0
파일: TestDriver.java 프로젝트: cdma/presto
  @Test
  public void testBrokenOperatorCloseWhileProcessing() throws Exception {
    BrokenOperator brokenOperator =
        new BrokenOperator(
            driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), false);
    final Driver driver =
        new Driver(driverContext, brokenOperator, createSinkOperator(brokenOperator));

    assertSame(driver.getDriverContext(), driverContext);

    // block thread in operator processing
    Future<Boolean> driverProcessFor =
        executor.submit(
            new Callable<Boolean>() {
              @Override
              public Boolean call() throws Exception {
                return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
              }
            });
    brokenOperator.waitForLocked();

    driver.close();
    assertTrue(driver.isFinished());

    try {
      driverProcessFor.get(1, TimeUnit.SECONDS);
      fail("Expected InterruptedException");
    } catch (ExecutionException e) {
      checkArgument(
          getRootCause(e) instanceof InterruptedException,
          "Expected root cause exception to be an instance of InterruptedException");
    }
  }
예제 #4
0
  @Override
  protected List<Driver> createDrivers(TaskContext taskContext) {
    if (lookupSourceSupplier == null) {
      OperatorFactory ordersTableScan =
          createTableScanOperator(0, "orders", "orderkey", "totalprice");
      HashBuilderOperatorFactory hashBuilder =
          new HashBuilderOperatorFactory(1, ordersTableScan.getTypes(), Ints.asList(0), 1_500_000);

      DriverContext driverContext = taskContext.addPipelineContext(false, false).addDriverContext();
      Driver driver =
          new DriverFactory(false, false, ordersTableScan, hashBuilder).createDriver(driverContext);
      while (!driver.isFinished()) {
        driver.process();
      }
      lookupSourceSupplier = hashBuilder.getLookupSourceSupplier();
    }

    OperatorFactory lineItemTableScan =
        createTableScanOperator(0, "lineitem", "orderkey", "quantity");

    OperatorFactory joinOperator =
        LookupJoinOperators.innerJoin(
            1, lookupSourceSupplier, lineItemTableScan.getTypes(), Ints.asList(0));

    NullOutputOperatorFactory output = new NullOutputOperatorFactory(2, joinOperator.getTypes());

    DriverFactory driverFactory =
        new DriverFactory(true, true, lineItemTableScan, joinOperator, output);
    DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext();
    Driver driver = driverFactory.createDriver(driverContext);
    return ImmutableList.of(driver);
  }
예제 #5
0
  @Override
  public MaterializedResult execute(Session session, @Language("SQL") String sql) {
    lock.readLock().lock();
    try {
      MaterializedOutputFactory outputFactory = new MaterializedOutputFactory();

      TaskContext taskContext = createTaskContext(executor, session);
      List<Driver> drivers = createDrivers(session, sql, outputFactory, taskContext);

      boolean done = false;
      while (!done) {
        boolean processed = false;
        for (Driver driver : drivers) {
          if (!driver.isFinished()) {
            driver.process();
            processed = true;
          }
        }
        done = !processed;
      }

      return outputFactory.getMaterializingOperator().getMaterializedResult();
    } finally {
      lock.readLock().unlock();
    }
  }
예제 #6
0
  @Test
  public void testAddSourceFinish() {
    PlanNodeId sourceId = new PlanNodeId("source");
    TableScanOperator source =
        new TableScanOperator(
            driverContext.addOperatorContext(99, "values"),
            sourceId,
            new DataStreamProvider() {
              @Override
              public Operator createNewDataStream(
                  OperatorContext operatorContext, Split split, List<ColumnHandle> columns) {
                return new ValuesOperator(
                    driverContext.addOperatorContext(0, "values"),
                    rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
                        .addSequencePage(10, 20, 30, 40)
                        .build());
              }
            },
            ImmutableList.of(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG),
            ImmutableList.<ColumnHandle>of());

    MaterializingOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    // todo TableScanOperator should be blocked until split is set
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.updateSource(
        new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, new MockSplit())), true));

    assertFalse(driver.isFinished());
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
예제 #7
0
    @Override
    public synchronized boolean isFinished() {
      if (closed) {
        return true;
      }

      if (driver == null) {
        return false;
      }

      return driver.isFinished();
    }
예제 #8
0
  @Test
  public void testAbruptFinish() {
    ValuesOperator source =
        new ValuesOperator(
            driverContext.addOperatorContext(0, "values"),
            rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
                .addSequencePage(10, 20, 30, 40)
                .build());

    MaterializingOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    driver.close();
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
예제 #9
0
파일: TestDriver.java 프로젝트: cdma/presto
  @Test
  public void testNormalFinish() {
    List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    ValuesOperator source =
        new ValuesOperator(
            driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"),
            types,
            rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());

    Operator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS));
    assertTrue(blocked.isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
예제 #10
0
파일: TestDriver.java 프로젝트: cdma/presto
  @Test
  public void testAddSourceFinish() {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    TableScanOperator source =
        new TableScanOperator(
            driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"),
            sourceId,
            new PageSourceProvider() {
              @Override
              public ConnectorPageSource createPageSource(
                  Session session, Split split, List<ColumnHandle> columns) {
                return new FixedPageSource(
                    rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());
              }
            },
            types,
            ImmutableList.<ColumnHandle>of());

    PageConsumerOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.updateSource(
        new TaskSource(
            sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));

    assertFalse(driver.isFinished());
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
예제 #11
0
  @Test
  public void testNormalFinish() {
    ValuesOperator source =
        new ValuesOperator(
            driverContext.addOperatorContext(0, "values"),
            rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
                .addSequencePage(10, 20, 30, 40)
                .build());

    MaterializingOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS));
    assertTrue(blocked.isDone());
    assertTrue(driver.isFinished());

    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
  }
예제 #12
0
  public MaterializedResult execute(@Language("SQL") String sql) {
    MaterializedOutputFactory outputFactory = new MaterializedOutputFactory();
    List<Driver> drivers = createDrivers(sql, outputFactory);

    boolean done = false;
    while (!done) {
      boolean processed = false;
      for (Driver driver : drivers) {
        if (!driver.isFinished()) {
          driver.process();
          processed = true;
        }
      }
      done = !processed;
    }

    return outputFactory.getMaterializingOperator().getMaterializedResult();
  }
예제 #13
0
파일: TestDriver.java 프로젝트: cdma/presto
  @Test
  public void testBrokenOperatorAddSource() throws Exception {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    // create a table scan operator that does not block, which will cause the driver loop to busy
    // wait
    TableScanOperator source =
        new NotBlockedTableScanOperator(
            driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"),
            sourceId,
            new PageSourceProvider() {
              @Override
              public ConnectorPageSource createPageSource(
                  Session session, Split split, List<ColumnHandle> columns) {
                return new FixedPageSource(
                    rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());
              }
            },
            types,
            ImmutableList.<ColumnHandle>of());

    BrokenOperator brokenOperator =
        new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
    final Driver driver = new Driver(driverContext, source, brokenOperator);

    // block thread in operator processing
    Future<Boolean> driverProcessFor =
        executor.submit(
            new Callable<Boolean>() {
              @Override
              public Boolean call() throws Exception {
                return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
              }
            });
    brokenOperator.waitForLocked();

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.updateSource(
        new TaskSource(
            sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));

    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.close();
    assertTrue(driver.isFinished());

    try {
      driverProcessFor.get(1, TimeUnit.SECONDS);
      fail("Expected InterruptedException");
    } catch (ExecutionException e) {
      checkArgument(
          getRootCause(e) instanceof InterruptedException,
          "Expected root cause exception to be an instance of InterruptedException");
    }
  }
예제 #14
0
 @Override
 public synchronized boolean isFinished() {
   return driver.isFinished();
 }
예제 #15
0
  private static LookupSourceSupplier buildHash(
      boolean parallelBuild,
      TaskContext taskContext,
      List<Integer> hashChannels,
      RowPagesBuilder buildPages) {
    if (parallelBuild) {
      ParallelHashBuilder parallelHashBuilder =
          new ParallelHashBuilder(
              buildPages.getTypes(),
              hashChannels,
              buildPages.getHashChannel(),
              100,
              PARTITION_COUNT);

      // collect input data
      DriverContext collectDriverContext =
          taskContext.addPipelineContext(true, true).addDriverContext();
      ValuesOperatorFactory valuesOperatorFactory =
          new ValuesOperatorFactory(
              0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
      OperatorFactory collectOperatorFactory =
          parallelHashBuilder.getCollectOperatorFactory(1, new PlanNodeId("test"));
      Driver driver =
          new Driver(
              collectDriverContext,
              valuesOperatorFactory.createOperator(collectDriverContext),
              collectOperatorFactory.createOperator(collectDriverContext));

      while (!driver.isFinished()) {
        driver.process();
      }

      // build hash tables
      PipelineContext buildPipeline = taskContext.addPipelineContext(true, true);
      OperatorFactory buildOperatorFactory =
          parallelHashBuilder.getBuildOperatorFactory(new PlanNodeId("test"));
      for (int i = 0; i < PARTITION_COUNT; i++) {
        DriverContext buildDriverContext = buildPipeline.addDriverContext();
        Driver buildDriver =
            new Driver(buildDriverContext, buildOperatorFactory.createOperator(buildDriverContext));

        while (!buildDriver.isFinished()) {
          buildDriver.process();
        }
      }

      return parallelHashBuilder.getLookupSourceSupplier();
    } else {
      DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext();

      ValuesOperatorFactory valuesOperatorFactory =
          new ValuesOperatorFactory(
              0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
      HashBuilderOperatorFactory hashBuilderOperatorFactory =
          new HashBuilderOperatorFactory(
              1,
              new PlanNodeId("test"),
              buildPages.getTypes(),
              hashChannels,
              buildPages.getHashChannel(),
              100);

      Driver driver =
          new Driver(
              driverContext,
              valuesOperatorFactory.createOperator(driverContext),
              hashBuilderOperatorFactory.createOperator(driverContext));

      while (!driver.isFinished()) {
        driver.process();
      }
      return hashBuilderOperatorFactory.getLookupSourceSupplier();
    }
  }
예제 #16
0
  @Test
  public void testBrokenOperatorAddSource() throws Exception {
    PlanNodeId sourceId = new PlanNodeId("source");
    TableScanOperator source =
        new TableScanOperator(
            driverContext.addOperatorContext(99, "values"),
            sourceId,
            new DataStreamProvider() {
              @Override
              public Operator createNewDataStream(
                  OperatorContext operatorContext, Split split, List<ColumnHandle> columns) {
                return new ValuesOperator(
                    driverContext.addOperatorContext(0, "values"),
                    rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
                        .addSequencePage(10, 20, 30, 40)
                        .build());
              }
            },
            ImmutableList.of(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG),
            ImmutableList.<ColumnHandle>of());

    BrokenOperator brokenOperator =
        new BrokenOperator(driverContext.addOperatorContext(0, "source"));
    final Driver driver = new Driver(driverContext, source, brokenOperator);

    // block thread in operator processing
    Future<Boolean> driverProcessFor =
        executor.submit(
            new Callable<Boolean>() {
              @Override
              public Boolean call() throws Exception {
                return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
              }
            });
    brokenOperator.waitForLocked();

    assertSame(driver.getDriverContext(), driverContext);

    assertFalse(driver.isFinished());
    // todo TableScanOperator should be blocked until split is set
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.updateSource(
        new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, new MockSplit())), true));

    assertFalse(driver.isFinished());
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertFalse(driver.isFinished());

    driver.close();
    assertTrue(driver.isFinished());

    try {
      driverProcessFor.get(1, TimeUnit.SECONDS);
      fail("Expected InterruptedException");
    } catch (ExecutionException e) {
      checkArgument(
          getRootCause(e) instanceof InterruptedException,
          "Expected root cause exception to be an instance of InterruptedException");
    }
  }