@Test public void testAbruptFinish() { List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT); ValuesOperator source = new ValuesOperator( driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), types, rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()); PageConsumerOperator sink = createSinkOperator(source); Driver driver = new Driver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); driver.close(); assertTrue(driver.isFinished()); // finish is only called in normal operations assertFalse(source.isFinished()); assertFalse(sink.isFinished()); // close is always called (values operator doesn't have a closed state) assertTrue(sink.isClosed()); }
@Test public void testBrokenOperatorProcessWhileClosing() throws Exception { BrokenOperator brokenOperator = new BrokenOperator( driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), true); final Driver driver = new Driver(driverContext, brokenOperator, createSinkOperator(brokenOperator)); assertSame(driver.getDriverContext(), driverContext); // block thread in operator close Future<Boolean> driverClose = executor.submit( new Callable<Boolean>() { @Override public Boolean call() throws Exception { driver.close(); return true; } }); brokenOperator.waitForLocked(); assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertTrue(driver.isFinished()); brokenOperator.unlock(); assertTrue(driverClose.get()); }
@Test public void testBrokenOperatorCloseWhileProcessing() throws Exception { BrokenOperator brokenOperator = new BrokenOperator( driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), false); final Driver driver = new Driver(driverContext, brokenOperator, createSinkOperator(brokenOperator)); assertSame(driver.getDriverContext(), driverContext); // block thread in operator processing Future<Boolean> driverProcessFor = executor.submit( new Callable<Boolean>() { @Override public Boolean call() throws Exception { return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone(); } }); brokenOperator.waitForLocked(); driver.close(); assertTrue(driver.isFinished()); try { driverProcessFor.get(1, TimeUnit.SECONDS); fail("Expected InterruptedException"); } catch (ExecutionException e) { checkArgument( getRootCause(e) instanceof InterruptedException, "Expected root cause exception to be an instance of InterruptedException"); } }
@Override protected List<Driver> createDrivers(TaskContext taskContext) { if (lookupSourceSupplier == null) { OperatorFactory ordersTableScan = createTableScanOperator(0, "orders", "orderkey", "totalprice"); HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, ordersTableScan.getTypes(), Ints.asList(0), 1_500_000); DriverContext driverContext = taskContext.addPipelineContext(false, false).addDriverContext(); Driver driver = new DriverFactory(false, false, ordersTableScan, hashBuilder).createDriver(driverContext); while (!driver.isFinished()) { driver.process(); } lookupSourceSupplier = hashBuilder.getLookupSourceSupplier(); } OperatorFactory lineItemTableScan = createTableScanOperator(0, "lineitem", "orderkey", "quantity"); OperatorFactory joinOperator = LookupJoinOperators.innerJoin( 1, lookupSourceSupplier, lineItemTableScan.getTypes(), Ints.asList(0)); NullOutputOperatorFactory output = new NullOutputOperatorFactory(2, joinOperator.getTypes()); DriverFactory driverFactory = new DriverFactory(true, true, lineItemTableScan, joinOperator, output); DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext(); Driver driver = driverFactory.createDriver(driverContext); return ImmutableList.of(driver); }
@Override public MaterializedResult execute(Session session, @Language("SQL") String sql) { lock.readLock().lock(); try { MaterializedOutputFactory outputFactory = new MaterializedOutputFactory(); TaskContext taskContext = createTaskContext(executor, session); List<Driver> drivers = createDrivers(session, sql, outputFactory, taskContext); boolean done = false; while (!done) { boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); processed = true; } } done = !processed; } return outputFactory.getMaterializingOperator().getMaterializedResult(); } finally { lock.readLock().unlock(); } }
@Test public void testAddSourceFinish() { PlanNodeId sourceId = new PlanNodeId("source"); TableScanOperator source = new TableScanOperator( driverContext.addOperatorContext(99, "values"), sourceId, new DataStreamProvider() { @Override public Operator createNewDataStream( OperatorContext operatorContext, Split split, List<ColumnHandle> columns) { return new ValuesOperator( driverContext.addOperatorContext(0, "values"), rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG) .addSequencePage(10, 20, 30, 40) .build()); } }, ImmutableList.of(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG), ImmutableList.<ColumnHandle>of()); MaterializingOperator sink = createSinkOperator(source); Driver driver = new Driver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); // todo TableScanOperator should be blocked until split is set assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertFalse(driver.isFinished()); driver.updateSource( new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, new MockSplit())), true)); assertFalse(driver.isFinished()); assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone()); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
@Override public synchronized boolean isFinished() { if (closed) { return true; } if (driver == null) { return false; } return driver.isFinished(); }
@Test public void testAbruptFinish() { ValuesOperator source = new ValuesOperator( driverContext.addOperatorContext(0, "values"), rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG) .addSequencePage(10, 20, 30, 40) .build()); MaterializingOperator sink = createSinkOperator(source); Driver driver = new Driver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); driver.close(); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
@Test public void testNormalFinish() { List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT); ValuesOperator source = new ValuesOperator( driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), types, rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()); Operator sink = createSinkOperator(source); Driver driver = new Driver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS)); assertTrue(blocked.isDone()); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
@Test public void testAddSourceFinish() { PlanNodeId sourceId = new PlanNodeId("source"); final List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT); TableScanOperator source = new TableScanOperator( driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, new PageSourceProvider() { @Override public ConnectorPageSource createPageSource( Session session, Split split, List<ColumnHandle> columns) { return new FixedPageSource( rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()); } }, types, ImmutableList.<ColumnHandle>of()); PageConsumerOperator sink = createSinkOperator(source); Driver driver = new Driver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertFalse(driver.isFinished()); driver.updateSource( new TaskSource( sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true)); assertFalse(driver.isFinished()); assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone()); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
@Test public void testNormalFinish() { ValuesOperator source = new ValuesOperator( driverContext.addOperatorContext(0, "values"), rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG) .addSequencePage(10, 20, 30, 40) .build()); MaterializingOperator sink = createSinkOperator(source); Driver driver = new Driver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS)); assertTrue(blocked.isDone()); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
public MaterializedResult execute(@Language("SQL") String sql) { MaterializedOutputFactory outputFactory = new MaterializedOutputFactory(); List<Driver> drivers = createDrivers(sql, outputFactory); boolean done = false; while (!done) { boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); processed = true; } } done = !processed; } return outputFactory.getMaterializingOperator().getMaterializedResult(); }
@Test public void testBrokenOperatorAddSource() throws Exception { PlanNodeId sourceId = new PlanNodeId("source"); final List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT); // create a table scan operator that does not block, which will cause the driver loop to busy // wait TableScanOperator source = new NotBlockedTableScanOperator( driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, new PageSourceProvider() { @Override public ConnectorPageSource createPageSource( Session session, Split split, List<ColumnHandle> columns) { return new FixedPageSource( rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()); } }, types, ImmutableList.<ColumnHandle>of()); BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source")); final Driver driver = new Driver(driverContext, source, brokenOperator); // block thread in operator processing Future<Boolean> driverProcessFor = executor.submit( new Callable<Boolean>() { @Override public Boolean call() throws Exception { return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone(); } }); brokenOperator.waitForLocked(); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertFalse(driver.isFinished()); driver.updateSource( new TaskSource( sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true)); assertFalse(driver.isFinished()); // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone()); assertFalse(driver.isFinished()); driver.close(); assertTrue(driver.isFinished()); try { driverProcessFor.get(1, TimeUnit.SECONDS); fail("Expected InterruptedException"); } catch (ExecutionException e) { checkArgument( getRootCause(e) instanceof InterruptedException, "Expected root cause exception to be an instance of InterruptedException"); } }
@Override public synchronized boolean isFinished() { return driver.isFinished(); }
private static LookupSourceSupplier buildHash( boolean parallelBuild, TaskContext taskContext, List<Integer> hashChannels, RowPagesBuilder buildPages) { if (parallelBuild) { ParallelHashBuilder parallelHashBuilder = new ParallelHashBuilder( buildPages.getTypes(), hashChannels, buildPages.getHashChannel(), 100, PARTITION_COUNT); // collect input data DriverContext collectDriverContext = taskContext.addPipelineContext(true, true).addDriverContext(); ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory( 0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build()); OperatorFactory collectOperatorFactory = parallelHashBuilder.getCollectOperatorFactory(1, new PlanNodeId("test")); Driver driver = new Driver( collectDriverContext, valuesOperatorFactory.createOperator(collectDriverContext), collectOperatorFactory.createOperator(collectDriverContext)); while (!driver.isFinished()) { driver.process(); } // build hash tables PipelineContext buildPipeline = taskContext.addPipelineContext(true, true); OperatorFactory buildOperatorFactory = parallelHashBuilder.getBuildOperatorFactory(new PlanNodeId("test")); for (int i = 0; i < PARTITION_COUNT; i++) { DriverContext buildDriverContext = buildPipeline.addDriverContext(); Driver buildDriver = new Driver(buildDriverContext, buildOperatorFactory.createOperator(buildDriverContext)); while (!buildDriver.isFinished()) { buildDriver.process(); } } return parallelHashBuilder.getLookupSourceSupplier(); } else { DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext(); ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory( 0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build()); HashBuilderOperatorFactory hashBuilderOperatorFactory = new HashBuilderOperatorFactory( 1, new PlanNodeId("test"), buildPages.getTypes(), hashChannels, buildPages.getHashChannel(), 100); Driver driver = new Driver( driverContext, valuesOperatorFactory.createOperator(driverContext), hashBuilderOperatorFactory.createOperator(driverContext)); while (!driver.isFinished()) { driver.process(); } return hashBuilderOperatorFactory.getLookupSourceSupplier(); } }
@Test public void testBrokenOperatorAddSource() throws Exception { PlanNodeId sourceId = new PlanNodeId("source"); TableScanOperator source = new TableScanOperator( driverContext.addOperatorContext(99, "values"), sourceId, new DataStreamProvider() { @Override public Operator createNewDataStream( OperatorContext operatorContext, Split split, List<ColumnHandle> columns) { return new ValuesOperator( driverContext.addOperatorContext(0, "values"), rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG) .addSequencePage(10, 20, 30, 40) .build()); } }, ImmutableList.of(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG), ImmutableList.<ColumnHandle>of()); BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, "source")); final Driver driver = new Driver(driverContext, source, brokenOperator); // block thread in operator processing Future<Boolean> driverProcessFor = executor.submit( new Callable<Boolean>() { @Override public Boolean call() throws Exception { return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone(); } }); brokenOperator.waitForLocked(); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); // todo TableScanOperator should be blocked until split is set assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertFalse(driver.isFinished()); driver.updateSource( new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, new MockSplit())), true)); assertFalse(driver.isFinished()); assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone()); assertFalse(driver.isFinished()); driver.close(); assertTrue(driver.isFinished()); try { driverProcessFor.get(1, TimeUnit.SECONDS); fail("Expected InterruptedException"); } catch (ExecutionException e) { checkArgument( getRootCause(e) instanceof InterruptedException, "Expected root cause exception to be an instance of InterruptedException"); } }