public List<Driver> createDrivers(
Session session,
@Language("SQL") String sql,
OutputFactory outputFactory,
TaskContext taskContext) {
Statement statement = sqlParser.createStatement(sql);
assertFormattedSql(sqlParser, statement);
PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
FeaturesConfig featuresConfig =
new FeaturesConfig()
.setExperimentalSyntaxEnabled(true)
.setDistributedIndexJoinsEnabled(false)
.setOptimizeHashGeneration(true);
PlanOptimizersFactory planOptimizersFactory =
new PlanOptimizersFactory(metadata, sqlParser, indexManager, featuresConfig, true);
QueryExplainer queryExplainer =
new QueryExplainer(
planOptimizersFactory.get(),
metadata,
accessControl,
sqlParser,
dataDefinitionTask,
featuresConfig.isExperimentalSyntaxEnabled());
Analyzer analyzer =
new Analyzer(
session,
metadata,
sqlParser,
accessControl,
Optional.of(queryExplainer),
featuresConfig.isExperimentalSyntaxEnabled());
Analysis analysis = analyzer.analyze(statement);
Plan plan =
new LogicalPlanner(session, planOptimizersFactory.get(), idAllocator, metadata)
.plan(analysis);
if (printPlan) {
System.out.println(
PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata, session));
}
SubPlan subplan = new PlanFragmenter().createSubPlans(plan);
if (!subplan.getChildren().isEmpty()) {
throw new AssertionError("Expected subplan to have no children");
}
LocalExecutionPlanner executionPlanner =
new LocalExecutionPlanner(
metadata,
sqlParser,
pageSourceManager,
indexManager,
pageSinkManager,
null,
compiler,
new IndexJoinLookupStats(),
new CompilerConfig()
.setInterpreterEnabled(false), // make sure tests fail if compiler breaks
new TaskManagerConfig().setTaskDefaultConcurrency(4));
// plan query
LocalExecutionPlan localExecutionPlan =
executionPlanner.plan(
session,
subplan.getFragment().getRoot(),
subplan.getFragment().getOutputLayout(),
plan.getTypes(),
subplan.getFragment().getDistribution(),
outputFactory);
// generate sources
List<TaskSource> sources = new ArrayList<>();
long sequenceId = 0;
for (TableScanNode tableScan : findTableScanNodes(subplan.getFragment().getRoot())) {
TableLayoutHandle layout = tableScan.getLayout().get();
SplitSource splitSource = splitManager.getSplits(session, layout);
ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
while (!splitSource.isFinished()) {
for (Split split : getFutureValue(splitSource.getNextBatch(1000))) {
scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
}
}
sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
}
// create drivers
List<Driver> drivers = new ArrayList<>();
Map<PlanNodeId, Driver> driversBySource = new HashMap<>();
for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
for (int i = 0; i < driverFactory.getDriverInstances(); i++) {
DriverContext driverContext =
taskContext
.addPipelineContext(driverFactory.isInputDriver(), driverFactory.isOutputDriver())
.addDriverContext();
Driver driver = driverFactory.createDriver(driverContext);
drivers.add(driver);
for (PlanNodeId sourceId : driver.getSourceIds()) {
driversBySource.put(sourceId, driver);
}
}
driverFactory.close();
}
// add sources to the drivers
for (TaskSource source : sources) {
for (Driver driver : driversBySource.values()) {
driver.updateSource(source);
}
}
return ImmutableList.copyOf(drivers);
}
public List<Driver> createDrivers(
@Language("SQL") String sql, OutputFactory outputFactory, TaskContext taskContext) {
Statement statement = SqlParser.createStatement(sql);
if (printPlan) {
assertFormattedSql(statement);
}
PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
FeaturesConfig featuresConfig = new FeaturesConfig().setExperimentalSyntaxEnabled(true);
PlanOptimizersFactory planOptimizersFactory =
new PlanOptimizersFactory(metadata, splitManager, indexManager, featuresConfig);
QueryExplainer queryExplainer =
new QueryExplainer(
session,
planOptimizersFactory.get(),
metadata,
featuresConfig.isExperimentalSyntaxEnabled());
Analyzer analyzer =
new Analyzer(
session,
metadata,
Optional.of(queryExplainer),
featuresConfig.isExperimentalSyntaxEnabled());
Analysis analysis = analyzer.analyze(statement);
Plan plan =
new LogicalPlanner(session, planOptimizersFactory.get(), idAllocator, metadata)
.plan(analysis);
if (printPlan) {
System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata));
}
SubPlan subplan =
new DistributedLogicalPlanner(session, metadata, idAllocator).createSubPlans(plan, true);
assertTrue(subplan.getChildren().isEmpty(), "Expected subplan to have no children");
LocalExecutionPlanner executionPlanner =
new LocalExecutionPlanner(
new NodeInfo(new NodeConfig().setEnvironment("test").setNodeId("test-node")),
metadata,
dataStreamProvider,
indexManager,
storageManager,
recordSinkManager,
null,
compiler);
// plan query
LocalExecutionPlan localExecutionPlan =
executionPlanner.plan(
session, subplan.getFragment().getRoot(), plan.getTypes(), outputFactory);
// generate sources
List<TaskSource> sources = new ArrayList<>();
long sequenceId = 0;
for (PlanNode sourceNode : subplan.getFragment().getSources()) {
if (sourceNode instanceof ValuesNode) {
continue;
}
TableScanNode tableScan = (TableScanNode) sourceNode;
SplitSource splitSource =
splitManager.getPartitionSplits(tableScan.getTable(), getPartitions(tableScan));
ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
while (!splitSource.isFinished()) {
try {
for (Split split : splitSource.getNextBatch(1000)) {
scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw Throwables.propagate(e);
}
}
sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
}
// create drivers
List<Driver> drivers = new ArrayList<>();
Map<PlanNodeId, Driver> driversBySource = new HashMap<>();
for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
DriverContext driverContext =
taskContext
.addPipelineContext(driverFactory.isInputDriver(), driverFactory.isOutputDriver())
.addDriverContext();
Driver driver = driverFactory.createDriver(driverContext);
drivers.add(driver);
for (PlanNodeId sourceId : driver.getSourceIds()) {
driversBySource.put(sourceId, driver);
}
driverFactory.close();
}
// add sources to the drivers
for (TaskSource source : sources) {
for (Driver driver : driversBySource.values()) {
driver.updateSource(source);
}
}
return ImmutableList.copyOf(drivers);
}
public PlanOptimizers(
Metadata metadata,
SqlParser sqlParser,
FeaturesConfig featuresConfig,
boolean forceSingleNode) {
ImmutableList.Builder<PlanOptimizer> builder = ImmutableList.builder();
builder.add(
new DesugaringOptimizer(
metadata,
sqlParser), // Clean up all the sugar in expressions, e.g. AtTimeZone, must be run
// before all the other optimizers
new CanonicalizeExpressions(),
new ImplementFilteredAggregations(),
new ImplementSampleAsFilter(),
new SimplifyExpressions(metadata, sqlParser),
new UnaliasSymbolReferences(),
new PruneIdentityProjections(),
new SetFlatteningOptimizer(),
new ImplementIntersectAndExceptAsUnion(),
new LimitPushDown(), // Run the LimitPushDown after flattening set operators to make it
// easier to do the set flattening
new PruneUnreferencedOutputs(),
new MergeProjections(),
new TransformExistsApplyToScalarApply(metadata),
new TransformQuantifiedComparisonApplyToScalarApply(metadata),
new RemoveUnreferencedScalarInputApplyNodes(),
new TransformUncorrelatedInPredicateSubqueryToSemiJoin(),
new TransformUncorrelatedScalarToJoin(),
new TransformCorrelatedScalarAggregationToJoin(metadata),
new PredicatePushDown(metadata, sqlParser),
new MergeProjections(),
new SimplifyExpressions(
metadata,
sqlParser), // Re-run the SimplifyExpressions to simplify any recomposed expressions
// from other optimizations
new ProjectionPushDown(),
new UnaliasSymbolReferences(), // Run again because predicate pushdown and projection
// pushdown might add more projections
new PruneUnreferencedOutputs(), // Make sure to run this before index join. Filtered
// projections may not have all the columns.
new IndexJoinOptimizer(
metadata), // Run this after projections and filters have been fully simplified and
// pushed down
new CountConstantOptimizer(),
new WindowFilterPushDown(
metadata), // This must run after PredicatePushDown and LimitPushDown so that it
// squashes any successive filter nodes and limits
new MergeWindows(),
new ReorderWindows(), // Should be after MergeWindows to avoid unnecessary reordering of
// mergeable windows
new MergeProjections(),
new PruneUnreferencedOutputs(), // Make sure to run this at the end to help clean the plan
// for logging/execution and not remove info that other
// optimizers might need at an earlier point
new PruneIdentityProjections(), // This MUST run after PruneUnreferencedOutputs as it may
// introduce new redundant projections
new MetadataQueryOptimizer(metadata),
new EliminateCrossJoins(), // This can pull up Filter and Project nodes from between Joins,
// so we need to push them down again
new PredicatePushDown(metadata, sqlParser),
new ProjectionPushDown());
if (featuresConfig.isOptimizeSingleDistinct()) {
builder.add(new SingleDistinctOptimizer());
builder.add(new PruneUnreferencedOutputs());
}
builder.add(new OptimizeMixedDistinctAggregations(metadata));
if (!forceSingleNode) {
builder.add(new PushTableWriteThroughUnion()); // Must run before AddExchanges
builder.add(new AddExchanges(metadata, sqlParser));
}
builder.add(new PickLayout(metadata));
builder.add(new EmptyDeleteOptimizer()); // Run after table scan is removed by PickLayout
builder.add(
new PredicatePushDown(
metadata,
sqlParser)); // Run predicate push down one more time in case we can leverage new
// information from layouts' effective predicate
builder.add(new ProjectionPushDown());
builder.add(new MergeProjections());
builder.add(
new UnaliasSymbolReferences()); // Run unalias after merging projections to simplify
// projections more efficiently
builder.add(new PruneUnreferencedOutputs());
builder.add(new PruneIdentityProjections());
// Optimizers above this don't understand local exchanges, so be careful moving this.
builder.add(new AddLocalExchanges(metadata, sqlParser));
// Optimizers above this do not need to care about aggregations with the type other than SINGLE
// This optimizer must be run after all exchange-related optimizers
builder.add(new PartialAggregationPushDown(metadata.getFunctionRegistry()));
builder.add(new PruneIdentityProjections());
// DO NOT add optimizers that change the plan shape (computations) after this point
// Precomputed hashes - this assumes that partitioning will not change
builder.add(new HashGenerationOptimizer());
builder.add(new MetadataDeleteOptimizer(metadata));
builder.add(new BeginTableWrite(metadata)); // HACK! see comments in BeginTableWrite
// TODO: consider adding a formal final plan sanitization optimizer that prepares the plan for
// transmission/execution/logging
// TODO: figure out how to improve the set flattening optimizer so that it can run at any point
this.optimizers = builder.build();
}
