Ejemplo n.º 1
0
  public List<Driver> createDrivers(
      Session session,
      @Language("SQL") String sql,
      OutputFactory outputFactory,
      TaskContext taskContext) {
    Statement statement = sqlParser.createStatement(sql);

    assertFormattedSql(sqlParser, statement);

    PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
    FeaturesConfig featuresConfig =
        new FeaturesConfig()
            .setExperimentalSyntaxEnabled(true)
            .setDistributedIndexJoinsEnabled(false)
            .setOptimizeHashGeneration(true);
    PlanOptimizersFactory planOptimizersFactory =
        new PlanOptimizersFactory(metadata, sqlParser, indexManager, featuresConfig, true);

    QueryExplainer queryExplainer =
        new QueryExplainer(
            planOptimizersFactory.get(),
            metadata,
            accessControl,
            sqlParser,
            dataDefinitionTask,
            featuresConfig.isExperimentalSyntaxEnabled());
    Analyzer analyzer =
        new Analyzer(
            session,
            metadata,
            sqlParser,
            accessControl,
            Optional.of(queryExplainer),
            featuresConfig.isExperimentalSyntaxEnabled());

    Analysis analysis = analyzer.analyze(statement);
    Plan plan =
        new LogicalPlanner(session, planOptimizersFactory.get(), idAllocator, metadata)
            .plan(analysis);

    if (printPlan) {
      System.out.println(
          PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata, session));
    }

    SubPlan subplan = new PlanFragmenter().createSubPlans(plan);
    if (!subplan.getChildren().isEmpty()) {
      throw new AssertionError("Expected subplan to have no children");
    }

    LocalExecutionPlanner executionPlanner =
        new LocalExecutionPlanner(
            metadata,
            sqlParser,
            pageSourceManager,
            indexManager,
            pageSinkManager,
            null,
            compiler,
            new IndexJoinLookupStats(),
            new CompilerConfig()
                .setInterpreterEnabled(false), // make sure tests fail if compiler breaks
            new TaskManagerConfig().setTaskDefaultConcurrency(4));

    // plan query
    LocalExecutionPlan localExecutionPlan =
        executionPlanner.plan(
            session,
            subplan.getFragment().getRoot(),
            subplan.getFragment().getOutputLayout(),
            plan.getTypes(),
            subplan.getFragment().getDistribution(),
            outputFactory);

    // generate sources
    List<TaskSource> sources = new ArrayList<>();
    long sequenceId = 0;
    for (TableScanNode tableScan : findTableScanNodes(subplan.getFragment().getRoot())) {
      TableLayoutHandle layout = tableScan.getLayout().get();

      SplitSource splitSource = splitManager.getSplits(session, layout);

      ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
      while (!splitSource.isFinished()) {
        for (Split split : getFutureValue(splitSource.getNextBatch(1000))) {
          scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
        }
      }

      sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
    }

    // create drivers
    List<Driver> drivers = new ArrayList<>();
    Map<PlanNodeId, Driver> driversBySource = new HashMap<>();
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
      for (int i = 0; i < driverFactory.getDriverInstances(); i++) {
        DriverContext driverContext =
            taskContext
                .addPipelineContext(driverFactory.isInputDriver(), driverFactory.isOutputDriver())
                .addDriverContext();
        Driver driver = driverFactory.createDriver(driverContext);
        drivers.add(driver);
        for (PlanNodeId sourceId : driver.getSourceIds()) {
          driversBySource.put(sourceId, driver);
        }
      }
      driverFactory.close();
    }

    // add sources to the drivers
    for (TaskSource source : sources) {
      for (Driver driver : driversBySource.values()) {
        driver.updateSource(source);
      }
    }

    return ImmutableList.copyOf(drivers);
  }
Ejemplo n.º 2
0
  public List<Driver> createDrivers(
      @Language("SQL") String sql, OutputFactory outputFactory, TaskContext taskContext) {
    Statement statement = SqlParser.createStatement(sql);

    if (printPlan) {
      assertFormattedSql(statement);
    }

    PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
    FeaturesConfig featuresConfig = new FeaturesConfig().setExperimentalSyntaxEnabled(true);
    PlanOptimizersFactory planOptimizersFactory =
        new PlanOptimizersFactory(metadata, splitManager, indexManager, featuresConfig);

    QueryExplainer queryExplainer =
        new QueryExplainer(
            session,
            planOptimizersFactory.get(),
            metadata,
            featuresConfig.isExperimentalSyntaxEnabled());
    Analyzer analyzer =
        new Analyzer(
            session,
            metadata,
            Optional.of(queryExplainer),
            featuresConfig.isExperimentalSyntaxEnabled());

    Analysis analysis = analyzer.analyze(statement);

    Plan plan =
        new LogicalPlanner(session, planOptimizersFactory.get(), idAllocator, metadata)
            .plan(analysis);
    if (printPlan) {
      System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata));
    }

    SubPlan subplan =
        new DistributedLogicalPlanner(session, metadata, idAllocator).createSubPlans(plan, true);
    assertTrue(subplan.getChildren().isEmpty(), "Expected subplan to have no children");

    LocalExecutionPlanner executionPlanner =
        new LocalExecutionPlanner(
            new NodeInfo(new NodeConfig().setEnvironment("test").setNodeId("test-node")),
            metadata,
            dataStreamProvider,
            indexManager,
            storageManager,
            recordSinkManager,
            null,
            compiler);

    // plan query
    LocalExecutionPlan localExecutionPlan =
        executionPlanner.plan(
            session, subplan.getFragment().getRoot(), plan.getTypes(), outputFactory);

    // generate sources
    List<TaskSource> sources = new ArrayList<>();
    long sequenceId = 0;
    for (PlanNode sourceNode : subplan.getFragment().getSources()) {
      if (sourceNode instanceof ValuesNode) {
        continue;
      }

      TableScanNode tableScan = (TableScanNode) sourceNode;

      SplitSource splitSource =
          splitManager.getPartitionSplits(tableScan.getTable(), getPartitions(tableScan));

      ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
      while (!splitSource.isFinished()) {
        try {
          for (Split split : splitSource.getNextBatch(1000)) {
            scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
          }
        } catch (InterruptedException e) {
          Thread.currentThread().interrupt();
          throw Throwables.propagate(e);
        }
      }

      sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
    }

    // create drivers
    List<Driver> drivers = new ArrayList<>();
    Map<PlanNodeId, Driver> driversBySource = new HashMap<>();
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
      DriverContext driverContext =
          taskContext
              .addPipelineContext(driverFactory.isInputDriver(), driverFactory.isOutputDriver())
              .addDriverContext();
      Driver driver = driverFactory.createDriver(driverContext);
      drivers.add(driver);
      for (PlanNodeId sourceId : driver.getSourceIds()) {
        driversBySource.put(sourceId, driver);
      }
      driverFactory.close();
    }

    // add sources to the drivers
    for (TaskSource source : sources) {
      for (Driver driver : driversBySource.values()) {
        driver.updateSource(source);
      }
    }

    return ImmutableList.copyOf(drivers);
  }
Ejemplo n.º 3
0
  public PlanOptimizers(
      Metadata metadata,
      SqlParser sqlParser,
      FeaturesConfig featuresConfig,
      boolean forceSingleNode) {
    ImmutableList.Builder<PlanOptimizer> builder = ImmutableList.builder();

    builder.add(
        new DesugaringOptimizer(
            metadata,
            sqlParser), // Clean up all the sugar in expressions, e.g. AtTimeZone, must be run
                        // before all the other optimizers
        new CanonicalizeExpressions(),
        new ImplementFilteredAggregations(),
        new ImplementSampleAsFilter(),
        new SimplifyExpressions(metadata, sqlParser),
        new UnaliasSymbolReferences(),
        new PruneIdentityProjections(),
        new SetFlatteningOptimizer(),
        new ImplementIntersectAndExceptAsUnion(),
        new LimitPushDown(), // Run the LimitPushDown after flattening set operators to make it
                             // easier to do the set flattening
        new PruneUnreferencedOutputs(),
        new MergeProjections(),
        new TransformExistsApplyToScalarApply(metadata),
        new TransformQuantifiedComparisonApplyToScalarApply(metadata),
        new RemoveUnreferencedScalarInputApplyNodes(),
        new TransformUncorrelatedInPredicateSubqueryToSemiJoin(),
        new TransformUncorrelatedScalarToJoin(),
        new TransformCorrelatedScalarAggregationToJoin(metadata),
        new PredicatePushDown(metadata, sqlParser),
        new MergeProjections(),
        new SimplifyExpressions(
            metadata,
            sqlParser), // Re-run the SimplifyExpressions to simplify any recomposed expressions
                        // from other optimizations
        new ProjectionPushDown(),
        new UnaliasSymbolReferences(), // Run again because predicate pushdown and projection
                                       // pushdown might add more projections
        new PruneUnreferencedOutputs(), // Make sure to run this before index join. Filtered
                                        // projections may not have all the columns.
        new IndexJoinOptimizer(
            metadata), // Run this after projections and filters have been fully simplified and
                       // pushed down
        new CountConstantOptimizer(),
        new WindowFilterPushDown(
            metadata), // This must run after PredicatePushDown and LimitPushDown so that it
                       // squashes any successive filter nodes and limits
        new MergeWindows(),
        new ReorderWindows(), // Should be after MergeWindows to avoid unnecessary reordering of
                              // mergeable windows
        new MergeProjections(),
        new PruneUnreferencedOutputs(), // Make sure to run this at the end to help clean the plan
                                        // for logging/execution and not remove info that other
                                        // optimizers might need at an earlier point
        new PruneIdentityProjections(), // This MUST run after PruneUnreferencedOutputs as it may
                                        // introduce new redundant projections
        new MetadataQueryOptimizer(metadata),
        new EliminateCrossJoins(), // This can pull up Filter and Project nodes from between Joins,
                                   // so we need to push them down again
        new PredicatePushDown(metadata, sqlParser),
        new ProjectionPushDown());

    if (featuresConfig.isOptimizeSingleDistinct()) {
      builder.add(new SingleDistinctOptimizer());
      builder.add(new PruneUnreferencedOutputs());
    }

    builder.add(new OptimizeMixedDistinctAggregations(metadata));

    if (!forceSingleNode) {
      builder.add(new PushTableWriteThroughUnion()); // Must run before AddExchanges
      builder.add(new AddExchanges(metadata, sqlParser));
    }

    builder.add(new PickLayout(metadata));

    builder.add(new EmptyDeleteOptimizer()); // Run after table scan is removed by PickLayout

    builder.add(
        new PredicatePushDown(
            metadata,
            sqlParser)); // Run predicate push down one more time in case we can leverage new
                         // information from layouts' effective predicate
    builder.add(new ProjectionPushDown());
    builder.add(new MergeProjections());
    builder.add(
        new UnaliasSymbolReferences()); // Run unalias after merging projections to simplify
                                        // projections more efficiently
    builder.add(new PruneUnreferencedOutputs());
    builder.add(new PruneIdentityProjections());

    // Optimizers above this don't understand local exchanges, so be careful moving this.
    builder.add(new AddLocalExchanges(metadata, sqlParser));

    // Optimizers above this do not need to care about aggregations with the type other than SINGLE
    // This optimizer must be run after all exchange-related optimizers
    builder.add(new PartialAggregationPushDown(metadata.getFunctionRegistry()));
    builder.add(new PruneIdentityProjections());

    // DO NOT add optimizers that change the plan shape (computations) after this point

    // Precomputed hashes - this assumes that partitioning will not change
    builder.add(new HashGenerationOptimizer());

    builder.add(new MetadataDeleteOptimizer(metadata));
    builder.add(new BeginTableWrite(metadata)); // HACK! see comments in BeginTableWrite

    // TODO: consider adding a formal final plan sanitization optimizer that prepares the plan for
    // transmission/execution/logging
    // TODO: figure out how to improve the set flattening optimizer so that it can run at any point

    this.optimizers = builder.build();
  }