@Override
 public SubPlanBuilder visitUnion(UnionNode node, Void context) {
   if (createSingleNodePlan) {
     ImmutableList.Builder<PlanNode> sourceBuilder = ImmutableList.builder();
     for (PlanNode source : node.getSources()) {
       sourceBuilder.add(source.accept(this, context).getRoot());
     }
     UnionNode unionNode =
         new UnionNode(node.getId(), sourceBuilder.build(), node.getSymbolMapping());
     return createSingleNodePlan(unionNode);
   } else {
     ImmutableList.Builder<SubPlan> sourceBuilder = ImmutableList.builder();
     ImmutableList.Builder<PlanFragmentId> fragmentIdBuilder = ImmutableList.builder();
     for (int i = 0; i < node.getSources().size(); i++) {
       PlanNode subPlan = node.getSources().get(i);
       SubPlanBuilder current = subPlan.accept(this, context);
       current.setRoot(
           new SinkNode(idAllocator.getNextId(), current.getRoot(), node.sourceOutputLayout(i)));
       fragmentIdBuilder.add(current.getId());
       sourceBuilder.add(current.build());
     }
     ExchangeNode exchangeNode =
         new ExchangeNode(
             idAllocator.getNextId(), fragmentIdBuilder.build(), node.getOutputSymbols());
     return createSingleNodePlan(exchangeNode).setChildren(sourceBuilder.build());
   }
 }
Example #2
0
    @Override
    public Void visitUnion(UnionNode node, Void context) {
      printNode(node, "Union", NODE_COLORS.get(NodeType.UNION));

      for (PlanNode planNode : node.getSources()) {
        planNode.accept(this, context);
      }

      return null;
    }
Example #3
0
  private static void findSources(
      PlanNode node, Builder<PlanNode> builder, PlanNodeId partitionedSource) {
    for (PlanNode source : node.getSources()) {
      findSources(source, builder, partitionedSource);
    }

    if (node.getSources().isEmpty() || node.getId().equals(partitionedSource)) {
      builder.add(node);
    }
  }
Example #4
0
    @Override
    protected Void visitPlan(PlanNode node, Void context) {
      for (PlanNode child : node.getSources()) {
        printEdge(node, child);

        child.accept(this, context);
      }

      return null;
    }
Example #5
0
  private static void printFragmentNodes(
      StringBuilder output, PlanFragment fragment, PlanNodeIdGenerator idGenerator) {
    String clusterId = "cluster_" + fragment.getId();
    output.append("subgraph ").append(clusterId).append(" {").append('\n');

    output.append(format("label = \"%s\"", fragment.getDistribution())).append('\n');

    PlanNode plan = fragment.getRoot();
    plan.accept(new NodePrinter(output, idGenerator), null);

    output.append("}").append('\n');
  }
    @Override
    public PlanNode visitProject(ProjectNode node, RewriteContext<Void> context) {
      PlanNode source = context.rewrite(node.getSource());

      if (!node.getOutputSymbols().equals(source.getOutputSymbols())) {
        // Can't get rid of this projection. It constrains the output tuple from the underlying
        // operator
        return replaceChildren(node, ImmutableList.of(source));
      }

      if (node.isIdentity()) {
        return source;
      }

      return replaceChildren(node, ImmutableList.of(source));
    }
Example #7
0
 @Override
 public Void visitExchange(ExchangeNode node, Void context) {
   List<Symbol> symbols = node.getOutputSymbols();
   if (node.getType() == REPARTITION || node.getType() == REPARTITION_WITH_NULL_REPLICATION) {
     symbols = node.getPartitionKeys().orElseGet(() -> ImmutableList.of(new Symbol("(absent)")));
   }
   String columns = Joiner.on(", ").join(symbols);
   printNode(
       node,
       format("ExchangeNode[%s]", node.getType()),
       columns,
       NODE_COLORS.get(NodeType.EXCHANGE));
   for (PlanNode planNode : node.getSources()) {
     planNode.accept(this, context);
   }
   return null;
 }
Example #8
0
 private PlanNode removeAllRecursive(PlanNode node) {
   if (where.test(node)) {
     checkArgument(
         node.getSources().size() == 1,
         "Unable to remove plan node as it contains 0 or more than 1 children");
     return node.getSources().get(0);
   }
   if (skipOnly.test(node)) {
     List<PlanNode> sources =
         node.getSources()
             .stream()
             .map(source -> removeAllRecursive(source))
             .collect(toImmutableList());
     return replaceChildren(node, sources);
   }
   return node;
 }
 public static ActualProperties streamBackdoorDeriveProperties(
     PlanNode node,
     List<ActualProperties> inputProperties,
     Metadata metadata,
     Session session,
     Map<Symbol, Type> types,
     SqlParser parser) {
   return node.accept(new Visitor(metadata, session, types, parser), inputProperties);
 }
Example #10
0
  private static void findTableScanNodes(
      PlanNode node, ImmutableList.Builder<TableScanNode> builder) {
    for (PlanNode source : node.getSources()) {
      findTableScanNodes(source, builder);
    }

    if (node instanceof TableScanNode) {
      builder.add((TableScanNode) node);
    }
  }
    @Override
    public SubPlanBuilder visitMarkDistinct(MarkDistinctNode node, Void context) {
      SubPlanBuilder current = node.getSource().accept(this, context);
      // Check if the subplan is already partitioned the way we want it
      boolean alreadyPartitioned = false;
      if (current.getDistribution() == PlanDistribution.FIXED) {
        for (SubPlan child : current.getChildren()) {
          if (child.getFragment().getOutputPartitioning() == OutputPartitioning.HASH
              && ImmutableSet.copyOf(child.getFragment().getPartitionBy())
                  .equals(ImmutableSet.copyOf(node.getDistinctSymbols()))) {
            alreadyPartitioned = true;
            break;
          }
        }
      }
      if (createSingleNodePlan || alreadyPartitioned || !current.isDistributed()) {
        MarkDistinctNode markNode =
            new MarkDistinctNode(
                idAllocator.getNextId(),
                current.getRoot(),
                node.getMarkerSymbol(),
                node.getDistinctSymbols(),
                node.getSampleWeightSymbol());
        current.setRoot(markNode);
        return current;
      } else {
        PlanNode sink =
            new SinkNode(
                idAllocator.getNextId(), current.getRoot(), current.getRoot().getOutputSymbols());
        current.setRoot(sink).setHashOutputPartitioning(node.getDistinctSymbols());

        PlanNode exchange =
            new ExchangeNode(idAllocator.getNextId(), current.getId(), sink.getOutputSymbols());
        MarkDistinctNode markNode =
            new MarkDistinctNode(
                idAllocator.getNextId(),
                exchange,
                node.getMarkerSymbol(),
                node.getDistinctSymbols(),
                node.getSampleWeightSymbol());
        return createFixedDistributionPlan(markNode).addChild(current.build());
      }
    }
Example #12
0
  @JsonCreator
  public PlanFragment(
      @JsonProperty("id") PlanFragmentId id,
      @JsonProperty("root") PlanNode root,
      @JsonProperty("symbols") Map<Symbol, Type> symbols,
      @JsonProperty("distribution") PlanDistribution distribution,
      @JsonProperty("partitionedSource") PlanNodeId partitionedSource,
      @JsonProperty("outputPartitioning") OutputPartitioning outputPartitioning) {
    this.id = checkNotNull(id, "id is null");
    this.root = checkNotNull(root, "root is null");
    this.symbols = checkNotNull(symbols, "symbols is null");
    this.distribution = checkNotNull(distribution, "distribution is null");
    this.partitionedSource = partitionedSource;

    tupleInfos =
        IterableTransformer.on(root.getOutputSymbols())
            .transform(Functions.forMap(symbols))
            .transform(Type.toRaw())
            .transform(
                new Function<TupleInfo.Type, TupleInfo>() {
                  @Override
                  public TupleInfo apply(TupleInfo.Type input) {
                    return new TupleInfo(input);
                  }
                })
            .list();

    ImmutableList.Builder<PlanNode> sources = ImmutableList.builder();
    findSources(root, sources, partitionedSource);
    this.sources = sources.build();

    ImmutableSet.Builder<PlanNodeId> sourceIds = ImmutableSet.builder();
    for (PlanNode source : this.sources) {
      sourceIds.add(source.getId());
    }
    if (partitionedSource != null) {
      sourceIds.add(partitionedSource);
    }
    this.sourceIds = sourceIds.build();

    this.outputPartitioning = checkNotNull(outputPartitioning, "outputPartitioning is null");
  }
Example #13
0
 private PlanNode removeFirstRecursive(PlanNode node) {
   if (where.test(node)) {
     checkArgument(
         node.getSources().size() == 1,
         "Unable to remove plan node as it contains 0 or more than 1 children");
     return node.getSources().get(0);
   }
   if (skipOnly.test(node)) {
     List<PlanNode> sources = node.getSources();
     if (sources.isEmpty()) {
       return node;
     } else if (sources.size() == 1) {
       return replaceChildren(node, ImmutableList.of(removeFirstRecursive(sources.get(0))));
     } else {
       throw new IllegalArgumentException(
           "Unable to remove first node when a node has multiple children, use removeAll instead");
     }
   }
   return node;
 }
Example #14
0
 private <T extends PlanNode> void findAllRecursive(
     PlanNode node, ImmutableList.Builder<T> nodes) {
   if (where.test(node)) {
     nodes.add((T) node);
   }
   if (skipOnly.test(node)) {
     for (PlanNode source : node.getSources()) {
       findAllRecursive(source, nodes);
     }
   }
 }
Example #15
0
 private PlanNode distinct(PlanNode node) {
   return new AggregationNode(
       idAllocator.getNextId(),
       node,
       node.getOutputSymbols(),
       ImmutableMap.<Symbol, FunctionCall>of(),
       ImmutableMap.<Symbol, Signature>of(),
       ImmutableMap.<Symbol, Symbol>of(),
       AggregationNode.Step.SINGLE,
       Optional.empty(),
       1.0,
       Optional.empty());
 }
Example #16
0
 private <T extends PlanNode> Optional<T> findFirstRecursive(PlanNode node) {
   if (where.test(node)) {
     return Optional.of((T) node);
   }
   if (skipOnly.test(node)) {
     for (PlanNode source : node.getSources()) {
       Optional<T> found = findFirstRecursive(source);
       if (found.isPresent()) {
         return found;
       }
     }
   }
   return Optional.empty();
 }
Example #17
0
 private PlanNode replaceFirstRecursive(PlanNode node, PlanNode nodeToReplace) {
   if (where.test(node)) {
     return nodeToReplace;
   }
   List<PlanNode> sources = node.getSources();
   if (sources.isEmpty()) {
     return node;
   } else if (sources.size() == 1) {
     return replaceChildren(node, ImmutableList.of(replaceFirstRecursive(node, sources.get(0))));
   } else {
     throw new IllegalArgumentException(
         "Unable to replace first node when a node has multiple children, use replaceAll instead");
   }
 }
Example #18
0
 private PlanNode replaceAllRecursive(PlanNode node, PlanNode nodeToReplace) {
   if (where.test(node)) {
     return nodeToReplace;
   }
   if (skipOnly.test(node)) {
     List<PlanNode> sources =
         node.getSources()
             .stream()
             .map(source -> replaceAllRecursive(source, nodeToReplace))
             .collect(toImmutableList());
     return replaceChildren(node, sources);
   }
   return node;
 }
 private <T> Optional<T> findNode(PlanNode source, Class<T> clazz) {
   while (true) {
     // allow any chain of linear exchanges
     if (source instanceof ExchangeNode) {
       List<PlanNode> sources = source.getSources();
       if (sources.size() != 1) {
         return Optional.empty();
       }
       source = sources.get(0);
     } else if (clazz.isInstance(source)) {
       return Optional.of(clazz.cast(source));
     } else {
       return Optional.empty();
     }
   }
 }
  public static ActualProperties deriveProperties(
      PlanNode node,
      List<ActualProperties> inputProperties,
      Metadata metadata,
      Session session,
      Map<Symbol, Type> types,
      SqlParser parser) {
    ActualProperties output =
        node.accept(new Visitor(metadata, session, types, parser), inputProperties);

    // TODO: ideally this logic would be somehow moved to PlanSanityChecker
    verify(
        node instanceof SemiJoinNode
            || inputProperties.stream().noneMatch(ActualProperties::isNullsReplicated)
            || output.isNullsReplicated(),
        "SemiJoinNode is the only node that can strip null replication");

    return output;
  }
 public static Expression extract(PlanNode node) {
   return node.accept(new EffectivePredicateExtractor(), null);
 }
Example #22
0
 @Override
 protected Void visitPlan(PlanNode node, Void context) {
   throw new UnsupportedOperationException(
       format("Node %s does not have a Graphviz visitor", node.getClass().getName()));
 }
Example #23
0
  @Override
  protected RelationPlan visitJoin(Join node, Void context) {
    // TODO: translate the RIGHT join into a mirrored LEFT join when we refactor (@martint)
    RelationPlan leftPlan = process(node.getLeft(), context);

    // Convert CROSS JOIN UNNEST to an UnnestNode
    if (node.getRight() instanceof Unnest
        || (node.getRight() instanceof AliasedRelation
            && ((AliasedRelation) node.getRight()).getRelation() instanceof Unnest)) {
      Unnest unnest;
      if (node.getRight() instanceof AliasedRelation) {
        unnest = (Unnest) ((AliasedRelation) node.getRight()).getRelation();
      } else {
        unnest = (Unnest) node.getRight();
      }
      if (node.getType() != Join.Type.CROSS && node.getType() != Join.Type.IMPLICIT) {
        throw new SemanticException(
            NOT_SUPPORTED, unnest, "UNNEST only supported on the right side of CROSS JOIN");
      }
      return planCrossJoinUnnest(leftPlan, node, unnest);
    }

    RelationPlan rightPlan = process(node.getRight(), context);

    PlanBuilder leftPlanBuilder = initializePlanBuilder(leftPlan);
    PlanBuilder rightPlanBuilder = initializePlanBuilder(rightPlan);

    TupleDescriptor outputDescriptor = analysis.getOutputDescriptor(node);

    // NOTE: symbols must be in the same order as the outputDescriptor
    List<Symbol> outputSymbols =
        ImmutableList.<Symbol>builder()
            .addAll(leftPlan.getOutputSymbols())
            .addAll(rightPlan.getOutputSymbols())
            .build();

    ImmutableList.Builder<JoinNode.EquiJoinClause> equiClauses = ImmutableList.builder();
    Expression postInnerJoinCriteria = new BooleanLiteral("TRUE");
    if (node.getType() != Join.Type.CROSS && node.getType() != Join.Type.IMPLICIT) {
      Expression criteria = analysis.getJoinCriteria(node);

      TupleDescriptor left = analysis.getOutputDescriptor(node.getLeft());
      TupleDescriptor right = analysis.getOutputDescriptor(node.getRight());
      List<Expression> leftExpressions = new ArrayList<>();
      List<Expression> rightExpressions = new ArrayList<>();
      List<ComparisonExpression.Type> comparisonTypes = new ArrayList<>();
      for (Expression conjunct : ExpressionUtils.extractConjuncts(criteria)) {
        if (!(conjunct instanceof ComparisonExpression)) {
          throw new SemanticException(
              NOT_SUPPORTED, node, "Unsupported non-equi join form: %s", conjunct);
        }

        ComparisonExpression comparison = (ComparisonExpression) conjunct;
        ComparisonExpression.Type comparisonType = comparison.getType();
        if (comparison.getType() != EQUAL && node.getType() != INNER) {
          throw new SemanticException(
              NOT_SUPPORTED, node, "Non-equi joins only supported for inner join: %s", conjunct);
        }
        Set<QualifiedName> firstDependencies =
            DependencyExtractor.extractNames(comparison.getLeft());
        Set<QualifiedName> secondDependencies =
            DependencyExtractor.extractNames(comparison.getRight());

        Expression leftExpression;
        Expression rightExpression;
        if (Iterables.all(firstDependencies, left.canResolvePredicate())
            && Iterables.all(secondDependencies, right.canResolvePredicate())) {
          leftExpression = comparison.getLeft();
          rightExpression = comparison.getRight();
        } else if (Iterables.all(firstDependencies, right.canResolvePredicate())
            && Iterables.all(secondDependencies, left.canResolvePredicate())) {
          leftExpression = comparison.getRight();
          rightExpression = comparison.getLeft();
          comparisonType = flipComparison(comparisonType);
        } else {
          // must have a complex expression that involves both tuples on one side of the comparison
          // expression (e.g., coalesce(left.x, right.x) = 1)
          throw new SemanticException(
              NOT_SUPPORTED, node, "Unsupported non-equi join form: %s", conjunct);
        }
        leftExpressions.add(leftExpression);
        rightExpressions.add(rightExpression);
        comparisonTypes.add(comparisonType);
      }

      Analysis.JoinInPredicates joinInPredicates = analysis.getJoinInPredicates(node);

      // Add semi joins if necessary
      if (joinInPredicates != null) {
        leftPlanBuilder = appendSemiJoins(leftPlanBuilder, joinInPredicates.getLeftInPredicates());
        rightPlanBuilder =
            appendSemiJoins(rightPlanBuilder, joinInPredicates.getRightInPredicates());
      }

      // Add projections for join criteria
      leftPlanBuilder = appendProjections(leftPlanBuilder, leftExpressions);
      rightPlanBuilder = appendProjections(rightPlanBuilder, rightExpressions);

      List<Expression> postInnerJoinComparisons = new ArrayList<>();
      for (int i = 0; i < comparisonTypes.size(); i++) {
        Symbol leftSymbol = leftPlanBuilder.translate(leftExpressions.get(i));
        Symbol rightSymbol = rightPlanBuilder.translate(rightExpressions.get(i));

        equiClauses.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol));

        Expression leftExpression = leftPlanBuilder.rewrite(leftExpressions.get(i));
        Expression rightExpression = rightPlanBuilder.rewrite(rightExpressions.get(i));
        postInnerJoinComparisons.add(
            new ComparisonExpression(comparisonTypes.get(i), leftExpression, rightExpression));
      }
      postInnerJoinCriteria = ExpressionUtils.and(postInnerJoinComparisons);
    }

    PlanNode root;
    if (node.getType() == INNER) {
      root =
          new JoinNode(
              idAllocator.getNextId(),
              JoinNode.Type.CROSS,
              leftPlanBuilder.getRoot(),
              rightPlanBuilder.getRoot(),
              ImmutableList.<JoinNode.EquiJoinClause>of(),
              Optional.empty(),
              Optional.empty());
      root = new FilterNode(idAllocator.getNextId(), root, postInnerJoinCriteria);
    } else {
      root =
          new JoinNode(
              idAllocator.getNextId(),
              JoinNode.Type.typeConvert(node.getType()),
              leftPlanBuilder.getRoot(),
              rightPlanBuilder.getRoot(),
              equiClauses.build(),
              Optional.empty(),
              Optional.empty());
    }
    Optional<Symbol> sampleWeight = Optional.empty();
    if (leftPlanBuilder.getSampleWeight().isPresent()
        || rightPlanBuilder.getSampleWeight().isPresent()) {
      Expression expression =
          new ArithmeticBinaryExpression(
              ArithmeticBinaryExpression.Type.MULTIPLY,
              oneIfNull(leftPlanBuilder.getSampleWeight()),
              oneIfNull(rightPlanBuilder.getSampleWeight()));
      sampleWeight = Optional.of(symbolAllocator.newSymbol(expression, BIGINT));
      ImmutableMap.Builder<Symbol, Expression> projections = ImmutableMap.builder();
      projections.put(sampleWeight.get(), expression);
      for (Symbol symbol : root.getOutputSymbols()) {
        projections.put(symbol, new QualifiedNameReference(symbol.toQualifiedName()));
      }
      root = new ProjectNode(idAllocator.getNextId(), root, projections.build());
    }

    return new RelationPlan(root, outputDescriptor, outputSymbols, sampleWeight);
  }
 @Override
 protected ActualProperties visitPlan(PlanNode node, List<ActualProperties> inputProperties) {
   throw new UnsupportedOperationException("not yet implemented: " + node.getClass().getName());
 }
Example #25
0
 public static Map<Symbol, Symbol> trace(PlanNode node, Set<Symbol> lookupSymbols) {
   return node.accept(new Visitor(), lookupSymbols);
 }
 @Override
 protected SubPlanBuilder visitPlan(PlanNode node, Void context) {
   throw new UnsupportedOperationException("not yet implemented: " + node.getClass().getName());
 }
Example #27
0
  @Override
  protected RelationPlan visitUnion(Union node, Void context) {
    checkArgument(!node.getRelations().isEmpty(), "No relations specified for UNION");

    List<Symbol> unionOutputSymbols = null;
    ImmutableList.Builder<PlanNode> sources = ImmutableList.builder();
    ImmutableListMultimap.Builder<Symbol, Symbol> symbolMapping = ImmutableListMultimap.builder();
    List<RelationPlan> subPlans =
        node.getRelations()
            .stream()
            .map(relation -> processAndCoerceIfNecessary(relation, context))
            .collect(toImmutableList());

    boolean hasSampleWeight = false;
    for (RelationPlan subPlan : subPlans) {
      if (subPlan.getSampleWeight().isPresent()) {
        hasSampleWeight = true;
        break;
      }
    }

    Optional<Symbol> outputSampleWeight = Optional.empty();
    for (RelationPlan relationPlan : subPlans) {
      if (hasSampleWeight && !relationPlan.getSampleWeight().isPresent()) {
        relationPlan = addConstantSampleWeight(relationPlan);
      }

      List<Symbol> childOutputSymbols = relationPlan.getOutputSymbols();
      if (unionOutputSymbols == null) {
        // Use the first Relation to derive output symbol names
        TupleDescriptor descriptor = relationPlan.getDescriptor();
        ImmutableList.Builder<Symbol> outputSymbolBuilder = ImmutableList.builder();
        for (Field field : descriptor.getVisibleFields()) {
          int fieldIndex = descriptor.indexOf(field);
          Symbol symbol = childOutputSymbols.get(fieldIndex);
          outputSymbolBuilder.add(
              symbolAllocator.newSymbol(symbol.getName(), symbolAllocator.getTypes().get(symbol)));
        }
        unionOutputSymbols = outputSymbolBuilder.build();
        outputSampleWeight = relationPlan.getSampleWeight();
      }

      TupleDescriptor descriptor = relationPlan.getDescriptor();
      checkArgument(
          descriptor.getVisibleFieldCount() == unionOutputSymbols.size(),
          "Expected relation to have %s symbols but has %s symbols",
          descriptor.getVisibleFieldCount(),
          unionOutputSymbols.size());

      int unionFieldId = 0;
      for (Field field : descriptor.getVisibleFields()) {
        int fieldIndex = descriptor.indexOf(field);
        symbolMapping.put(unionOutputSymbols.get(unionFieldId), childOutputSymbols.get(fieldIndex));
        unionFieldId++;
      }

      sources.add(relationPlan.getRoot());
    }

    PlanNode planNode =
        new UnionNode(idAllocator.getNextId(), sources.build(), symbolMapping.build());
    if (node.isDistinct()) {
      planNode = distinct(planNode);
    }
    return new RelationPlan(
        planNode,
        analysis.getOutputDescriptor(node),
        planNode.getOutputSymbols(),
        outputSampleWeight);
  }