@Override
public SubPlanBuilder visitUnion(UnionNode node, Void context) {
if (createSingleNodePlan) {
ImmutableList.Builder<PlanNode> sourceBuilder = ImmutableList.builder();
for (PlanNode source : node.getSources()) {
sourceBuilder.add(source.accept(this, context).getRoot());
}
UnionNode unionNode =
new UnionNode(node.getId(), sourceBuilder.build(), node.getSymbolMapping());
return createSingleNodePlan(unionNode);
} else {
ImmutableList.Builder<SubPlan> sourceBuilder = ImmutableList.builder();
ImmutableList.Builder<PlanFragmentId> fragmentIdBuilder = ImmutableList.builder();
for (int i = 0; i < node.getSources().size(); i++) {
PlanNode subPlan = node.getSources().get(i);
SubPlanBuilder current = subPlan.accept(this, context);
current.setRoot(
new SinkNode(idAllocator.getNextId(), current.getRoot(), node.sourceOutputLayout(i)));
fragmentIdBuilder.add(current.getId());
sourceBuilder.add(current.build());
}
ExchangeNode exchangeNode =
new ExchangeNode(
idAllocator.getNextId(), fragmentIdBuilder.build(), node.getOutputSymbols());
return createSingleNodePlan(exchangeNode).setChildren(sourceBuilder.build());
}
}
@Override
public Void visitUnion(UnionNode node, Void context) {
printNode(node, "Union", NODE_COLORS.get(NodeType.UNION));
for (PlanNode planNode : node.getSources()) {
planNode.accept(this, context);
}
return null;
}
private static void findSources(
PlanNode node, Builder<PlanNode> builder, PlanNodeId partitionedSource) {
for (PlanNode source : node.getSources()) {
findSources(source, builder, partitionedSource);
}
if (node.getSources().isEmpty() || node.getId().equals(partitionedSource)) {
builder.add(node);
}
}
@Override
protected Void visitPlan(PlanNode node, Void context) {
for (PlanNode child : node.getSources()) {
printEdge(node, child);
child.accept(this, context);
}
return null;
}
private static void printFragmentNodes(
StringBuilder output, PlanFragment fragment, PlanNodeIdGenerator idGenerator) {
String clusterId = "cluster_" + fragment.getId();
output.append("subgraph ").append(clusterId).append(" {").append('\n');
output.append(format("label = \"%s\"", fragment.getDistribution())).append('\n');
PlanNode plan = fragment.getRoot();
plan.accept(new NodePrinter(output, idGenerator), null);
output.append("}").append('\n');
}
@Override
public PlanNode visitProject(ProjectNode node, RewriteContext<Void> context) {
PlanNode source = context.rewrite(node.getSource());
if (!node.getOutputSymbols().equals(source.getOutputSymbols())) {
// Can't get rid of this projection. It constrains the output tuple from the underlying
// operator
return replaceChildren(node, ImmutableList.of(source));
}
if (node.isIdentity()) {
return source;
}
return replaceChildren(node, ImmutableList.of(source));
}
@Override
public Void visitExchange(ExchangeNode node, Void context) {
List<Symbol> symbols = node.getOutputSymbols();
if (node.getType() == REPARTITION || node.getType() == REPARTITION_WITH_NULL_REPLICATION) {
symbols = node.getPartitionKeys().orElseGet(() -> ImmutableList.of(new Symbol("(absent)")));
}
String columns = Joiner.on(", ").join(symbols);
printNode(
node,
format("ExchangeNode[%s]", node.getType()),
columns,
NODE_COLORS.get(NodeType.EXCHANGE));
for (PlanNode planNode : node.getSources()) {
planNode.accept(this, context);
}
return null;
}
private PlanNode removeAllRecursive(PlanNode node) {
if (where.test(node)) {
checkArgument(
node.getSources().size() == 1,
"Unable to remove plan node as it contains 0 or more than 1 children");
return node.getSources().get(0);
}
if (skipOnly.test(node)) {
List<PlanNode> sources =
node.getSources()
.stream()
.map(source -> removeAllRecursive(source))
.collect(toImmutableList());
return replaceChildren(node, sources);
}
return node;
}
public static ActualProperties streamBackdoorDeriveProperties(
PlanNode node,
List<ActualProperties> inputProperties,
Metadata metadata,
Session session,
Map<Symbol, Type> types,
SqlParser parser) {
return node.accept(new Visitor(metadata, session, types, parser), inputProperties);
}
private static void findTableScanNodes(
PlanNode node, ImmutableList.Builder<TableScanNode> builder) {
for (PlanNode source : node.getSources()) {
findTableScanNodes(source, builder);
}
if (node instanceof TableScanNode) {
builder.add((TableScanNode) node);
}
}
@Override
public SubPlanBuilder visitMarkDistinct(MarkDistinctNode node, Void context) {
SubPlanBuilder current = node.getSource().accept(this, context);
// Check if the subplan is already partitioned the way we want it
boolean alreadyPartitioned = false;
if (current.getDistribution() == PlanDistribution.FIXED) {
for (SubPlan child : current.getChildren()) {
if (child.getFragment().getOutputPartitioning() == OutputPartitioning.HASH
&& ImmutableSet.copyOf(child.getFragment().getPartitionBy())
.equals(ImmutableSet.copyOf(node.getDistinctSymbols()))) {
alreadyPartitioned = true;
break;
}
}
}
if (createSingleNodePlan || alreadyPartitioned || !current.isDistributed()) {
MarkDistinctNode markNode =
new MarkDistinctNode(
idAllocator.getNextId(),
current.getRoot(),
node.getMarkerSymbol(),
node.getDistinctSymbols(),
node.getSampleWeightSymbol());
current.setRoot(markNode);
return current;
} else {
PlanNode sink =
new SinkNode(
idAllocator.getNextId(), current.getRoot(), current.getRoot().getOutputSymbols());
current.setRoot(sink).setHashOutputPartitioning(node.getDistinctSymbols());
PlanNode exchange =
new ExchangeNode(idAllocator.getNextId(), current.getId(), sink.getOutputSymbols());
MarkDistinctNode markNode =
new MarkDistinctNode(
idAllocator.getNextId(),
exchange,
node.getMarkerSymbol(),
node.getDistinctSymbols(),
node.getSampleWeightSymbol());
return createFixedDistributionPlan(markNode).addChild(current.build());
}
}
@JsonCreator
public PlanFragment(
@JsonProperty("id") PlanFragmentId id,
@JsonProperty("root") PlanNode root,
@JsonProperty("symbols") Map<Symbol, Type> symbols,
@JsonProperty("distribution") PlanDistribution distribution,
@JsonProperty("partitionedSource") PlanNodeId partitionedSource,
@JsonProperty("outputPartitioning") OutputPartitioning outputPartitioning) {
this.id = checkNotNull(id, "id is null");
this.root = checkNotNull(root, "root is null");
this.symbols = checkNotNull(symbols, "symbols is null");
this.distribution = checkNotNull(distribution, "distribution is null");
this.partitionedSource = partitionedSource;
tupleInfos =
IterableTransformer.on(root.getOutputSymbols())
.transform(Functions.forMap(symbols))
.transform(Type.toRaw())
.transform(
new Function<TupleInfo.Type, TupleInfo>() {
@Override
public TupleInfo apply(TupleInfo.Type input) {
return new TupleInfo(input);
}
})
.list();
ImmutableList.Builder<PlanNode> sources = ImmutableList.builder();
findSources(root, sources, partitionedSource);
this.sources = sources.build();
ImmutableSet.Builder<PlanNodeId> sourceIds = ImmutableSet.builder();
for (PlanNode source : this.sources) {
sourceIds.add(source.getId());
}
if (partitionedSource != null) {
sourceIds.add(partitionedSource);
}
this.sourceIds = sourceIds.build();
this.outputPartitioning = checkNotNull(outputPartitioning, "outputPartitioning is null");
}
private PlanNode removeFirstRecursive(PlanNode node) {
if (where.test(node)) {
checkArgument(
node.getSources().size() == 1,
"Unable to remove plan node as it contains 0 or more than 1 children");
return node.getSources().get(0);
}
if (skipOnly.test(node)) {
List<PlanNode> sources = node.getSources();
if (sources.isEmpty()) {
return node;
} else if (sources.size() == 1) {
return replaceChildren(node, ImmutableList.of(removeFirstRecursive(sources.get(0))));
} else {
throw new IllegalArgumentException(
"Unable to remove first node when a node has multiple children, use removeAll instead");
}
}
return node;
}
private <T extends PlanNode> void findAllRecursive(
PlanNode node, ImmutableList.Builder<T> nodes) {
if (where.test(node)) {
nodes.add((T) node);
}
if (skipOnly.test(node)) {
for (PlanNode source : node.getSources()) {
findAllRecursive(source, nodes);
}
}
}
private PlanNode distinct(PlanNode node) {
return new AggregationNode(
idAllocator.getNextId(),
node,
node.getOutputSymbols(),
ImmutableMap.<Symbol, FunctionCall>of(),
ImmutableMap.<Symbol, Signature>of(),
ImmutableMap.<Symbol, Symbol>of(),
AggregationNode.Step.SINGLE,
Optional.empty(),
1.0,
Optional.empty());
}
private <T extends PlanNode> Optional<T> findFirstRecursive(PlanNode node) {
if (where.test(node)) {
return Optional.of((T) node);
}
if (skipOnly.test(node)) {
for (PlanNode source : node.getSources()) {
Optional<T> found = findFirstRecursive(source);
if (found.isPresent()) {
return found;
}
}
}
return Optional.empty();
}
private PlanNode replaceFirstRecursive(PlanNode node, PlanNode nodeToReplace) {
if (where.test(node)) {
return nodeToReplace;
}
List<PlanNode> sources = node.getSources();
if (sources.isEmpty()) {
return node;
} else if (sources.size() == 1) {
return replaceChildren(node, ImmutableList.of(replaceFirstRecursive(node, sources.get(0))));
} else {
throw new IllegalArgumentException(
"Unable to replace first node when a node has multiple children, use replaceAll instead");
}
}
private PlanNode replaceAllRecursive(PlanNode node, PlanNode nodeToReplace) {
if (where.test(node)) {
return nodeToReplace;
}
if (skipOnly.test(node)) {
List<PlanNode> sources =
node.getSources()
.stream()
.map(source -> replaceAllRecursive(source, nodeToReplace))
.collect(toImmutableList());
return replaceChildren(node, sources);
}
return node;
}
private <T> Optional<T> findNode(PlanNode source, Class<T> clazz) {
while (true) {
// allow any chain of linear exchanges
if (source instanceof ExchangeNode) {
List<PlanNode> sources = source.getSources();
if (sources.size() != 1) {
return Optional.empty();
}
source = sources.get(0);
} else if (clazz.isInstance(source)) {
return Optional.of(clazz.cast(source));
} else {
return Optional.empty();
}
}
}
public static ActualProperties deriveProperties(
PlanNode node,
List<ActualProperties> inputProperties,
Metadata metadata,
Session session,
Map<Symbol, Type> types,
SqlParser parser) {
ActualProperties output =
node.accept(new Visitor(metadata, session, types, parser), inputProperties);
// TODO: ideally this logic would be somehow moved to PlanSanityChecker
verify(
node instanceof SemiJoinNode
|| inputProperties.stream().noneMatch(ActualProperties::isNullsReplicated)
|| output.isNullsReplicated(),
"SemiJoinNode is the only node that can strip null replication");
return output;
}
public static Expression extract(PlanNode node) {
return node.accept(new EffectivePredicateExtractor(), null);
}
@Override
protected Void visitPlan(PlanNode node, Void context) {
throw new UnsupportedOperationException(
format("Node %s does not have a Graphviz visitor", node.getClass().getName()));
}
@Override
protected RelationPlan visitJoin(Join node, Void context) {
// TODO: translate the RIGHT join into a mirrored LEFT join when we refactor (@martint)
RelationPlan leftPlan = process(node.getLeft(), context);
// Convert CROSS JOIN UNNEST to an UnnestNode
if (node.getRight() instanceof Unnest
|| (node.getRight() instanceof AliasedRelation
&& ((AliasedRelation) node.getRight()).getRelation() instanceof Unnest)) {
Unnest unnest;
if (node.getRight() instanceof AliasedRelation) {
unnest = (Unnest) ((AliasedRelation) node.getRight()).getRelation();
} else {
unnest = (Unnest) node.getRight();
}
if (node.getType() != Join.Type.CROSS && node.getType() != Join.Type.IMPLICIT) {
throw new SemanticException(
NOT_SUPPORTED, unnest, "UNNEST only supported on the right side of CROSS JOIN");
}
return planCrossJoinUnnest(leftPlan, node, unnest);
}
RelationPlan rightPlan = process(node.getRight(), context);
PlanBuilder leftPlanBuilder = initializePlanBuilder(leftPlan);
PlanBuilder rightPlanBuilder = initializePlanBuilder(rightPlan);
TupleDescriptor outputDescriptor = analysis.getOutputDescriptor(node);
// NOTE: symbols must be in the same order as the outputDescriptor
List<Symbol> outputSymbols =
ImmutableList.<Symbol>builder()
.addAll(leftPlan.getOutputSymbols())
.addAll(rightPlan.getOutputSymbols())
.build();
ImmutableList.Builder<JoinNode.EquiJoinClause> equiClauses = ImmutableList.builder();
Expression postInnerJoinCriteria = new BooleanLiteral("TRUE");
if (node.getType() != Join.Type.CROSS && node.getType() != Join.Type.IMPLICIT) {
Expression criteria = analysis.getJoinCriteria(node);
TupleDescriptor left = analysis.getOutputDescriptor(node.getLeft());
TupleDescriptor right = analysis.getOutputDescriptor(node.getRight());
List<Expression> leftExpressions = new ArrayList<>();
List<Expression> rightExpressions = new ArrayList<>();
List<ComparisonExpression.Type> comparisonTypes = new ArrayList<>();
for (Expression conjunct : ExpressionUtils.extractConjuncts(criteria)) {
if (!(conjunct instanceof ComparisonExpression)) {
throw new SemanticException(
NOT_SUPPORTED, node, "Unsupported non-equi join form: %s", conjunct);
}
ComparisonExpression comparison = (ComparisonExpression) conjunct;
ComparisonExpression.Type comparisonType = comparison.getType();
if (comparison.getType() != EQUAL && node.getType() != INNER) {
throw new SemanticException(
NOT_SUPPORTED, node, "Non-equi joins only supported for inner join: %s", conjunct);
}
Set<QualifiedName> firstDependencies =
DependencyExtractor.extractNames(comparison.getLeft());
Set<QualifiedName> secondDependencies =
DependencyExtractor.extractNames(comparison.getRight());
Expression leftExpression;
Expression rightExpression;
if (Iterables.all(firstDependencies, left.canResolvePredicate())
&& Iterables.all(secondDependencies, right.canResolvePredicate())) {
leftExpression = comparison.getLeft();
rightExpression = comparison.getRight();
} else if (Iterables.all(firstDependencies, right.canResolvePredicate())
&& Iterables.all(secondDependencies, left.canResolvePredicate())) {
leftExpression = comparison.getRight();
rightExpression = comparison.getLeft();
comparisonType = flipComparison(comparisonType);
} else {
// must have a complex expression that involves both tuples on one side of the comparison
// expression (e.g., coalesce(left.x, right.x) = 1)
throw new SemanticException(
NOT_SUPPORTED, node, "Unsupported non-equi join form: %s", conjunct);
}
leftExpressions.add(leftExpression);
rightExpressions.add(rightExpression);
comparisonTypes.add(comparisonType);
}
Analysis.JoinInPredicates joinInPredicates = analysis.getJoinInPredicates(node);
// Add semi joins if necessary
if (joinInPredicates != null) {
leftPlanBuilder = appendSemiJoins(leftPlanBuilder, joinInPredicates.getLeftInPredicates());
rightPlanBuilder =
appendSemiJoins(rightPlanBuilder, joinInPredicates.getRightInPredicates());
}
// Add projections for join criteria
leftPlanBuilder = appendProjections(leftPlanBuilder, leftExpressions);
rightPlanBuilder = appendProjections(rightPlanBuilder, rightExpressions);
List<Expression> postInnerJoinComparisons = new ArrayList<>();
for (int i = 0; i < comparisonTypes.size(); i++) {
Symbol leftSymbol = leftPlanBuilder.translate(leftExpressions.get(i));
Symbol rightSymbol = rightPlanBuilder.translate(rightExpressions.get(i));
equiClauses.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol));
Expression leftExpression = leftPlanBuilder.rewrite(leftExpressions.get(i));
Expression rightExpression = rightPlanBuilder.rewrite(rightExpressions.get(i));
postInnerJoinComparisons.add(
new ComparisonExpression(comparisonTypes.get(i), leftExpression, rightExpression));
}
postInnerJoinCriteria = ExpressionUtils.and(postInnerJoinComparisons);
}
PlanNode root;
if (node.getType() == INNER) {
root =
new JoinNode(
idAllocator.getNextId(),
JoinNode.Type.CROSS,
leftPlanBuilder.getRoot(),
rightPlanBuilder.getRoot(),
ImmutableList.<JoinNode.EquiJoinClause>of(),
Optional.empty(),
Optional.empty());
root = new FilterNode(idAllocator.getNextId(), root, postInnerJoinCriteria);
} else {
root =
new JoinNode(
idAllocator.getNextId(),
JoinNode.Type.typeConvert(node.getType()),
leftPlanBuilder.getRoot(),
rightPlanBuilder.getRoot(),
equiClauses.build(),
Optional.empty(),
Optional.empty());
}
Optional<Symbol> sampleWeight = Optional.empty();
if (leftPlanBuilder.getSampleWeight().isPresent()
|| rightPlanBuilder.getSampleWeight().isPresent()) {
Expression expression =
new ArithmeticBinaryExpression(
ArithmeticBinaryExpression.Type.MULTIPLY,
oneIfNull(leftPlanBuilder.getSampleWeight()),
oneIfNull(rightPlanBuilder.getSampleWeight()));
sampleWeight = Optional.of(symbolAllocator.newSymbol(expression, BIGINT));
ImmutableMap.Builder<Symbol, Expression> projections = ImmutableMap.builder();
projections.put(sampleWeight.get(), expression);
for (Symbol symbol : root.getOutputSymbols()) {
projections.put(symbol, new QualifiedNameReference(symbol.toQualifiedName()));
}
root = new ProjectNode(idAllocator.getNextId(), root, projections.build());
}
return new RelationPlan(root, outputDescriptor, outputSymbols, sampleWeight);
}
@Override
protected ActualProperties visitPlan(PlanNode node, List<ActualProperties> inputProperties) {
throw new UnsupportedOperationException("not yet implemented: " + node.getClass().getName());
}
public static Map<Symbol, Symbol> trace(PlanNode node, Set<Symbol> lookupSymbols) {
return node.accept(new Visitor(), lookupSymbols);
}
@Override
protected SubPlanBuilder visitPlan(PlanNode node, Void context) {
throw new UnsupportedOperationException("not yet implemented: " + node.getClass().getName());
}
@Override
protected RelationPlan visitUnion(Union node, Void context) {
checkArgument(!node.getRelations().isEmpty(), "No relations specified for UNION");
List<Symbol> unionOutputSymbols = null;
ImmutableList.Builder<PlanNode> sources = ImmutableList.builder();
ImmutableListMultimap.Builder<Symbol, Symbol> symbolMapping = ImmutableListMultimap.builder();
List<RelationPlan> subPlans =
node.getRelations()
.stream()
.map(relation -> processAndCoerceIfNecessary(relation, context))
.collect(toImmutableList());
boolean hasSampleWeight = false;
for (RelationPlan subPlan : subPlans) {
if (subPlan.getSampleWeight().isPresent()) {
hasSampleWeight = true;
break;
}
}
Optional<Symbol> outputSampleWeight = Optional.empty();
for (RelationPlan relationPlan : subPlans) {
if (hasSampleWeight && !relationPlan.getSampleWeight().isPresent()) {
relationPlan = addConstantSampleWeight(relationPlan);
}
List<Symbol> childOutputSymbols = relationPlan.getOutputSymbols();
if (unionOutputSymbols == null) {
// Use the first Relation to derive output symbol names
TupleDescriptor descriptor = relationPlan.getDescriptor();
ImmutableList.Builder<Symbol> outputSymbolBuilder = ImmutableList.builder();
for (Field field : descriptor.getVisibleFields()) {
int fieldIndex = descriptor.indexOf(field);
Symbol symbol = childOutputSymbols.get(fieldIndex);
outputSymbolBuilder.add(
symbolAllocator.newSymbol(symbol.getName(), symbolAllocator.getTypes().get(symbol)));
}
unionOutputSymbols = outputSymbolBuilder.build();
outputSampleWeight = relationPlan.getSampleWeight();
}
TupleDescriptor descriptor = relationPlan.getDescriptor();
checkArgument(
descriptor.getVisibleFieldCount() == unionOutputSymbols.size(),
"Expected relation to have %s symbols but has %s symbols",
descriptor.getVisibleFieldCount(),
unionOutputSymbols.size());
int unionFieldId = 0;
for (Field field : descriptor.getVisibleFields()) {
int fieldIndex = descriptor.indexOf(field);
symbolMapping.put(unionOutputSymbols.get(unionFieldId), childOutputSymbols.get(fieldIndex));
unionFieldId++;
}
sources.add(relationPlan.getRoot());
}
PlanNode planNode =
new UnionNode(idAllocator.getNextId(), sources.build(), symbolMapping.build());
if (node.isDistinct()) {
planNode = distinct(planNode);
}
return new RelationPlan(
planNode,
analysis.getOutputDescriptor(node),
planNode.getOutputSymbols(),
outputSampleWeight);
}