private PlanNode planTableScan(TableScanNode node, Expression predicate) {
DomainTranslator.ExtractionResult decomposedPredicate =
DomainTranslator.fromPredicate(metadata, session, predicate, symbolAllocator.getTypes());
TupleDomain<ColumnHandle> simplifiedConstraint =
decomposedPredicate
.getTupleDomain()
.transform(node.getAssignments()::get)
.intersect(node.getCurrentConstraint());
List<TableLayoutResult> layouts =
metadata.getLayouts(
session,
node.getTable(),
new Constraint<>(simplifiedConstraint, bindings -> true),
Optional.of(ImmutableSet.copyOf(node.getAssignments().values())));
if (layouts.isEmpty()) {
return new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of());
}
TableLayoutResult layout = layouts.get(0);
TableScanNode result =
new TableScanNode(
node.getId(),
node.getTable(),
node.getOutputSymbols(),
node.getAssignments(),
Optional.of(layout.getLayout().getHandle()),
simplifiedConstraint.intersect(layout.getLayout().getPredicate()),
Optional.ofNullable(node.getOriginalConstraint()).orElse(predicate));
Map<ColumnHandle, Symbol> assignments =
ImmutableBiMap.copyOf(node.getAssignments()).inverse();
Expression resultingPredicate =
combineConjuncts(
decomposedPredicate.getRemainingExpression(),
DomainTranslator.toPredicate(
layout.getUnenforcedConstraint().transform(assignments::get)));
if (!BooleanLiteral.TRUE_LITERAL.equals(resultingPredicate)) {
return new FilterNode(idAllocator.getNextId(), result, resultingPredicate);
}
return result;
}
@Override
public PlanNode rewriteAggregation(
AggregationNode node,
Expression inheritedPredicate,
PlanRewriter<Expression> planRewriter) {
EqualityInference equalityInference = createEqualityInference(inheritedPredicate);
List<Expression> pushdownConjuncts = new ArrayList<>();
List<Expression> postAggregationConjuncts = new ArrayList<>();
// Strip out non-deterministic conjuncts
postAggregationConjuncts.addAll(
ImmutableList.copyOf(filter(extractConjuncts(inheritedPredicate), not(deterministic()))));
inheritedPredicate = stripNonDeterministicConjuncts(inheritedPredicate);
// Sort non-equality predicates by those that can be pushed down and those that cannot
for (Expression conjunct : EqualityInference.nonInferrableConjuncts(inheritedPredicate)) {
Expression rewrittenConjunct =
equalityInference.rewriteExpression(conjunct, in(node.getGroupBy()));
if (rewrittenConjunct != null) {
pushdownConjuncts.add(rewrittenConjunct);
} else {
postAggregationConjuncts.add(conjunct);
}
}
// Add the equality predicates back in
EqualityInference.EqualityPartition equalityPartition =
equalityInference.generateEqualitiesPartitionedBy(in(node.getGroupBy()));
pushdownConjuncts.addAll(equalityPartition.getScopeEqualities());
postAggregationConjuncts.addAll(equalityPartition.getScopeComplementEqualities());
postAggregationConjuncts.addAll(equalityPartition.getScopeStraddlingEqualities());
PlanNode rewrittenSource =
planRewriter.rewrite(node.getSource(), combineConjuncts(pushdownConjuncts));
PlanNode output = node;
if (rewrittenSource != node.getSource()) {
output =
new AggregationNode(
node.getId(),
rewrittenSource,
node.getGroupBy(),
node.getAggregations(),
node.getFunctions(),
node.getMasks(),
node.getStep(),
node.getSampleWeight(),
node.getConfidence());
}
if (!postAggregationConjuncts.isEmpty()) {
output =
new FilterNode(
idAllocator.getNextId(), output, combineConjuncts(postAggregationConjuncts));
}
return output;
}
@Override
public PlanNode rewriteNode(
PlanNode node, Expression inheritedPredicate, PlanRewriter<Expression> planRewriter) {
PlanNode rewrittenNode = planRewriter.defaultRewrite(node, BooleanLiteral.TRUE_LITERAL);
if (!inheritedPredicate.equals(BooleanLiteral.TRUE_LITERAL)) {
// Drop in a FilterNode b/c we cannot push our predicate down any further
rewrittenNode = new FilterNode(idAllocator.getNextId(), rewrittenNode, inheritedPredicate);
}
return rewrittenNode;
}
@Override
public PlanNode visitTableCommit(TableCommitNode node, RewriteContext<Void> context) {
Optional<DeleteNode> delete = findNode(node.getSource(), DeleteNode.class);
if (!delete.isPresent()) {
return context.defaultRewrite(node);
}
Optional<TableScanNode> tableScan = findNode(delete.get().getSource(), TableScanNode.class);
if (!tableScan.isPresent()) {
return context.defaultRewrite(node);
}
TableScanNode tableScanNode = tableScan.get();
if (!metadata.supportsMetadataDelete(
session, tableScanNode.getTable(), tableScanNode.getLayout().get())) {
return context.defaultRewrite(node);
}
return new MetadataDeleteNode(
idAllocator.getNextId(),
delete.get().getTarget(),
Iterables.getOnlyElement(node.getOutputSymbols()),
tableScanNode.getLayout().get());
}
private PlanNode planTableScan(TableScanNode node, Expression predicate, Context context) {
DomainTranslator.ExtractionResult decomposedPredicate =
DomainTranslator.fromPredicate(metadata, session, predicate, symbolAllocator.getTypes());
TupleDomain<ColumnHandle> simplifiedConstraint =
decomposedPredicate
.getTupleDomain()
.transform(node.getAssignments()::get)
.intersect(node.getCurrentConstraint());
checkState(node.getOutputSymbols().containsAll(context.getLookupSymbols()));
Set<ColumnHandle> lookupColumns =
context
.getLookupSymbols()
.stream()
.map(node.getAssignments()::get)
.collect(toImmutableSet());
Set<ColumnHandle> outputColumns =
node.getOutputSymbols()
.stream()
.map(node.getAssignments()::get)
.collect(toImmutableSet());
Optional<ResolvedIndex> optionalResolvedIndex =
metadata.resolveIndex(
session, node.getTable(), lookupColumns, outputColumns, simplifiedConstraint);
if (!optionalResolvedIndex.isPresent()) {
// No index available, so give up by returning something
return node;
}
ResolvedIndex resolvedIndex = optionalResolvedIndex.get();
Map<ColumnHandle, Symbol> inverseAssignments =
ImmutableBiMap.copyOf(node.getAssignments()).inverse();
PlanNode source =
new IndexSourceNode(
idAllocator.getNextId(),
resolvedIndex.getIndexHandle(),
node.getTable(),
node.getLayout(),
context.getLookupSymbols(),
node.getOutputSymbols(),
node.getAssignments(),
simplifiedConstraint);
Expression resultingPredicate =
combineConjuncts(
DomainTranslator.toPredicate(
resolvedIndex.getUnresolvedTupleDomain().transform(inverseAssignments::get)),
decomposedPredicate.getRemainingExpression());
if (!resultingPredicate.equals(TRUE_LITERAL)) {
// todo it is likely we end up with redundant filters here because the predicate push down
// has already been run... the fix is to run predicate push down again
source = new FilterNode(idAllocator.getNextId(), source, resultingPredicate);
}
context.markSuccess();
return source;
}
@Override
public PlanNode visitJoin(JoinNode node, RewriteContext<Void> context) {
PlanNode leftRewritten = context.rewrite(node.getLeft());
PlanNode rightRewritten = context.rewrite(node.getRight());
if (!node.getCriteria().isEmpty()) { // Index join only possible with JOIN criteria
List<Symbol> leftJoinSymbols =
Lists.transform(node.getCriteria(), JoinNode.EquiJoinClause::getLeft);
List<Symbol> rightJoinSymbols =
Lists.transform(node.getCriteria(), JoinNode.EquiJoinClause::getRight);
Optional<PlanNode> leftIndexCandidate =
IndexSourceRewriter.rewriteWithIndex(
leftRewritten,
ImmutableSet.copyOf(leftJoinSymbols),
symbolAllocator,
idAllocator,
metadata,
session);
if (leftIndexCandidate.isPresent()) {
// Sanity check that we can trace the path for the index lookup key
Map<Symbol, Symbol> trace =
IndexKeyTracer.trace(leftIndexCandidate.get(), ImmutableSet.copyOf(leftJoinSymbols));
checkState(!trace.isEmpty() && leftJoinSymbols.containsAll(trace.keySet()));
}
Optional<PlanNode> rightIndexCandidate =
IndexSourceRewriter.rewriteWithIndex(
rightRewritten,
ImmutableSet.copyOf(rightJoinSymbols),
symbolAllocator,
idAllocator,
metadata,
session);
if (rightIndexCandidate.isPresent()) {
// Sanity check that we can trace the path for the index lookup key
Map<Symbol, Symbol> trace =
IndexKeyTracer.trace(
rightIndexCandidate.get(), ImmutableSet.copyOf(rightJoinSymbols));
checkState(!trace.isEmpty() && rightJoinSymbols.containsAll(trace.keySet()));
}
switch (node.getType()) {
case INNER:
// Prefer the right candidate over the left candidate
IndexJoinNode indexJoinNode = null;
if (rightIndexCandidate.isPresent()) {
indexJoinNode =
new IndexJoinNode(
idAllocator.getNextId(),
IndexJoinNode.Type.INNER,
leftRewritten,
rightIndexCandidate.get(),
createEquiJoinClause(leftJoinSymbols, rightJoinSymbols),
Optional.empty(),
Optional.empty());
} else if (leftIndexCandidate.isPresent()) {
indexJoinNode =
new IndexJoinNode(
idAllocator.getNextId(),
IndexJoinNode.Type.INNER,
rightRewritten,
leftIndexCandidate.get(),
createEquiJoinClause(rightJoinSymbols, leftJoinSymbols),
Optional.empty(),
Optional.empty());
}
if (indexJoinNode != null) {
if (node.getFilter().isPresent()) {
return new FilterNode(
idAllocator.getNextId(), indexJoinNode, node.getFilter().get());
}
return indexJoinNode;
}
break;
case LEFT:
// We cannot use indices for outer joins until index join supports in-line filtering
if (!node.getFilter().isPresent() && rightIndexCandidate.isPresent()) {
return new IndexJoinNode(
idAllocator.getNextId(),
IndexJoinNode.Type.SOURCE_OUTER,
leftRewritten,
rightIndexCandidate.get(),
createEquiJoinClause(leftJoinSymbols, rightJoinSymbols),
Optional.empty(),
Optional.empty());
}
break;
case RIGHT:
// We cannot use indices for outer joins until index join supports in-line filtering
if (!node.getFilter().isPresent() && leftIndexCandidate.isPresent()) {
return new IndexJoinNode(
idAllocator.getNextId(),
IndexJoinNode.Type.SOURCE_OUTER,
rightRewritten,
leftIndexCandidate.get(),
createEquiJoinClause(rightJoinSymbols, leftJoinSymbols),
Optional.empty(),
Optional.empty());
}
break;
case FULL:
break;
default:
throw new IllegalArgumentException("Unknown type: " + node.getType());
}
}
if (leftRewritten != node.getLeft() || rightRewritten != node.getRight()) {
return new JoinNode(
node.getId(),
node.getType(),
leftRewritten,
rightRewritten,
node.getCriteria(),
node.getFilter(),
node.getLeftHashSymbol(),
node.getRightHashSymbol());
}
return node;
}
@Override
public PlanNode rewriteTableScan(
TableScanNode node, Expression inheritedPredicate, PlanRewriter<Expression> planRewriter) {
DomainTranslator.ExtractionResult extractionResult =
DomainTranslator.fromPredicate(
inheritedPredicate, symbolAllocator.getTypes(), node.getAssignments());
Expression extractionRemainingExpression = extractionResult.getRemainingExpression();
TupleDomain tupleDomain = extractionResult.getTupleDomain();
if (node.getGeneratedPartitions().isPresent()) {
// Add back in the TupleDomain that was used to generate the previous set of Partitions if
// present
// And just for kicks, throw in the domain summary too (as that can only help prune down the
// ranges)
// The domains should never widen between each pass.
tupleDomain =
tupleDomain
.intersect(node.getGeneratedPartitions().get().getTupleDomainInput())
.intersect(node.getPartitionsDomainSummary());
}
PartitionResult matchingPartitions =
splitManager.getPartitions(node.getTable(), Optional.of(tupleDomain));
List<Partition> partitions = matchingPartitions.getPartitions();
TupleDomain undeterminedTupleDomain = matchingPartitions.getUndeterminedTupleDomain();
Expression unevaluatedDomainPredicate =
DomainTranslator.toPredicate(
undeterminedTupleDomain, ImmutableBiMap.copyOf(node.getAssignments()).inverse());
// Construct the post scan predicate. Add the unevaluated TupleDomain back first since those
// are generally cheaper to evaluate than anything we can't extract
Expression postScanPredicate =
combineConjuncts(unevaluatedDomainPredicate, extractionRemainingExpression);
// Do some early partition pruning
partitions =
ImmutableList.copyOf(
filter(
partitions, not(shouldPrunePartition(postScanPredicate, node.getAssignments()))));
GeneratedPartitions generatedPartitions = new GeneratedPartitions(tupleDomain, partitions);
PlanNode output = node;
if (!node.getGeneratedPartitions().equals(Optional.of(generatedPartitions))) {
// Only overwrite the originalConstraint if it was previously null
Expression originalConstraint =
node.getOriginalConstraint() == null
? inheritedPredicate
: node.getOriginalConstraint();
output =
new TableScanNode(
node.getId(),
node.getTable(),
node.getOutputSymbols(),
node.getAssignments(),
originalConstraint,
Optional.of(generatedPartitions));
}
if (!postScanPredicate.equals(BooleanLiteral.TRUE_LITERAL)) {
output = new FilterNode(idAllocator.getNextId(), output, postScanPredicate);
}
return output;
}
@Override
public PlanNode rewriteSemiJoin(
SemiJoinNode node, Expression inheritedPredicate, PlanRewriter<Expression> planRewriter) {
Expression sourceEffectivePredicate = EffectivePredicateExtractor.extract(node.getSource());
List<Expression> sourceConjuncts = new ArrayList<>();
List<Expression> filteringSourceConjuncts = new ArrayList<>();
List<Expression> postJoinConjuncts = new ArrayList<>();
// TODO: see if there are predicates that can be inferred from the semi join output
// Push inherited and source predicates to filtering source via a contrived join predicate
// (but needs to avoid touching NULL values in the filtering source)
Expression joinPredicate =
equalsExpression(node.getSourceJoinSymbol(), node.getFilteringSourceJoinSymbol());
EqualityInference joinInference =
createEqualityInference(inheritedPredicate, sourceEffectivePredicate, joinPredicate);
for (Expression conjunct :
Iterables.concat(
EqualityInference.nonInferrableConjuncts(inheritedPredicate),
EqualityInference.nonInferrableConjuncts(sourceEffectivePredicate))) {
Expression rewrittenConjunct =
joinInference.rewriteExpression(conjunct, equalTo(node.getFilteringSourceJoinSymbol()));
if (rewrittenConjunct != null && DeterminismEvaluator.isDeterministic(rewrittenConjunct)) {
// Alter conjunct to include an OR filteringSourceJoinSymbol IS NULL disjunct
Expression rewrittenConjunctOrNull =
expressionOrNullSymbols(equalTo(node.getFilteringSourceJoinSymbol()))
.apply(rewrittenConjunct);
filteringSourceConjuncts.add(rewrittenConjunctOrNull);
}
}
EqualityInference.EqualityPartition joinInferenceEqualityPartition =
joinInference.generateEqualitiesPartitionedBy(
equalTo(node.getFilteringSourceJoinSymbol()));
filteringSourceConjuncts.addAll(
ImmutableList.copyOf(
transform(
joinInferenceEqualityPartition.getScopeEqualities(),
expressionOrNullSymbols(equalTo(node.getFilteringSourceJoinSymbol())))));
// Push inheritedPredicates down to the source if they don't involve the semi join output
EqualityInference inheritedInference = createEqualityInference(inheritedPredicate);
for (Expression conjunct : EqualityInference.nonInferrableConjuncts(inheritedPredicate)) {
Expression rewrittenConjunct =
inheritedInference.rewriteExpression(conjunct, in(node.getSource().getOutputSymbols()));
// Since each source row is reflected exactly once in the output, ok to push
// non-deterministic predicates down
if (rewrittenConjunct != null) {
sourceConjuncts.add(rewrittenConjunct);
} else {
postJoinConjuncts.add(conjunct);
}
}
// Add the inherited equality predicates back in
EqualityInference.EqualityPartition equalityPartition =
inheritedInference.generateEqualitiesPartitionedBy(
in(node.getSource().getOutputSymbols()));
sourceConjuncts.addAll(equalityPartition.getScopeEqualities());
postJoinConjuncts.addAll(equalityPartition.getScopeComplementEqualities());
postJoinConjuncts.addAll(equalityPartition.getScopeStraddlingEqualities());
PlanNode rewrittenSource =
planRewriter.rewrite(node.getSource(), combineConjuncts(sourceConjuncts));
PlanNode rewrittenFilteringSource =
planRewriter.rewrite(
node.getFilteringSource(), combineConjuncts(filteringSourceConjuncts));
PlanNode output = node;
if (rewrittenSource != node.getSource()
|| rewrittenFilteringSource != node.getFilteringSource()) {
output =
new SemiJoinNode(
node.getId(),
rewrittenSource,
rewrittenFilteringSource,
node.getSourceJoinSymbol(),
node.getFilteringSourceJoinSymbol(),
node.getSemiJoinOutput());
}
if (!postJoinConjuncts.isEmpty()) {
output =
new FilterNode(idAllocator.getNextId(), output, combineConjuncts(postJoinConjuncts));
}
return output;
}
@Override
public PlanNode rewriteJoin(
JoinNode node, Expression inheritedPredicate, PlanRewriter<Expression> planRewriter) {
boolean isCrossJoin = (node.getType() == JoinNode.Type.CROSS);
// See if we can rewrite outer joins in terms of a plain inner join
node = tryNormalizeToInnerJoin(node, inheritedPredicate);
Expression leftEffectivePredicate = EffectivePredicateExtractor.extract(node.getLeft());
Expression rightEffectivePredicate = EffectivePredicateExtractor.extract(node.getRight());
Expression joinPredicate = extractJoinPredicate(node);
Expression leftPredicate;
Expression rightPredicate;
Expression postJoinPredicate;
Expression newJoinPredicate;
switch (node.getType()) {
case INNER:
InnerJoinPushDownResult innerJoinPushDownResult =
processInnerJoin(
inheritedPredicate,
leftEffectivePredicate,
rightEffectivePredicate,
joinPredicate,
node.getLeft().getOutputSymbols());
leftPredicate = innerJoinPushDownResult.getLeftPredicate();
rightPredicate = innerJoinPushDownResult.getRightPredicate();
postJoinPredicate = innerJoinPushDownResult.getPostJoinPredicate();
newJoinPredicate = innerJoinPushDownResult.getJoinPredicate();
break;
case LEFT:
OuterJoinPushDownResult leftOuterJoinPushDownResult =
processOuterJoin(
inheritedPredicate,
leftEffectivePredicate,
rightEffectivePredicate,
joinPredicate,
node.getLeft().getOutputSymbols());
leftPredicate = leftOuterJoinPushDownResult.getOuterJoinPredicate();
rightPredicate = leftOuterJoinPushDownResult.getInnerJoinPredicate();
postJoinPredicate = leftOuterJoinPushDownResult.getPostJoinPredicate();
newJoinPredicate = joinPredicate; // Use the same as the original
break;
case RIGHT:
OuterJoinPushDownResult rightOuterJoinPushDownResult =
processOuterJoin(
inheritedPredicate,
rightEffectivePredicate,
leftEffectivePredicate,
joinPredicate,
node.getRight().getOutputSymbols());
leftPredicate = rightOuterJoinPushDownResult.getInnerJoinPredicate();
rightPredicate = rightOuterJoinPushDownResult.getOuterJoinPredicate();
postJoinPredicate = rightOuterJoinPushDownResult.getPostJoinPredicate();
newJoinPredicate = joinPredicate; // Use the same as the original
break;
default:
throw new UnsupportedOperationException("Unsupported join type: " + node.getType());
}
PlanNode leftSource = planRewriter.rewrite(node.getLeft(), leftPredicate);
PlanNode rightSource = planRewriter.rewrite(node.getRight(), rightPredicate);
PlanNode output = node;
if (leftSource != node.getLeft()
|| rightSource != node.getRight()
|| !newJoinPredicate.equals(joinPredicate)) {
List<JoinNode.EquiJoinClause> criteria = node.getCriteria();
// Rewrite criteria and add projections if there is a new join predicate
if (!newJoinPredicate.equals(joinPredicate) || isCrossJoin) {
// Create identity projections for all existing symbols
ImmutableMap.Builder<Symbol, Expression> leftProjections = ImmutableMap.builder();
leftProjections.putAll(
IterableTransformer.<Symbol>on(node.getLeft().getOutputSymbols())
.toMap(symbolToQualifiedNameReference())
.map());
ImmutableMap.Builder<Symbol, Expression> rightProjections = ImmutableMap.builder();
rightProjections.putAll(
IterableTransformer.<Symbol>on(node.getRight().getOutputSymbols())
.toMap(symbolToQualifiedNameReference())
.map());
// HACK! we don't support cross joins right now, so put in a simple fake join predicate
// instead if all of the join clauses got simplified out
// TODO: remove this code when cross join support is added
Iterable<Expression> simplifiedJoinConjuncts =
transform(extractConjuncts(newJoinPredicate), simplifyExpressions());
simplifiedJoinConjuncts =
filter(
simplifiedJoinConjuncts,
not(Predicates.<Expression>equalTo(BooleanLiteral.TRUE_LITERAL)));
if (Iterables.isEmpty(simplifiedJoinConjuncts)) {
simplifiedJoinConjuncts =
ImmutableList.<Expression>of(
new ComparisonExpression(
ComparisonExpression.Type.EQUAL,
new LongLiteral("0"),
new LongLiteral("0")));
}
// Create new projections for the new join clauses
ImmutableList.Builder<JoinNode.EquiJoinClause> builder = ImmutableList.builder();
for (Expression conjunct : simplifiedJoinConjuncts) {
checkState(
joinEqualityExpression(node.getLeft().getOutputSymbols()).apply(conjunct),
"Expected join predicate to be a valid join equality");
ComparisonExpression equality = (ComparisonExpression) conjunct;
boolean alignedComparison =
Iterables.all(
DependencyExtractor.extractUnique(equality.getLeft()),
in(node.getLeft().getOutputSymbols()));
Expression leftExpression =
(alignedComparison) ? equality.getLeft() : equality.getRight();
Expression rightExpression =
(alignedComparison) ? equality.getRight() : equality.getLeft();
Symbol leftSymbol =
symbolAllocator.newSymbol(leftExpression, extractType(leftExpression));
leftProjections.put(leftSymbol, leftExpression);
Symbol rightSymbol =
symbolAllocator.newSymbol(rightExpression, extractType(rightExpression));
rightProjections.put(rightSymbol, rightExpression);
builder.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol));
}
leftSource =
new ProjectNode(idAllocator.getNextId(), leftSource, leftProjections.build());
rightSource =
new ProjectNode(idAllocator.getNextId(), rightSource, rightProjections.build());
criteria = builder.build();
}
output = new JoinNode(node.getId(), node.getType(), leftSource, rightSource, criteria);
}
if (!postJoinPredicate.equals(BooleanLiteral.TRUE_LITERAL)) {
output = new FilterNode(idAllocator.getNextId(), output, postJoinPredicate);
}
return output;
}