private PlanBuilder appendProjections(PlanBuilder subPlan, Iterable<Expression> expressions) {
TranslationMap translations = new TranslationMap(subPlan.getRelationPlan(), analysis);
// Carry over the translations from the source because we are appending projections
translations.copyMappingsFrom(subPlan.getTranslations());
ImmutableMap.Builder<Symbol, Expression> projections = ImmutableMap.builder();
// add an identity projection for underlying plan
for (Symbol symbol : subPlan.getRoot().getOutputSymbols()) {
Expression expression = new QualifiedNameReference(symbol.toQualifiedName());
projections.put(symbol, expression);
}
ImmutableMap.Builder<Symbol, Expression> newTranslations = ImmutableMap.builder();
for (Expression expression : expressions) {
Symbol symbol = symbolAllocator.newSymbol(expression, analysis.getType(expression));
// TODO: CHECK IF THE REWRITE OF A SEMI JOINED EXPRESSION WILL WORK!!!!!!!
projections.put(symbol, translations.rewrite(expression));
newTranslations.put(symbol, expression);
}
// Now append the new translations into the TranslationMap
for (Map.Entry<Symbol, Expression> entry : newTranslations.build().entrySet()) {
translations.put(entry.getValue(), entry.getKey());
}
return new PlanBuilder(
translations,
new ProjectNode(idAllocator.getNextId(), subPlan.getRoot(), projections.build()),
subPlan.getSampleWeight());
}
private RelationPlan processAndCoerceIfNecessary(Relation node, Void context) {
Type[] coerceToTypes = analysis.getRelationCoercion(node);
RelationPlan plan = node.accept(this, context);
if (coerceToTypes == null) {
return plan;
}
List<Symbol> oldSymbols = plan.getOutputSymbols();
TupleDescriptor oldDescriptor = plan.getDescriptor().withOnlyVisibleFields();
verify(coerceToTypes.length == oldSymbols.size());
ImmutableList.Builder<Symbol> newSymbols = new ImmutableList.Builder<>();
Field[] newFields = new Field[coerceToTypes.length];
ImmutableMap.Builder<Symbol, Expression> assignments = new ImmutableMap.Builder<>();
for (int i = 0; i < coerceToTypes.length; i++) {
Symbol inputSymbol = oldSymbols.get(i);
Type inputType = symbolAllocator.getTypes().get(inputSymbol);
Type outputType = coerceToTypes[i];
if (outputType != inputType) {
Cast cast =
new Cast(
new QualifiedNameReference(inputSymbol.toQualifiedName()),
outputType.getTypeSignature().toString());
Symbol outputSymbol = symbolAllocator.newSymbol(cast, outputType);
assignments.put(outputSymbol, cast);
newSymbols.add(outputSymbol);
} else {
assignments.put(inputSymbol, new QualifiedNameReference(inputSymbol.toQualifiedName()));
newSymbols.add(inputSymbol);
}
Field oldField = oldDescriptor.getFieldByIndex(i);
newFields[i] =
new Field(
oldField.getRelationAlias(),
oldField.getName(),
coerceToTypes[i],
oldField.isHidden());
}
ProjectNode projectNode =
new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
return new RelationPlan(
projectNode, new TupleDescriptor(newFields), newSymbols.build(), plan.getSampleWeight());
}
private RelationPlan addConstantSampleWeight(RelationPlan subPlan) {
ImmutableMap.Builder<Symbol, Expression> projections = ImmutableMap.builder();
for (Symbol symbol : subPlan.getOutputSymbols()) {
Expression expression = new QualifiedNameReference(symbol.toQualifiedName());
projections.put(symbol, expression);
}
Expression one = new LongLiteral("1");
Symbol sampleWeightSymbol = symbolAllocator.newSymbol("$sampleWeight", BIGINT);
projections.put(sampleWeightSymbol, one);
ProjectNode projectNode =
new ProjectNode(idAllocator.getNextId(), subPlan.getRoot(), projections.build());
return new RelationPlan(
projectNode,
subPlan.getDescriptor(),
projectNode.getOutputSymbols(),
Optional.of(sampleWeightSymbol));
}
private SubPlanBuilder addDistributedAggregation(
SubPlanBuilder plan,
Map<Symbol, FunctionCall> aggregations,
Map<Symbol, Signature> functions,
Map<Symbol, Symbol> masks,
List<Symbol> groupBy,
Optional<Symbol> sampleWeight,
double confidence) {
Map<Symbol, FunctionCall> finalCalls = new HashMap<>();
Map<Symbol, FunctionCall> intermediateCalls = new HashMap<>();
Map<Symbol, Signature> intermediateFunctions = new HashMap<>();
Map<Symbol, Symbol> intermediateMask = new HashMap<>();
for (Map.Entry<Symbol, FunctionCall> entry : aggregations.entrySet()) {
Signature signature = functions.get(entry.getKey());
FunctionInfo function = metadata.getFunction(signature);
Symbol intermediateSymbol =
allocator.newSymbol(function.getName().getSuffix(), function.getIntermediateType());
intermediateCalls.put(intermediateSymbol, entry.getValue());
intermediateFunctions.put(intermediateSymbol, signature);
if (masks.containsKey(entry.getKey())) {
intermediateMask.put(intermediateSymbol, masks.get(entry.getKey()));
}
// rewrite final aggregation in terms of intermediate function
finalCalls.put(
entry.getKey(),
new FunctionCall(
function.getName(),
ImmutableList.<Expression>of(
new QualifiedNameReference(intermediateSymbol.toQualifiedName()))));
}
// create partial aggregation plan
AggregationNode partialAggregation =
new AggregationNode(
idAllocator.getNextId(),
plan.getRoot(),
groupBy,
intermediateCalls,
intermediateFunctions,
intermediateMask,
PARTIAL,
sampleWeight,
confidence);
plan.setRoot(
new SinkNode(
idAllocator.getNextId(), partialAggregation, partialAggregation.getOutputSymbols()));
// create final aggregation plan
ExchangeNode source =
new ExchangeNode(
idAllocator.getNextId(), plan.getId(), plan.getRoot().getOutputSymbols());
AggregationNode finalAggregation =
new AggregationNode(
idAllocator.getNextId(),
source,
groupBy,
finalCalls,
functions,
ImmutableMap.<Symbol, Symbol>of(),
FINAL,
Optional.<Symbol>absent(),
confidence);
if (groupBy.isEmpty()) {
plan = createSingleNodePlan(finalAggregation).addChild(plan.build());
} else {
plan.setHashOutputPartitioning(groupBy);
plan = createFixedDistributionPlan(finalAggregation).addChild(plan.build());
}
return plan;
}
@Override
protected RelationPlan visitJoin(Join node, Void context) {
// TODO: translate the RIGHT join into a mirrored LEFT join when we refactor (@martint)
RelationPlan leftPlan = process(node.getLeft(), context);
// Convert CROSS JOIN UNNEST to an UnnestNode
if (node.getRight() instanceof Unnest
|| (node.getRight() instanceof AliasedRelation
&& ((AliasedRelation) node.getRight()).getRelation() instanceof Unnest)) {
Unnest unnest;
if (node.getRight() instanceof AliasedRelation) {
unnest = (Unnest) ((AliasedRelation) node.getRight()).getRelation();
} else {
unnest = (Unnest) node.getRight();
}
if (node.getType() != Join.Type.CROSS && node.getType() != Join.Type.IMPLICIT) {
throw new SemanticException(
NOT_SUPPORTED, unnest, "UNNEST only supported on the right side of CROSS JOIN");
}
return planCrossJoinUnnest(leftPlan, node, unnest);
}
RelationPlan rightPlan = process(node.getRight(), context);
PlanBuilder leftPlanBuilder = initializePlanBuilder(leftPlan);
PlanBuilder rightPlanBuilder = initializePlanBuilder(rightPlan);
TupleDescriptor outputDescriptor = analysis.getOutputDescriptor(node);
// NOTE: symbols must be in the same order as the outputDescriptor
List<Symbol> outputSymbols =
ImmutableList.<Symbol>builder()
.addAll(leftPlan.getOutputSymbols())
.addAll(rightPlan.getOutputSymbols())
.build();
ImmutableList.Builder<JoinNode.EquiJoinClause> equiClauses = ImmutableList.builder();
Expression postInnerJoinCriteria = new BooleanLiteral("TRUE");
if (node.getType() != Join.Type.CROSS && node.getType() != Join.Type.IMPLICIT) {
Expression criteria = analysis.getJoinCriteria(node);
TupleDescriptor left = analysis.getOutputDescriptor(node.getLeft());
TupleDescriptor right = analysis.getOutputDescriptor(node.getRight());
List<Expression> leftExpressions = new ArrayList<>();
List<Expression> rightExpressions = new ArrayList<>();
List<ComparisonExpression.Type> comparisonTypes = new ArrayList<>();
for (Expression conjunct : ExpressionUtils.extractConjuncts(criteria)) {
if (!(conjunct instanceof ComparisonExpression)) {
throw new SemanticException(
NOT_SUPPORTED, node, "Unsupported non-equi join form: %s", conjunct);
}
ComparisonExpression comparison = (ComparisonExpression) conjunct;
ComparisonExpression.Type comparisonType = comparison.getType();
if (comparison.getType() != EQUAL && node.getType() != INNER) {
throw new SemanticException(
NOT_SUPPORTED, node, "Non-equi joins only supported for inner join: %s", conjunct);
}
Set<QualifiedName> firstDependencies =
DependencyExtractor.extractNames(comparison.getLeft());
Set<QualifiedName> secondDependencies =
DependencyExtractor.extractNames(comparison.getRight());
Expression leftExpression;
Expression rightExpression;
if (Iterables.all(firstDependencies, left.canResolvePredicate())
&& Iterables.all(secondDependencies, right.canResolvePredicate())) {
leftExpression = comparison.getLeft();
rightExpression = comparison.getRight();
} else if (Iterables.all(firstDependencies, right.canResolvePredicate())
&& Iterables.all(secondDependencies, left.canResolvePredicate())) {
leftExpression = comparison.getRight();
rightExpression = comparison.getLeft();
comparisonType = flipComparison(comparisonType);
} else {
// must have a complex expression that involves both tuples on one side of the comparison
// expression (e.g., coalesce(left.x, right.x) = 1)
throw new SemanticException(
NOT_SUPPORTED, node, "Unsupported non-equi join form: %s", conjunct);
}
leftExpressions.add(leftExpression);
rightExpressions.add(rightExpression);
comparisonTypes.add(comparisonType);
}
Analysis.JoinInPredicates joinInPredicates = analysis.getJoinInPredicates(node);
// Add semi joins if necessary
if (joinInPredicates != null) {
leftPlanBuilder = appendSemiJoins(leftPlanBuilder, joinInPredicates.getLeftInPredicates());
rightPlanBuilder =
appendSemiJoins(rightPlanBuilder, joinInPredicates.getRightInPredicates());
}
// Add projections for join criteria
leftPlanBuilder = appendProjections(leftPlanBuilder, leftExpressions);
rightPlanBuilder = appendProjections(rightPlanBuilder, rightExpressions);
List<Expression> postInnerJoinComparisons = new ArrayList<>();
for (int i = 0; i < comparisonTypes.size(); i++) {
Symbol leftSymbol = leftPlanBuilder.translate(leftExpressions.get(i));
Symbol rightSymbol = rightPlanBuilder.translate(rightExpressions.get(i));
equiClauses.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol));
Expression leftExpression = leftPlanBuilder.rewrite(leftExpressions.get(i));
Expression rightExpression = rightPlanBuilder.rewrite(rightExpressions.get(i));
postInnerJoinComparisons.add(
new ComparisonExpression(comparisonTypes.get(i), leftExpression, rightExpression));
}
postInnerJoinCriteria = ExpressionUtils.and(postInnerJoinComparisons);
}
PlanNode root;
if (node.getType() == INNER) {
root =
new JoinNode(
idAllocator.getNextId(),
JoinNode.Type.CROSS,
leftPlanBuilder.getRoot(),
rightPlanBuilder.getRoot(),
ImmutableList.<JoinNode.EquiJoinClause>of(),
Optional.empty(),
Optional.empty());
root = new FilterNode(idAllocator.getNextId(), root, postInnerJoinCriteria);
} else {
root =
new JoinNode(
idAllocator.getNextId(),
JoinNode.Type.typeConvert(node.getType()),
leftPlanBuilder.getRoot(),
rightPlanBuilder.getRoot(),
equiClauses.build(),
Optional.empty(),
Optional.empty());
}
Optional<Symbol> sampleWeight = Optional.empty();
if (leftPlanBuilder.getSampleWeight().isPresent()
|| rightPlanBuilder.getSampleWeight().isPresent()) {
Expression expression =
new ArithmeticBinaryExpression(
ArithmeticBinaryExpression.Type.MULTIPLY,
oneIfNull(leftPlanBuilder.getSampleWeight()),
oneIfNull(rightPlanBuilder.getSampleWeight()));
sampleWeight = Optional.of(symbolAllocator.newSymbol(expression, BIGINT));
ImmutableMap.Builder<Symbol, Expression> projections = ImmutableMap.builder();
projections.put(sampleWeight.get(), expression);
for (Symbol symbol : root.getOutputSymbols()) {
projections.put(symbol, new QualifiedNameReference(symbol.toQualifiedName()));
}
root = new ProjectNode(idAllocator.getNextId(), root, projections.build());
}
return new RelationPlan(root, outputDescriptor, outputSymbols, sampleWeight);
}
