@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Context> context) { // Lookup symbols can only be passed through the probe side of an index join Set<Symbol> probeLookupSymbols = context .get() .getLookupSymbols() .stream() .filter(node.getProbeSource().getOutputSymbols()::contains) .collect(toImmutableSet()); if (probeLookupSymbols.isEmpty()) { return node; } PlanNode rewrittenProbeSource = context.rewrite( node.getProbeSource(), new Context(probeLookupSymbols, context.get().getSuccess())); PlanNode source = node; if (rewrittenProbeSource != node.getProbeSource()) { source = new IndexJoinNode( node.getId(), node.getType(), rewrittenProbeSource, node.getIndexSource(), node.getCriteria(), node.getProbeHashSymbol(), node.getIndexHashSymbol()); } return source; }
/** * create a CAstNode l representing a loop that traverses the prototype chain from receiver * searching for the constant property element. update nodeMap to map root to an expression that * reads the property from the right node. * * @param root * @param receiver * @param element * @param context * @param nodeMap * @return */ private CAstNode makeConstRead( CAstNode root, CAstNode receiver, CAstNode element, RewriteContext context, Map<Pair<CAstNode, ExpanderKey>, CAstNode> nodeMap) { CAstNode get, result; String receiverTemp = TEMP_NAME + (readTempCounter++); String elt = (String) element.getValue(); if (elt.equals("prototype") || elt.equals("__proto__")) { result = Ast.makeNode( CAstNode.BLOCK_EXPR, get = Ast.makeNode(CAstNode.OBJECT_REF, receiver, Ast.makeConstant(elt))); } else { if (context.inAssignment()) { context.setAssign( Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeConstant(elt)); } result = Ast.makeNode( CAstNode.BLOCK_EXPR, // declare loop variable and initialize to the receiver Ast.makeNode( CAstNode.DECL_STMT, Ast.makeConstant(new InternalCAstSymbol(receiverTemp, false, false)), receiver), Ast.makeNode( CAstNode.LOOP, // while the desired property of the loop variable is not // defined... Ast.makeNode( CAstNode.UNARY_EXPR, CAstOperator.OP_NOT, Ast.makeNode( CAstNode.IS_DEFINED_EXPR, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeConstant(elt))), // set the loop variable to be its prototype Ast.makeNode( CAstNode.ASSIGN, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeNode( CAstNode.OBJECT_REF, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeConstant("__proto__")))), get = Ast.makeNode( CAstNode.OBJECT_REF, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeConstant(elt))); } nodeMap.put(Pair.make(root, context.key()), result); nodeMap.put(Pair.make(root, ExpanderKey.EXTRA), get); return result; }
@Override public PlanNode visitExchange( ExchangeNode exchange, RewriteContext<FragmentProperties> context) { ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); for (int i = 0; i < exchange.getSources().size(); i++) { FragmentProperties childProperties = new FragmentProperties(); childProperties.setUnpartitionedOutput(); childProperties.setOutputLayout(exchange.getInputs().get(i)); builder.add(buildSubPlan(exchange.getSources().get(i), childProperties, context)); } } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setFixedDistribution(); FragmentProperties childProperties = new FragmentProperties() .setPartitionedOutput(exchange.getPartitionKeys(), exchange.getHashSymbol()) .setOutputLayout(Iterables.getOnlyElement(exchange.getInputs())); builder.add( buildSubPlan( Iterables.getOnlyElement(exchange.getSources()), childProperties, context)); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION_WITH_NULL_REPLICATION) { context.get().setFixedDistribution(); FragmentProperties childProperties = new FragmentProperties() .setPartitionedOutput(exchange.getPartitionKeys(), exchange.getHashSymbol()) .replicateNulls() .setOutputLayout(Iterables.getOnlyElement(exchange.getInputs())); builder.add( buildSubPlan( Iterables.getOnlyElement(exchange.getSources()), childProperties, context)); } else if (exchange.getType() == ExchangeNode.Type.REPLICATE) { FragmentProperties childProperties = new FragmentProperties(); childProperties.setUnpartitionedOutput(); childProperties.setOutputLayout(Iterables.getOnlyElement(exchange.getInputs())); builder.add( buildSubPlan( Iterables.getOnlyElement(exchange.getSources()), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children .stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols()); }
@Override public PlanNode visitFilter(FilterNode node, RewriteContext<Context> context) { if (node.getSource() instanceof TableScanNode) { return planTableScan((TableScanNode) node.getSource(), node.getPredicate(), context.get()); } return context.defaultRewrite( node, new Context(context.get().getLookupSymbols(), context.get().getSuccess())); }
@Override public PlanNode visitOutput(OutputNode node, RewriteContext<FragmentProperties> context) { context .get() .setSingleNodeDistribution() // TODO: add support for distributed output .setOutputLayout(node.getOutputSymbols()) .setUnpartitionedOutput(); return context.defaultRewrite(node, context.get()); }
/** * similar to makeConstRead(), but desired property is some expression instead of a constant * * @see #makeConstRead(CAstNode, CAstNode, CAstNode, RewriteContext, Map) */ private CAstNode makeVarRead( CAstNode root, CAstNode receiver, CAstNode element, RewriteContext context, Map<Pair<CAstNode, ExpanderKey>, CAstNode> nodeMap) { String receiverTemp = TEMP_NAME + (readTempCounter++); String elementTemp = TEMP_NAME + (readTempCounter++); if (context.inAssignment()) { context.setAssign( Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeNode(CAstNode.VAR, Ast.makeConstant(elementTemp))); } CAstNode get; CAstNode result = Ast.makeNode( CAstNode.BLOCK_EXPR, Ast.makeNode( CAstNode.DECL_STMT, Ast.makeConstant(new InternalCAstSymbol(receiverTemp, false, false)), receiver), Ast.makeNode( CAstNode.DECL_STMT, Ast.makeConstant(new InternalCAstSymbol(elementTemp, false, false)), element), Ast.makeNode( CAstNode.LOOP, Ast.makeNode( CAstNode.UNARY_EXPR, CAstOperator.OP_NOT, Ast.makeNode( CAstNode.IS_DEFINED_EXPR, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeNode(CAstNode.VAR, Ast.makeConstant(elementTemp)))), Ast.makeNode( CAstNode.ASSIGN, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeNode( CAstNode.OBJECT_REF, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeConstant("__proto__")))), get = Ast.makeNode( CAstNode.OBJECT_REF, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(receiverTemp)), Ast.makeNode(CAstNode.VAR, Ast.makeConstant(elementTemp)))); nodeMap.put(Pair.make(root, context.key()), get); return result; }
@Override public PlanNode visitWindow(WindowNode node, RewriteContext<Context> context) { if (!node.getWindowFunctions() .values() .stream() .map(function -> function.getFunctionCall().getName()) .allMatch(metadata.getFunctionRegistry()::isAggregationFunction)) { return node; } // Don't need this restriction if we can prove that all order by symbols are deterministically // produced if (!node.getOrderBy().isEmpty()) { return node; } // Only RANGE frame type currently supported for aggregation functions because it guarantees // the // same value for each peer group. // ROWS frame type requires the ordering to be fully deterministic (e.g. deterministically // sorted on all columns) if (node.getFrames() .stream() .map(WindowNode.Frame::getType) .anyMatch( type -> type != WindowFrame.Type .RANGE)) { // TODO: extract frames of type RANGE and allow optimization on // them return node; } // Lookup symbols can only be passed through if they are part of the partitioning Set<Symbol> partitionByLookupSymbols = context .get() .getLookupSymbols() .stream() .filter(node.getPartitionBy()::contains) .collect(toImmutableSet()); if (partitionByLookupSymbols.isEmpty()) { return node; } return context.defaultRewrite( node, new Context(partitionByLookupSymbols, context.get().getSuccess())); }
@Override public PlanNode visitProject(ProjectNode node, RewriteContext<Void> context) { PlanNode source = context.rewrite(node.getSource()); Map<Symbol, Expression> assignments = ImmutableMap.copyOf( Maps.transformValues(node.getAssignments(), this::simplifyExpression)); return new ProjectNode(node.getId(), source, assignments); }
@Override public PlanNode visitFilter(FilterNode node, RewriteContext<Void> context) { if (node.getSource() instanceof TableScanNode && !((TableScanNode) node.getSource()).getLayout().isPresent()) { return planTableScan((TableScanNode) node.getSource(), node.getPredicate()); } return context.defaultRewrite(node); }
@Override public PlanNode visitFilter(FilterNode node, RewriteContext<Void> context) { PlanNode source = context.rewrite(node.getSource()); Expression simplified = simplifyExpression(node.getPredicate()); if (simplified.equals(BooleanLiteral.TRUE_LITERAL)) { return source; } return new FilterNode(node.getId(), source, simplified); }
@Override public PlanNode visitAggregation(AggregationNode node, RewriteContext<Context> context) { // Lookup symbols can only be passed through if they are part of the group by columns Set<Symbol> groupByLookupSymbols = context .get() .getLookupSymbols() .stream() .filter(node.getGroupingKeys()::contains) .collect(toImmutableSet()); if (groupByLookupSymbols.isEmpty()) { return node; } return context.defaultRewrite( node, new Context(groupByLookupSymbols, context.get().getSuccess())); }
@Override public PlanNode visitTableCommit(TableCommitNode node, RewriteContext<Void> context) { Optional<DeleteNode> delete = findNode(node.getSource(), DeleteNode.class); if (!delete.isPresent()) { return context.defaultRewrite(node); } Optional<TableScanNode> tableScan = findNode(delete.get().getSource(), TableScanNode.class); if (!tableScan.isPresent()) { return context.defaultRewrite(node); } TableScanNode tableScanNode = tableScan.get(); if (!metadata.supportsMetadataDelete( session, tableScanNode.getTable(), tableScanNode.getLayout().get())) { return context.defaultRewrite(node); } return new MetadataDeleteNode( idAllocator.getNextId(), delete.get().getTarget(), Iterables.getOnlyElement(node.getOutputSymbols()), tableScanNode.getLayout().get()); }
@Override public PlanNode visitProject(ProjectNode node, RewriteContext<Context> context) { // Rewrite the lookup symbols in terms of only the pre-projected symbols that have direct // translations Set<Symbol> newLookupSymbols = context .get() .getLookupSymbols() .stream() .map(node.getAssignments()::get) .filter(SymbolReference.class::isInstance) .map(Symbol::from) .collect(toImmutableSet()); if (newLookupSymbols.isEmpty()) { return node; } return context.defaultRewrite( node, new Context(newLookupSymbols, context.get().getSuccess())); }
@Override public PlanNode visitProject(ProjectNode node, RewriteContext<Void> context) { PlanNode source = context.rewrite(node.getSource()); if (!node.getOutputSymbols().equals(source.getOutputSymbols())) { // Can't get rid of this projection. It constrains the output tuple from the underlying // operator return replaceChildren(node, ImmutableList.of(source)); } if (node.isIdentity()) { return source; } return replaceChildren(node, ImmutableList.of(source)); }
@Override protected PlanNode visitPlan(PlanNode node, RewriteContext<Void> context) { return context.defaultRewrite(node); }
@Override public PlanNode visitSort(SortNode node, RewriteContext<Context> context) { // Sort has no bearing when building an index, so just ignore the sort return context.rewrite(node.getSource(), context.get()); }
@Override protected CAstNode copyNodes( CAstNode root, final CAstControlFlowMap cfg, RewriteContext context, Map<Pair<CAstNode, ExpanderKey>, CAstNode> nodeMap) { int kind = root.getKind(); if (kind == CAstNode.OBJECT_REF && context.inRead()) { // if we see a property access (OBJECT_REF) in a read context, transform // to a loop traversing the prototype chain CAstNode readLoop; CAstNode receiver = copyNodes(root.getChild(0), cfg, READ, nodeMap); CAstNode element = copyNodes(root.getChild(1), cfg, READ, nodeMap); if (element.getKind() == CAstNode.CONSTANT && element.getValue() instanceof String) { readLoop = makeConstRead(root, receiver, element, context, nodeMap); } else { readLoop = makeVarRead(root, receiver, element, context, nodeMap); } return readLoop; } else if (kind == CAstNode.ASSIGN_PRE_OP || kind == CAstNode.ASSIGN_POST_OP) { // handle cases like x.f++, represented as ASSIGN_POST_OP(x.f,1,+) AssignPreOrPostOpContext ctxt = new AssignPreOrPostOpContext(); // generate loop for the first child (x.f for example), keeping the loop var and element var // in ctxt CAstNode lval = copyNodes(root.getChild(0), cfg, ctxt, nodeMap); CAstNode rval = copyNodes(root.getChild(1), cfg, READ, nodeMap); CAstNode op = copyNodes(root.getChild(2), cfg, READ, nodeMap); if (ctxt.receiverTemp != null) { // if we found a nested property access String temp1 = TEMP_NAME + (readTempCounter++); String temp2 = TEMP_NAME + (readTempCounter++); CAstNode copy = Ast.makeNode( CAstNode.BLOCK_EXPR, // assign lval to temp1 (where lval is a block that includes the prototype chain // loop) Ast.makeNode( CAstNode.DECL_STMT, Ast.makeConstant(new InternalCAstSymbol(temp1, true, false)), lval), // ? --MS // rval, // assign temp2 the new value to be assigned Ast.makeNode( CAstNode.DECL_STMT, Ast.makeConstant(new InternalCAstSymbol(temp2, true, false)), Ast.makeNode( CAstNode.BINARY_EXPR, op, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(temp1)), rval)), // write temp2 into the property Ast.makeNode( CAstNode.ASSIGN, Ast.makeNode(CAstNode.OBJECT_REF, ctxt.receiverTemp, ctxt.elementTemp), Ast.makeNode(CAstNode.VAR, Ast.makeConstant(temp2))), // final value depends on whether we had a pre op or post op Ast.makeNode( CAstNode.VAR, Ast.makeConstant((kind == CAstNode.ASSIGN_PRE_OP) ? temp2 : temp1))); nodeMap.put(Pair.make(root, context.key()), copy); return copy; } else { CAstNode copy = Ast.makeNode(kind, lval, rval, op); nodeMap.put(Pair.make(root, context.key()), copy); return copy; } } else if (kind == CAstNode.ASSIGN) { // use ASSIGN context for LHS so we don't translate property accesses there CAstNode copy = Ast.makeNode( CAstNode.ASSIGN, copyNodes(root.getChild(0), cfg, ASSIGN, nodeMap), copyNodes(root.getChild(1), cfg, READ, nodeMap)); nodeMap.put(Pair.make(root, context.key()), copy); return copy; } else if (kind == CAstNode.BLOCK_EXPR) { CAstNode children[] = new CAstNode[root.getChildCount()]; int last = (children.length - 1); for (int i = 0; i < last; i++) { children[i] = copyNodes(root.getChild(i), cfg, READ, nodeMap); } children[last] = copyNodes(root.getChild(last), cfg, context, nodeMap); CAstNode copy = Ast.makeNode(CAstNode.BLOCK_EXPR, children); nodeMap.put(Pair.make(root, context.key()), copy); return copy; } else if (root.getKind() == CAstNode.CONSTANT) { CAstNode copy = Ast.makeConstant(root.getValue()); nodeMap.put(Pair.make(root, context.key()), copy); return copy; } else if (root.getKind() == CAstNode.OPERATOR) { nodeMap.put(Pair.make(root, context.key()), root); return root; } else { CAstNode children[] = new CAstNode[root.getChildCount()]; for (int i = 0; i < children.length; i++) { children[i] = copyNodes(root.getChild(i), cfg, READ, nodeMap); } for (Object label : cfg.getTargetLabels(root)) { if (label instanceof CAstNode) { copyNodes((CAstNode) label, cfg, READ, nodeMap); } } CAstNode copy = Ast.makeNode(kind, children); nodeMap.put(Pair.make(root, context.key()), copy); return copy; } }
@Override public PlanNode visitMetadataDelete( MetadataDeleteNode node, RewriteContext<FragmentProperties> context) { context.get().setCoordinatorOnlyDistribution(); return context.defaultRewrite(node, context.get()); }
@Override public PlanNode visitTableScan(TableScanNode node, RewriteContext<Context> context) { return planTableScan(node, BooleanLiteral.TRUE_LITERAL, context.get()); }
@Override public PlanNode visitJoin(JoinNode node, RewriteContext<Void> context) { PlanNode leftRewritten = context.rewrite(node.getLeft()); PlanNode rightRewritten = context.rewrite(node.getRight()); if (!node.getCriteria().isEmpty()) { // Index join only possible with JOIN criteria List<Symbol> leftJoinSymbols = Lists.transform(node.getCriteria(), JoinNode.EquiJoinClause::getLeft); List<Symbol> rightJoinSymbols = Lists.transform(node.getCriteria(), JoinNode.EquiJoinClause::getRight); Optional<PlanNode> leftIndexCandidate = IndexSourceRewriter.rewriteWithIndex( leftRewritten, ImmutableSet.copyOf(leftJoinSymbols), symbolAllocator, idAllocator, metadata, session); if (leftIndexCandidate.isPresent()) { // Sanity check that we can trace the path for the index lookup key Map<Symbol, Symbol> trace = IndexKeyTracer.trace(leftIndexCandidate.get(), ImmutableSet.copyOf(leftJoinSymbols)); checkState(!trace.isEmpty() && leftJoinSymbols.containsAll(trace.keySet())); } Optional<PlanNode> rightIndexCandidate = IndexSourceRewriter.rewriteWithIndex( rightRewritten, ImmutableSet.copyOf(rightJoinSymbols), symbolAllocator, idAllocator, metadata, session); if (rightIndexCandidate.isPresent()) { // Sanity check that we can trace the path for the index lookup key Map<Symbol, Symbol> trace = IndexKeyTracer.trace( rightIndexCandidate.get(), ImmutableSet.copyOf(rightJoinSymbols)); checkState(!trace.isEmpty() && rightJoinSymbols.containsAll(trace.keySet())); } switch (node.getType()) { case INNER: // Prefer the right candidate over the left candidate IndexJoinNode indexJoinNode = null; if (rightIndexCandidate.isPresent()) { indexJoinNode = new IndexJoinNode( idAllocator.getNextId(), IndexJoinNode.Type.INNER, leftRewritten, rightIndexCandidate.get(), createEquiJoinClause(leftJoinSymbols, rightJoinSymbols), Optional.empty(), Optional.empty()); } else if (leftIndexCandidate.isPresent()) { indexJoinNode = new IndexJoinNode( idAllocator.getNextId(), IndexJoinNode.Type.INNER, rightRewritten, leftIndexCandidate.get(), createEquiJoinClause(rightJoinSymbols, leftJoinSymbols), Optional.empty(), Optional.empty()); } if (indexJoinNode != null) { if (node.getFilter().isPresent()) { return new FilterNode( idAllocator.getNextId(), indexJoinNode, node.getFilter().get()); } return indexJoinNode; } break; case LEFT: // We cannot use indices for outer joins until index join supports in-line filtering if (!node.getFilter().isPresent() && rightIndexCandidate.isPresent()) { return new IndexJoinNode( idAllocator.getNextId(), IndexJoinNode.Type.SOURCE_OUTER, leftRewritten, rightIndexCandidate.get(), createEquiJoinClause(leftJoinSymbols, rightJoinSymbols), Optional.empty(), Optional.empty()); } break; case RIGHT: // We cannot use indices for outer joins until index join supports in-line filtering if (!node.getFilter().isPresent() && leftIndexCandidate.isPresent()) { return new IndexJoinNode( idAllocator.getNextId(), IndexJoinNode.Type.SOURCE_OUTER, rightRewritten, leftIndexCandidate.get(), createEquiJoinClause(rightJoinSymbols, leftJoinSymbols), Optional.empty(), Optional.empty()); } break; case FULL: break; default: throw new IllegalArgumentException("Unknown type: " + node.getType()); } } if (leftRewritten != node.getLeft() || rightRewritten != node.getRight()) { return new JoinNode( node.getId(), node.getType(), leftRewritten, rightRewritten, node.getCriteria(), node.getFilter(), node.getLeftHashSymbol(), node.getRightHashSymbol()); } return node; }
@Override public PlanNode visitTableScan(TableScanNode node, RewriteContext<FragmentProperties> context) { context.get().setSourceDistribution(node.getId()); return context.defaultRewrite(node, context.get()); }
@Override public PlanNode visitValues(ValuesNode node, RewriteContext<FragmentProperties> context) { context.get().setSingleNodeDistribution(); return context.defaultRewrite(node, context.get()); }
private SubPlan buildSubPlan( PlanNode node, FragmentProperties properties, RewriteContext<FragmentProperties> context) { PlanFragmentId planFragmentId = nextFragmentId(); PlanNode child = context.rewrite(node, properties); return buildFragment(child, properties, planFragmentId); }