@Override public ParseContext transform(ParseContext pctx) throws SemanticException { pGraphContext = pctx; Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put( new RuleRegExp( "R1", "(" + FilterOperator.getOperatorName() + "%" + ReduceSinkOperator.getOperatorName() + "%" + JoinOperator.getOperatorName() + "%)"), new JoinTransitive()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along TransitiveContext context = new TransitiveContext(); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context); GraphWalker ogw = new LevelOrderWalker(disp, 2); // Create a list of topop nodes List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pGraphContext.getTopOps().values()); ogw.startWalking(topNodes, null); Map<ReduceSinkOperator, List<ExprNodeDesc>> newFilters = context.getNewfilters(); // insert new filter between RS and parent of RS for (Map.Entry<ReduceSinkOperator, List<ExprNodeDesc>> entry : newFilters.entrySet()) { ReduceSinkOperator reducer = entry.getKey(); Operator<?> parent = reducer.getParentOperators().get(0); List<ExprNodeDesc> exprs = entry.getValue(); if (parent instanceof FilterOperator) { exprs = ExprNodeDescUtils.split(((FilterOperator) parent).getConf().getPredicate(), exprs); ExprNodeDesc merged = ExprNodeDescUtils.mergePredicates(exprs); ((FilterOperator) parent).getConf().setPredicate(merged); } else { ExprNodeDesc merged = ExprNodeDescUtils.mergePredicates(exprs); RowSchema parentRS = parent.getSchema(); Operator<FilterDesc> newFilter = createFilter(reducer, parent, parentRS, merged); } } return pGraphContext; }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { FilterOperator op = (FilterOperator) nd; ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; ExprNodeDesc condn = op.getConf().getPredicate(); // get list of columns used in the filter List<String> cl = condn.getCols(); // merge it with the downstream col list List<String> filterOpPrunedColLists = Utilities.mergeUniqElems(cppCtx.genColLists(op), cl); List<String> filterOpPrunedColListsOrderPreserved = preserveColumnOrder(op, filterOpPrunedColLists); cppCtx.getPrunedColLists().put(op, filterOpPrunedColListsOrderPreserved); pruneOperator(cppCtx, op, cppCtx.getPrunedColLists().get(op)); return null; }
/* * (non-Javadoc) * * @see * org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop * .hive.ql.parse.ParseContext) */ @Override public ParseContext transform(ParseContext pctx) throws SemanticException { // create a the context for walking operators List<PcrOpWalkerCtx.OpToDeleteInfo> opToRemove = new ArrayList<PcrOpWalkerCtx.OpToDeleteInfo>(); PcrOpWalkerCtx opWalkerCtx = new PcrOpWalkerCtx(pctx, opToRemove); Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put( new RuleRegExp( "R1", "(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)|(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)"), PcrOpProcFactory.getFilterProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(PcrOpProcFactory.getDefaultProc(), opRules, opWalkerCtx); GraphWalker ogw = new DefaultGraphWalker(disp); // Create a list of topop nodes ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); for (PcrOpWalkerCtx.OpToDeleteInfo entry : opToRemove) { entry.getParent().removeChildAndAdoptItsChildren(entry.getOperator()); } return pctx; }
@Override public Object process( Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { @SuppressWarnings("unchecked") CommonJoinOperator<JoinDesc> join = (CommonJoinOperator) nd; ReduceSinkOperator source = (ReduceSinkOperator) stack.get(stack.size() - 2); FilterOperator filter = (FilterOperator) stack.get(stack.size() - 3); int srcPos = join.getParentOperators().indexOf(source); TransitiveContext context = (TransitiveContext) procCtx; Map<CommonJoinOperator, int[][]> filterPropagates = context.getFilterPropagates(); Map<ReduceSinkOperator, List<ExprNodeDesc>> newFilters = context.getNewfilters(); int[][] targets = filterPropagates.get(join); if (targets == null) { filterPropagates.put(join, targets = getTargets(join)); } List<Operator<? extends OperatorDesc>> parents = join.getParentOperators(); for (int targetPos : targets[srcPos]) { ReduceSinkOperator target = (ReduceSinkOperator) parents.get(targetPos); List<ExprNodeDesc> sourceKeys = source.getConf().getKeyCols(); List<ExprNodeDesc> targetKeys = target.getConf().getKeyCols(); ExprNodeDesc predicate = filter.getConf().getPredicate(); ExprNodeDesc replaced = ExprNodeDescUtils.replace(predicate, sourceKeys, targetKeys); if (replaced != null && !filterExists(target, replaced)) { List<ExprNodeDesc> prev = newFilters.get(target); if (prev == null) { newFilters.put(target, ExprNodeDescUtils.split(replaced)); } else { ExprNodeDescUtils.split(replaced, prev); } } } return null; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { pGraphContext = pctx; // create a the context for walking operators OpWalkerInfo opWalkerInfo = new OpWalkerInfo(pGraphContext); Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put( new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"), OpProcFactory.getFilterProc()); opRules.put( new RuleRegExp("R2", PTFOperator.getOperatorName() + "%"), OpProcFactory.getPTFProc()); opRules.put( new RuleRegExp("R3", CommonJoinOperator.getOperatorName() + "%"), OpProcFactory.getJoinProc()); opRules.put( new RuleRegExp("R4", TableScanOperator.getOperatorName() + "%"), OpProcFactory.getTSProc()); opRules.put( new RuleRegExp("R5", ScriptOperator.getOperatorName() + "%"), OpProcFactory.getSCRProc()); opRules.put( new RuleRegExp("R6", LimitOperator.getOperatorName() + "%"), OpProcFactory.getLIMProc()); opRules.put( new RuleRegExp("R7", UDTFOperator.getOperatorName() + "%"), OpProcFactory.getUDTFProc()); opRules.put( new RuleRegExp("R8", LateralViewForwardOperator.getOperatorName() + "%"), OpProcFactory.getLVFProc()); opRules.put( new RuleRegExp("R9", LateralViewJoinOperator.getOperatorName() + "%"), OpProcFactory.getLVJProc()); opRules.put( new RuleRegExp("R10", ReduceSinkOperator.getOperatorName() + "%"), OpProcFactory.getRSProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(OpProcFactory.getDefaultProc(), opRules, opWalkerInfo); GraphWalker ogw = new DefaultGraphWalker(disp); // Create a list of topop nodes ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pGraphContext.getTopOps().values()); ogw.startWalking(topNodes, null); if (LOG.isDebugEnabled()) { LOG.debug("After PPD:\n" + Operator.toString(pctx.getTopOps().values())); } return pGraphContext; }