@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
pGraphContext = pctx;
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(
new RuleRegExp(
"R1",
"("
+ FilterOperator.getOperatorName()
+ "%"
+ ReduceSinkOperator.getOperatorName()
+ "%"
+ JoinOperator.getOperatorName()
+ "%)"),
new JoinTransitive());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
TransitiveContext context = new TransitiveContext();
Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context);
GraphWalker ogw = new LevelOrderWalker(disp, 2);
// Create a list of topop nodes
List<Node> topNodes = new ArrayList<Node>();
topNodes.addAll(pGraphContext.getTopOps().values());
ogw.startWalking(topNodes, null);
Map<ReduceSinkOperator, List<ExprNodeDesc>> newFilters = context.getNewfilters();
// insert new filter between RS and parent of RS
for (Map.Entry<ReduceSinkOperator, List<ExprNodeDesc>> entry : newFilters.entrySet()) {
ReduceSinkOperator reducer = entry.getKey();
Operator<?> parent = reducer.getParentOperators().get(0);
List<ExprNodeDesc> exprs = entry.getValue();
if (parent instanceof FilterOperator) {
exprs = ExprNodeDescUtils.split(((FilterOperator) parent).getConf().getPredicate(), exprs);
ExprNodeDesc merged = ExprNodeDescUtils.mergePredicates(exprs);
((FilterOperator) parent).getConf().setPredicate(merged);
} else {
ExprNodeDesc merged = ExprNodeDescUtils.mergePredicates(exprs);
RowSchema parentRS = parent.getSchema();
Operator<FilterDesc> newFilter = createFilter(reducer, parent, parentRS, merged);
}
}
return pGraphContext;
}
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
FilterOperator op = (FilterOperator) nd;
ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
ExprNodeDesc condn = op.getConf().getPredicate();
// get list of columns used in the filter
List<String> cl = condn.getCols();
// merge it with the downstream col list
List<String> filterOpPrunedColLists = Utilities.mergeUniqElems(cppCtx.genColLists(op), cl);
List<String> filterOpPrunedColListsOrderPreserved =
preserveColumnOrder(op, filterOpPrunedColLists);
cppCtx.getPrunedColLists().put(op, filterOpPrunedColListsOrderPreserved);
pruneOperator(cppCtx, op, cppCtx.getPrunedColLists().get(op));
return null;
}
/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop
* .hive.ql.parse.ParseContext)
*/
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
// create a the context for walking operators
List<PcrOpWalkerCtx.OpToDeleteInfo> opToRemove = new ArrayList<PcrOpWalkerCtx.OpToDeleteInfo>();
PcrOpWalkerCtx opWalkerCtx = new PcrOpWalkerCtx(pctx, opToRemove);
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(
new RuleRegExp(
"R1",
"("
+ TableScanOperator.getOperatorName()
+ "%"
+ FilterOperator.getOperatorName()
+ "%)|("
+ TableScanOperator.getOperatorName()
+ "%"
+ FilterOperator.getOperatorName()
+ "%"
+ FilterOperator.getOperatorName()
+ "%)"),
PcrOpProcFactory.getFilterProc());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Dispatcher disp =
new DefaultRuleDispatcher(PcrOpProcFactory.getDefaultProc(), opRules, opWalkerCtx);
GraphWalker ogw = new DefaultGraphWalker(disp);
// Create a list of topop nodes
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.addAll(pctx.getTopOps().values());
ogw.startWalking(topNodes, null);
for (PcrOpWalkerCtx.OpToDeleteInfo entry : opToRemove) {
entry.getParent().removeChildAndAdoptItsChildren(entry.getOperator());
}
return pctx;
}
@Override
public Object process(
Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs)
throws SemanticException {
@SuppressWarnings("unchecked")
CommonJoinOperator<JoinDesc> join = (CommonJoinOperator) nd;
ReduceSinkOperator source = (ReduceSinkOperator) stack.get(stack.size() - 2);
FilterOperator filter = (FilterOperator) stack.get(stack.size() - 3);
int srcPos = join.getParentOperators().indexOf(source);
TransitiveContext context = (TransitiveContext) procCtx;
Map<CommonJoinOperator, int[][]> filterPropagates = context.getFilterPropagates();
Map<ReduceSinkOperator, List<ExprNodeDesc>> newFilters = context.getNewfilters();
int[][] targets = filterPropagates.get(join);
if (targets == null) {
filterPropagates.put(join, targets = getTargets(join));
}
List<Operator<? extends OperatorDesc>> parents = join.getParentOperators();
for (int targetPos : targets[srcPos]) {
ReduceSinkOperator target = (ReduceSinkOperator) parents.get(targetPos);
List<ExprNodeDesc> sourceKeys = source.getConf().getKeyCols();
List<ExprNodeDesc> targetKeys = target.getConf().getKeyCols();
ExprNodeDesc predicate = filter.getConf().getPredicate();
ExprNodeDesc replaced = ExprNodeDescUtils.replace(predicate, sourceKeys, targetKeys);
if (replaced != null && !filterExists(target, replaced)) {
List<ExprNodeDesc> prev = newFilters.get(target);
if (prev == null) {
newFilters.put(target, ExprNodeDescUtils.split(replaced));
} else {
ExprNodeDescUtils.split(replaced, prev);
}
}
}
return null;
}
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
pGraphContext = pctx;
// create a the context for walking operators
OpWalkerInfo opWalkerInfo = new OpWalkerInfo(pGraphContext);
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(
new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"),
OpProcFactory.getFilterProc());
opRules.put(
new RuleRegExp("R2", PTFOperator.getOperatorName() + "%"), OpProcFactory.getPTFProc());
opRules.put(
new RuleRegExp("R3", CommonJoinOperator.getOperatorName() + "%"),
OpProcFactory.getJoinProc());
opRules.put(
new RuleRegExp("R4", TableScanOperator.getOperatorName() + "%"), OpProcFactory.getTSProc());
opRules.put(
new RuleRegExp("R5", ScriptOperator.getOperatorName() + "%"), OpProcFactory.getSCRProc());
opRules.put(
new RuleRegExp("R6", LimitOperator.getOperatorName() + "%"), OpProcFactory.getLIMProc());
opRules.put(
new RuleRegExp("R7", UDTFOperator.getOperatorName() + "%"), OpProcFactory.getUDTFProc());
opRules.put(
new RuleRegExp("R8", LateralViewForwardOperator.getOperatorName() + "%"),
OpProcFactory.getLVFProc());
opRules.put(
new RuleRegExp("R9", LateralViewJoinOperator.getOperatorName() + "%"),
OpProcFactory.getLVJProc());
opRules.put(
new RuleRegExp("R10", ReduceSinkOperator.getOperatorName() + "%"),
OpProcFactory.getRSProc());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Dispatcher disp =
new DefaultRuleDispatcher(OpProcFactory.getDefaultProc(), opRules, opWalkerInfo);
GraphWalker ogw = new DefaultGraphWalker(disp);
// Create a list of topop nodes
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.addAll(pGraphContext.getTopOps().values());
ogw.startWalking(topNodes, null);
if (LOG.isDebugEnabled()) {
LOG.debug("After PPD:\n" + Operator.toString(pctx.getTopOps().values()));
}
return pGraphContext;
}
