/** * Returns merge directions between two RSs for criterias (ordering, number of reducers, reducer * keys, partition keys). Returns null if any of categories is not mergeable. * * <p>Values for each index can be -1, 0, 1 1. 0 means two configuration in the category is the * same 2. for -1, configuration of parent RS is more specific than child RS 3. for 1, * configuration of child RS is more specific than parent RS */ private int[] checkStatus(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer) throws SemanticException { ReduceSinkDesc cConf = cRS.getConf(); ReduceSinkDesc pConf = pRS.getConf(); Integer moveRSOrderTo = checkOrder(cConf.getOrder(), pConf.getOrder()); if (moveRSOrderTo == null) { return null; } Integer moveReducerNumTo = checkNumReducer(cConf.getNumReducers(), pConf.getNumReducers()); if (moveReducerNumTo == null || moveReducerNumTo > 0 && cConf.getNumReducers() < minReducer) { return null; } List<ExprNodeDesc> ckeys = cConf.getKeyCols(); List<ExprNodeDesc> pkeys = pConf.getKeyCols(); Integer moveKeyColTo = checkExprs(ckeys, pkeys, cRS, pRS); if (moveKeyColTo == null) { return null; } List<ExprNodeDesc> cpars = cConf.getPartitionCols(); List<ExprNodeDesc> ppars = pConf.getPartitionCols(); Integer movePartitionColTo = checkExprs(cpars, ppars, cRS, pRS); if (movePartitionColTo == null) { return null; } Integer moveNumDistKeyTo = checkNumDistributionKey(cConf.getNumDistributionKeys(), pConf.getNumDistributionKeys()); return new int[] { moveKeyColTo, movePartitionColTo, moveRSOrderTo, moveReducerNumTo, moveNumDistKeyTo }; }
// for JOIN-RS case, it's not possible generally to merge if child has // less key/partition columns than parents protected boolean merge(ReduceSinkOperator cRS, JoinOperator pJoin, int minReducer) throws SemanticException { List<Operator<?>> parents = pJoin.getParentOperators(); ReduceSinkOperator[] pRSs = parents.toArray(new ReduceSinkOperator[parents.size()]); ReduceSinkDesc cRSc = cRS.getConf(); ReduceSinkDesc pRS0c = pRSs[0].getConf(); if (cRSc.getKeyCols().size() < pRS0c.getKeyCols().size()) { return false; } if (cRSc.getPartitionCols().size() != pRS0c.getPartitionCols().size()) { return false; } Integer moveReducerNumTo = checkNumReducer(cRSc.getNumReducers(), pRS0c.getNumReducers()); if (moveReducerNumTo == null || moveReducerNumTo > 0 && cRSc.getNumReducers() < minReducer) { return false; } Integer moveRSOrderTo = checkOrder(cRSc.getOrder(), pRS0c.getOrder()); if (moveRSOrderTo == null) { return false; } boolean[] sorted = CorrelationUtilities.getSortedTags(pJoin); int cKeySize = cRSc.getKeyCols().size(); for (int i = 0; i < cKeySize; i++) { ExprNodeDesc cexpr = cRSc.getKeyCols().get(i); ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; for (int tag = 0; tag < pRSs.length; tag++) { pexprs[tag] = pRSs[tag].getConf().getKeyCols().get(i); } int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); if (found != i) { return false; } } int cPartSize = cRSc.getPartitionCols().size(); for (int i = 0; i < cPartSize; i++) { ExprNodeDesc cexpr = cRSc.getPartitionCols().get(i); ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; for (int tag = 0; tag < pRSs.length; tag++) { pexprs[tag] = pRSs[tag].getConf().getPartitionCols().get(i); } int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); if (found != i) { return false; } } if (moveReducerNumTo > 0) { for (ReduceSinkOperator pRS : pRSs) { pRS.getConf().setNumReducers(cRS.getConf().getNumReducers()); } } return true; }