@Override
public Object clone() {
JoinDesc ret = new JoinDesc();
Map<Byte, List<ExprNodeDesc>> cloneExprs = new HashMap<Byte, List<ExprNodeDesc>>();
cloneExprs.putAll(getExprs());
ret.setExprs(cloneExprs);
Map<Byte, List<ExprNodeDesc>> cloneFilters = new HashMap<Byte, List<ExprNodeDesc>>();
cloneFilters.putAll(getFilters());
ret.setFilters(cloneFilters);
ret.setConds(getConds().clone());
ret.setNoOuterJoin(getNoOuterJoin());
ret.setNullSafes(getNullSafes());
ret.setHandleSkewJoin(handleSkewJoin);
ret.setSkewKeyDefinition(getSkewKeyDefinition());
ret.setTagOrder(getTagOrder().clone());
if (getKeyTableDesc() != null) {
ret.setKeyTableDesc((TableDesc) getKeyTableDesc().clone());
}
if (getBigKeysDirMap() != null) {
Map<Byte, String> cloneBigKeysDirMap = new HashMap<Byte, String>();
cloneBigKeysDirMap.putAll(getBigKeysDirMap());
ret.setBigKeysDirMap(cloneBigKeysDirMap);
}
if (getSmallKeysDirMap() != null) {
Map<Byte, Map<Byte, String>> cloneSmallKeysDirMap = new HashMap<Byte, Map<Byte, String>>();
cloneSmallKeysDirMap.putAll(getSmallKeysDirMap());
ret.setSmallKeysDirMap(cloneSmallKeysDirMap);
}
if (getSkewKeysValuesTables() != null) {
Map<Byte, TableDesc> cloneSkewKeysValuesTables = new HashMap<Byte, TableDesc>();
cloneSkewKeysValuesTables.putAll(getSkewKeysValuesTables());
ret.setSkewKeysValuesTables(cloneSkewKeysValuesTables);
}
if (getOutputColumnNames() != null) {
List<String> cloneOutputColumnNames = new ArrayList<String>();
cloneOutputColumnNames.addAll(getOutputColumnNames());
ret.setOutputColumnNames(cloneOutputColumnNames);
}
if (getReversedExprs() != null) {
Map<String, Byte> cloneReversedExprs = new HashMap<String, Byte>();
cloneReversedExprs.putAll(getReversedExprs());
ret.setReversedExprs(cloneReversedExprs);
}
return ret;
}
private static void pruneJoinOperator(
NodeProcessorCtx ctx,
CommonJoinOperator op,
JoinDesc conf,
Map<String, ExprNodeDesc> columnExprMap,
Map<Byte, List<Integer>> retainMap,
boolean mapJoin)
throws SemanticException {
ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
List<Operator<? extends OperatorDesc>> childOperators = op.getChildOperators();
LOG.info("JOIN " + op.getIdentifier() + " oldExprs: " + conf.getExprs());
List<String> childColLists = cppCtx.genColLists(op);
if (childColLists == null) {
return;
}
Map<Byte, List<String>> prunedColLists = new HashMap<Byte, List<String>>();
for (byte tag : conf.getTagOrder()) {
prunedColLists.put(tag, new ArrayList<String>());
}
// add the columns in join filters
Set<Map.Entry<Byte, List<ExprNodeDesc>>> filters = conf.getFilters().entrySet();
Iterator<Map.Entry<Byte, List<ExprNodeDesc>>> iter = filters.iterator();
while (iter.hasNext()) {
Map.Entry<Byte, List<ExprNodeDesc>> entry = iter.next();
Byte tag = entry.getKey();
for (ExprNodeDesc desc : entry.getValue()) {
List<String> cols = prunedColLists.get(tag);
cols = Utilities.mergeUniqElems(cols, desc.getCols());
prunedColLists.put(tag, cols);
}
}
RowResolver joinRR = cppCtx.getOpToParseCtxMap().get(op).getRowResolver();
RowResolver newJoinRR = new RowResolver();
ArrayList<String> outputCols = new ArrayList<String>();
ArrayList<ColumnInfo> rs = new ArrayList<ColumnInfo>();
Map<String, ExprNodeDesc> newColExprMap = new HashMap<String, ExprNodeDesc>();
for (int i = 0; i < conf.getOutputColumnNames().size(); i++) {
String internalName = conf.getOutputColumnNames().get(i);
ExprNodeDesc desc = columnExprMap.get(internalName);
Byte tag = conf.getReversedExprs().get(internalName);
if (!childColLists.contains(internalName)) {
int index = conf.getExprs().get(tag).indexOf(desc);
if (index < 0) {
continue;
}
conf.getExprs().get(tag).remove(desc);
if (retainMap != null) {
retainMap.get(tag).remove(index);
}
} else {
List<String> prunedRSList = prunedColLists.get(tag);
if (prunedRSList == null) {
prunedRSList = new ArrayList<String>();
prunedColLists.put(tag, prunedRSList);
}
prunedRSList = Utilities.mergeUniqElems(prunedRSList, desc.getCols());
outputCols.add(internalName);
newColExprMap.put(internalName, desc);
}
}
if (mapJoin) {
// regenerate the valueTableDesc
List<TableDesc> valueTableDescs = new ArrayList<TableDesc>();
for (int pos = 0; pos < op.getParentOperators().size(); pos++) {
List<ExprNodeDesc> valueCols = conf.getExprs().get(Byte.valueOf((byte) pos));
StringBuilder keyOrder = new StringBuilder();
for (int i = 0; i < valueCols.size(); i++) {
keyOrder.append("+");
}
TableDesc valueTableDesc =
PlanUtils.getMapJoinValueTableDesc(
PlanUtils.getFieldSchemasFromColumnList(valueCols, "mapjoinvalue"));
valueTableDescs.add(valueTableDesc);
}
((MapJoinDesc) conf).setValueTblDescs(valueTableDescs);
Set<Map.Entry<Byte, List<ExprNodeDesc>>> exprs = ((MapJoinDesc) conf).getKeys().entrySet();
Iterator<Map.Entry<Byte, List<ExprNodeDesc>>> iters = exprs.iterator();
while (iters.hasNext()) {
Map.Entry<Byte, List<ExprNodeDesc>> entry = iters.next();
List<ExprNodeDesc> lists = entry.getValue();
for (int j = 0; j < lists.size(); j++) {
ExprNodeDesc desc = lists.get(j);
Byte tag = entry.getKey();
List<String> cols = prunedColLists.get(tag);
cols = Utilities.mergeUniqElems(cols, desc.getCols());
prunedColLists.put(tag, cols);
}
}
}
for (Operator<? extends OperatorDesc> child : childOperators) {
if (child instanceof ReduceSinkOperator) {
boolean[] flags =
getPruneReduceSinkOpRetainFlags(childColLists, (ReduceSinkOperator) child);
pruneReduceSinkOperator(flags, (ReduceSinkOperator) child, cppCtx);
}
}
for (int i = 0; i < outputCols.size(); i++) {
String internalName = outputCols.get(i);
String[] nm = joinRR.reverseLookup(internalName);
ColumnInfo col = joinRR.get(nm[0], nm[1]);
newJoinRR.put(nm[0], nm[1], col);
rs.add(col);
}
LOG.info("JOIN " + op.getIdentifier() + " newExprs: " + conf.getExprs());
op.setColumnExprMap(newColExprMap);
conf.setOutputColumnNames(outputCols);
op.getSchema().setSignature(rs);
cppCtx.getOpToParseCtxMap().get(op).setRowResolver(newJoinRR);
cppCtx.getJoinPrunedColLists().put(op, prunedColLists);
}
