public Double getDistinctRowCount(Project rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
ImmutableBitSet.Builder baseCols = ImmutableBitSet.builder();
ImmutableBitSet.Builder projCols = ImmutableBitSet.builder();
List<RexNode> projExprs = rel.getProjects();
RelMdUtil.splitCols(projExprs, groupKey, baseCols, projCols);
List<RexNode> notPushable = new ArrayList<RexNode>();
List<RexNode> pushable = new ArrayList<RexNode>();
RelOptUtil.splitFilters(
ImmutableBitSet.range(rel.getRowType().getFieldCount()), predicate, pushable, notPushable);
final RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
// get the distinct row count of the child input, passing in the
// columns and filters that only reference the child; convert the
// filter to reference the children projection expressions
RexNode childPred = RexUtil.composeConjunction(rexBuilder, pushable, true);
RexNode modifiedPred;
if (childPred == null) {
modifiedPred = null;
} else {
modifiedPred = RelOptUtil.pushPastProject(childPred, rel);
}
Double distinctRowCount =
RelMetadataQuery.getDistinctRowCount(rel.getInput(), baseCols.build(), modifiedPred);
if (distinctRowCount == null) {
return null;
} else if (!notPushable.isEmpty()) {
RexNode preds = RexUtil.composeConjunction(rexBuilder, notPushable, true);
distinctRowCount *= RelMdUtil.guessSelectivity(preds);
}
// No further computation required if the projection expressions
// are all column references
if (projCols.cardinality() == 0) {
return distinctRowCount;
}
// multiply by the cardinality of the non-child projection expressions
for (int bit : projCols.build()) {
Double subRowCount = RelMdUtil.cardOfProjExpr(rel, projExprs.get(bit));
if (subRowCount == null) {
return null;
}
distinctRowCount *= subRowCount;
}
return RelMdUtil.numDistinctVals(distinctRowCount, RelMetadataQuery.getRowCount(rel));
}
/**
* Given a list of predicates to push down, this methods returns the set of predicates that still
* need to be pushed. Predicates need to be pushed because 1) their String representation is not
* included in input set of predicates to exclude, or 2) they are already in the subtree rooted at
* the input node. This method updates the set of predicates to exclude with the String
* representation of the predicates in the output and in the subtree.
*
* @param predicatesToExclude String representation of predicates that should be excluded
* @param inp root of the subtree
* @param predsToPushDown candidate predicates to push down through the subtree
* @return list of predicates to push down
*/
public static ImmutableList<RexNode> getPredsNotPushedAlready(
Set<String> predicatesToExclude, RelNode inp, List<RexNode> predsToPushDown) {
// Bail out if there is nothing to push
if (predsToPushDown.isEmpty()) {
return ImmutableList.of();
}
// Build map to not convert multiple times, further remove already included predicates
Map<String, RexNode> stringToRexNode = Maps.newLinkedHashMap();
for (RexNode r : predsToPushDown) {
String rexNodeString = r.toString();
if (predicatesToExclude.add(rexNodeString)) {
stringToRexNode.put(rexNodeString, r);
}
}
if (stringToRexNode.isEmpty()) {
return ImmutableList.of();
}
// Finally exclude preds that are already in the subtree as given by the metadata provider
// Note: this is the last step, trying to avoid the expensive call to the metadata provider
// if possible
Set<String> predicatesInSubtree = Sets.newHashSet();
for (RexNode pred : RelMetadataQuery.instance().getPulledUpPredicates(inp).pulledUpPredicates) {
predicatesInSubtree.add(pred.toString());
predicatesInSubtree.addAll(Lists.transform(RelOptUtil.conjunctions(pred), REX_STR_FN));
}
final ImmutableList.Builder<RexNode> newConjuncts = ImmutableList.builder();
for (Entry<String, RexNode> e : stringToRexNode.entrySet()) {
if (predicatesInSubtree.add(e.getKey())) {
newConjuncts.add(e.getValue());
}
}
predicatesToExclude.addAll(predicatesInSubtree);
return newConjuncts.build();
}
public Double getDistinctRowCount(Aggregate rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
// determine which predicates can be applied on the child of the
// aggregate
List<RexNode> notPushable = new ArrayList<RexNode>();
List<RexNode> pushable = new ArrayList<RexNode>();
RelOptUtil.splitFilters(rel.getGroupSet(), predicate, pushable, notPushable);
final RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
RexNode childPreds = RexUtil.composeConjunction(rexBuilder, pushable, true);
// set the bits as they correspond to the child input
ImmutableBitSet.Builder childKey = ImmutableBitSet.builder();
RelMdUtil.setAggChildKeys(groupKey, rel, childKey);
Double distinctRowCount =
RelMetadataQuery.getDistinctRowCount(rel.getInput(), childKey.build(), childPreds);
if (distinctRowCount == null) {
return null;
} else if (notPushable.isEmpty()) {
return distinctRowCount;
} else {
RexNode preds = RexUtil.composeConjunction(rexBuilder, notPushable, true);
return distinctRowCount * RelMdUtil.guessSelectivity(preds);
}
}
@Override
public RelNode toRel(ToRelContext context, RelOptTable relOptTable) {
ViewExpansionContext.ViewExpansionToken token = null;
try {
RelDataType rowType = relOptTable.getRowType();
RelNode rel;
if (viewExpansionContext.isImpersonationEnabled()) {
token = viewExpansionContext.reserveViewExpansionToken(viewOwner);
rel =
context.expandView(
rowType, view.getSql(), token.getSchemaTree(), view.getWorkspaceSchemaPath());
} else {
rel = context.expandView(rowType, view.getSql(), view.getWorkspaceSchemaPath());
}
// If the View's field list is not "*", create a cast.
if (!view.isDynamic() && !view.hasStar()) {
rel = RelOptUtil.createCastRel(rel, rowType, true);
}
return rel;
} finally {
if (token != null) {
token.release();
}
}
}
/** Add the Filter condition to the pulledPredicates list from the input. */
public RelOptPredicateList getPredicates(Filter filter, RelMetadataQuery mq) {
final RelNode input = filter.getInput();
final RelOptPredicateList inputInfo = mq.getPulledUpPredicates(input);
return Util.first(inputInfo, RelOptPredicateList.EMPTY)
.union(RelOptPredicateList.of(RelOptUtil.conjunctions(filter.getCondition())));
}
/** Add the Filter condition to the pulledPredicates list from the child. */
public RelOptPredicateList getPredicates(Filter filter) {
RelNode child = filter.getInput();
RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child);
return RelOptPredicateList.of(
Iterables.concat(
childInfo.pulledUpPredicates, RelOptUtil.conjunctions(filter.getCondition())));
}
/**
* Infers predicates for a Union.
*
* <p>The pulled up expression is a disjunction of its children's predicates.
*/
public RelOptPredicateList getPredicates(Union union) {
RexBuilder rB = union.getCluster().getRexBuilder();
List<RexNode> orList = Lists.newArrayList();
for (RelNode input : union.getInputs()) {
RelOptPredicateList info = RelMetadataQuery.getPulledUpPredicates(input);
if (info.pulledUpPredicates.isEmpty()) {
return RelOptPredicateList.EMPTY;
}
RelOptUtil.decomposeDisjunction(
RexUtil.composeConjunction(rB, info.pulledUpPredicates, false), orList);
}
if (orList.isEmpty()) {
return RelOptPredicateList.EMPTY;
}
return RelOptPredicateList.of(
RelOptUtil.conjunctions(RexUtil.composeDisjunction(rB, orList, false)));
}
private Object translateOr(RexNode condition) {
List<Object> list = new ArrayList<Object>();
for (RexNode node : RelOptUtil.disjunctions(condition)) {
list.add(translateAnd(node));
}
switch (list.size()) {
case 1:
return list.get(0);
default:
Map<String, Object> map = builder.map();
map.put("$or", list);
return map;
}
}
/**
* Translates a condition that may be an AND of other conditions. Gathers together conditions
* that apply to the same field.
*/
private Map<String, Object> translateAnd(RexNode node0) {
eqMap.clear();
multimap.clear();
for (RexNode node : RelOptUtil.conjunctions(node0)) {
translateMatch2(node);
}
Map<String, Object> map = builder.map();
for (Map.Entry<String, RexLiteral> entry : eqMap.entrySet()) {
multimap.removeAll(entry.getKey());
map.put(entry.getKey(), literalToString(entry.getValue()));
}
for (Map.Entry<String, Collection<Pair<String, RexLiteral>>> entry :
multimap.asMap().entrySet()) {
Map<String, Object> map2 = builder.map();
for (Pair<String, RexLiteral> s : entry.getValue()) {
map2.put(s.left, literalToString(s.right));
}
map.put(entry.getKey(), map2);
}
return map;
}
private void infer(
RexNode predicates,
Set<String> allExprsDigests,
List<RexNode> inferedPredicates,
boolean includeEqualityInference,
ImmutableBitSet inferringFields) {
for (RexNode r : RelOptUtil.conjunctions(predicates)) {
if (!includeEqualityInference && equalityPredicates.contains(r.toString())) {
continue;
}
for (Mapping m : mappings(r)) {
RexNode tr =
r.accept(new RexPermuteInputsShuttle(m, joinRel.getInput(0), joinRel.getInput(1)));
if (inferringFields.contains(RelOptUtil.InputFinder.bits(tr))
&& !allExprsDigests.contains(tr.toString())
&& !isAlwaysTrue(tr)) {
inferedPredicates.add(tr);
allExprsDigests.add(tr.toString());
}
}
}
}
/**
* The PullUp Strategy is sound but not complete.
*
* <ol>
* <li>We only pullUp inferred predicates for now. Pulling up existing predicates causes an
* explosion of duplicates. The existing predicates are pushed back down as new
* predicates. Once we have rules to eliminate duplicate Filter conditions, we should
* pullUp all predicates.
* <li>For Left Outer: we infer new predicates from the left and set them as applicable on the
* Right side. No predicates are pulledUp.
* <li>Right Outer Joins are handled in an analogous manner.
* <li>For Full Outer Joins no predicates are pulledUp or inferred.
* </ol>
*/
public RelOptPredicateList inferPredicates(boolean includeEqualityInference) {
List<RexNode> inferredPredicates = new ArrayList<RexNode>();
Set<String> allExprsDigests = new HashSet<String>(this.allExprsDigests);
final JoinRelType joinType = joinRel.getJoinType();
switch (joinType) {
case INNER:
case LEFT:
infer(
leftChildPredicates,
allExprsDigests,
inferredPredicates,
includeEqualityInference,
joinType == JoinRelType.LEFT ? rightFieldsBitSet : allFieldsBitSet);
break;
}
switch (joinType) {
case INNER:
case RIGHT:
infer(
rightChildPredicates,
allExprsDigests,
inferredPredicates,
includeEqualityInference,
joinType == JoinRelType.RIGHT ? leftFieldsBitSet : allFieldsBitSet);
break;
}
Mappings.TargetMapping rightMapping =
Mappings.createShiftMapping(
nSysFields + nFieldsLeft + nFieldsRight, 0, nSysFields + nFieldsLeft, nFieldsRight);
final RexPermuteInputsShuttle rightPermute =
new RexPermuteInputsShuttle(rightMapping, joinRel);
Mappings.TargetMapping leftMapping =
Mappings.createShiftMapping(nSysFields + nFieldsLeft, 0, nSysFields, nFieldsLeft);
final RexPermuteInputsShuttle leftPermute = new RexPermuteInputsShuttle(leftMapping, joinRel);
List<RexNode> leftInferredPredicates = new ArrayList<RexNode>();
List<RexNode> rightInferredPredicates = new ArrayList<RexNode>();
for (RexNode iP : inferredPredicates) {
ImmutableBitSet iPBitSet = RelOptUtil.InputFinder.bits(iP);
if (leftFieldsBitSet.contains(iPBitSet)) {
leftInferredPredicates.add(iP.accept(leftPermute));
} else if (rightFieldsBitSet.contains(iPBitSet)) {
rightInferredPredicates.add(iP.accept(rightPermute));
}
}
switch (joinType) {
case INNER:
Iterable<RexNode> pulledUpPredicates;
if (isSemiJoin) {
pulledUpPredicates =
Iterables.concat(
RelOptUtil.conjunctions(leftChildPredicates), leftInferredPredicates);
} else {
pulledUpPredicates =
Iterables.concat(
RelOptUtil.conjunctions(leftChildPredicates),
RelOptUtil.conjunctions(rightChildPredicates),
RelOptUtil.conjunctions(joinRel.getCondition()),
inferredPredicates);
}
return RelOptPredicateList.of(
pulledUpPredicates, leftInferredPredicates, rightInferredPredicates);
case LEFT:
return RelOptPredicateList.of(
RelOptUtil.conjunctions(leftChildPredicates),
leftInferredPredicates,
rightInferredPredicates);
case RIGHT:
return RelOptPredicateList.of(
RelOptUtil.conjunctions(rightChildPredicates), inferredPredicates, EMPTY_LIST);
default:
assert inferredPredicates.size() == 0;
return RelOptPredicateList.EMPTY;
}
}
private JoinConditionBasedPredicateInference(
Join joinRel, boolean isSemiJoin, RexNode lPreds, RexNode rPreds) {
super();
this.joinRel = joinRel;
this.isSemiJoin = isSemiJoin;
nFieldsLeft = joinRel.getLeft().getRowType().getFieldList().size();
nFieldsRight = joinRel.getRight().getRowType().getFieldList().size();
nSysFields = joinRel.getSystemFieldList().size();
leftFieldsBitSet = ImmutableBitSet.range(nSysFields, nSysFields + nFieldsLeft);
rightFieldsBitSet =
ImmutableBitSet.range(nSysFields + nFieldsLeft, nSysFields + nFieldsLeft + nFieldsRight);
allFieldsBitSet = ImmutableBitSet.range(0, nSysFields + nFieldsLeft + nFieldsRight);
exprFields = Maps.newHashMap();
allExprsDigests = new HashSet<String>();
if (lPreds == null) {
leftChildPredicates = null;
} else {
Mappings.TargetMapping leftMapping =
Mappings.createShiftMapping(nSysFields + nFieldsLeft, nSysFields, 0, nFieldsLeft);
leftChildPredicates =
lPreds.accept(new RexPermuteInputsShuttle(leftMapping, joinRel.getInput(0)));
for (RexNode r : RelOptUtil.conjunctions(leftChildPredicates)) {
exprFields.put(r.toString(), RelOptUtil.InputFinder.bits(r));
allExprsDigests.add(r.toString());
}
}
if (rPreds == null) {
rightChildPredicates = null;
} else {
Mappings.TargetMapping rightMapping =
Mappings.createShiftMapping(
nSysFields + nFieldsLeft + nFieldsRight, nSysFields + nFieldsLeft, 0, nFieldsRight);
rightChildPredicates =
rPreds.accept(new RexPermuteInputsShuttle(rightMapping, joinRel.getInput(1)));
for (RexNode r : RelOptUtil.conjunctions(rightChildPredicates)) {
exprFields.put(r.toString(), RelOptUtil.InputFinder.bits(r));
allExprsDigests.add(r.toString());
}
}
equivalence = Maps.newTreeMap();
equalityPredicates = new HashSet<String>();
for (int i = 0; i < nSysFields + nFieldsLeft + nFieldsRight; i++) {
equivalence.put(i, BitSets.of(i));
}
// Only process equivalences found in the join conditions. Processing
// Equivalences from the left or right side infer predicates that are
// already present in the Tree below the join.
RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder();
List<RexNode> exprs =
RelOptUtil.conjunctions(compose(rexBuilder, ImmutableList.of(joinRel.getCondition())));
final EquivalenceFinder eF = new EquivalenceFinder();
new ArrayList<Void>(
Lists.transform(
exprs,
new Function<RexNode, Void>() {
public Void apply(RexNode input) {
return input.accept(eF);
}
}));
equivalence = BitSets.closure(equivalence);
}
/**
* Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for {@link
* org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin}.
*/
public TrimResult trimFields(
HiveMultiJoin join, ImmutableBitSet fieldsUsed, Set<RelDataTypeField> extraFields) {
final int fieldCount = join.getRowType().getFieldCount();
final RexNode conditionExpr = join.getCondition();
// Add in fields used in the condition.
final Set<RelDataTypeField> combinedInputExtraFields =
new LinkedHashSet<RelDataTypeField>(extraFields);
RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(combinedInputExtraFields);
inputFinder.inputBitSet.addAll(fieldsUsed);
conditionExpr.accept(inputFinder);
final ImmutableBitSet fieldsUsedPlus = inputFinder.inputBitSet.build();
int inputStartPos = 0;
int changeCount = 0;
int newFieldCount = 0;
List<RelNode> newInputs = new ArrayList<RelNode>();
List<Mapping> inputMappings = new ArrayList<Mapping>();
for (RelNode input : join.getInputs()) {
final RelDataType inputRowType = input.getRowType();
final int inputFieldCount = inputRowType.getFieldCount();
// Compute required mapping.
ImmutableBitSet.Builder inputFieldsUsed = ImmutableBitSet.builder();
for (int bit : fieldsUsedPlus) {
if (bit >= inputStartPos && bit < inputStartPos + inputFieldCount) {
inputFieldsUsed.set(bit - inputStartPos);
}
}
Set<RelDataTypeField> inputExtraFields = Collections.<RelDataTypeField>emptySet();
TrimResult trimResult = trimChild(join, input, inputFieldsUsed.build(), inputExtraFields);
newInputs.add(trimResult.left);
if (trimResult.left != input) {
++changeCount;
}
final Mapping inputMapping = trimResult.right;
inputMappings.add(inputMapping);
// Move offset to point to start of next input.
inputStartPos += inputFieldCount;
newFieldCount += inputMapping.getTargetCount();
}
Mapping mapping = Mappings.create(MappingType.INVERSE_SURJECTION, fieldCount, newFieldCount);
int offset = 0;
int newOffset = 0;
for (int i = 0; i < inputMappings.size(); i++) {
Mapping inputMapping = inputMappings.get(i);
for (IntPair pair : inputMapping) {
mapping.set(pair.source + offset, pair.target + newOffset);
}
offset += inputMapping.getSourceCount();
newOffset += inputMapping.getTargetCount();
}
if (changeCount == 0 && mapping.isIdentity()) {
return new TrimResult(join, Mappings.createIdentity(fieldCount));
}
// Build new join.
final RexVisitor<RexNode> shuttle =
new RexPermuteInputsShuttle(mapping, newInputs.toArray(new RelNode[newInputs.size()]));
RexNode newConditionExpr = conditionExpr.accept(shuttle);
final RelDataType newRowType =
RelOptUtil.permute(join.getCluster().getTypeFactory(), join.getRowType(), mapping);
final RelNode newJoin =
new HiveMultiJoin(
join.getCluster(),
newInputs,
newConditionExpr,
newRowType,
join.getJoinInputs(),
join.getJoinTypes(),
join.getJoinFilters());
return new TrimResult(newJoin, mapping);
}
public static JoinPredicateInfo constructJoinPredicateInfo(
List<RelNode> inputs, List<RelDataTypeField> systemFieldList, RexNode predicate)
throws CalciteSemanticException {
JoinPredicateInfo jpi = null;
JoinLeafPredicateInfo jlpi = null;
List<JoinLeafPredicateInfo> equiLPIList = new ArrayList<JoinLeafPredicateInfo>();
List<JoinLeafPredicateInfo> nonEquiLPIList = new ArrayList<JoinLeafPredicateInfo>();
List<Set<Integer>> projsJoinKeys = new ArrayList<Set<Integer>>();
for (int i = 0; i < inputs.size(); i++) {
Set<Integer> projsJoinKeysInput = Sets.newHashSet();
projsJoinKeys.add(projsJoinKeysInput);
}
List<Set<Integer>> projsJoinKeysInJoinSchema = new ArrayList<Set<Integer>>();
for (int i = 0; i < inputs.size(); i++) {
Set<Integer> projsJoinKeysInJoinSchemaInput = Sets.newHashSet();
projsJoinKeysInJoinSchema.add(projsJoinKeysInJoinSchemaInput);
}
Map<Integer, List<JoinLeafPredicateInfo>> tmpMapOfProjIndxInJoinSchemaToLeafPInfo =
new HashMap<Integer, List<JoinLeafPredicateInfo>>();
Map<Integer, ImmutableList<JoinLeafPredicateInfo>> mapOfProjIndxInJoinSchemaToLeafPInfo =
new HashMap<Integer, ImmutableList<JoinLeafPredicateInfo>>();
List<JoinLeafPredicateInfo> tmpJLPILst = null;
List<RexNode> conjuctiveElements;
// 1. Decompose Join condition to a number of leaf predicates
// (conjuctive elements)
conjuctiveElements = RelOptUtil.conjunctions(predicate);
// 2. Walk through leaf predicates building up JoinLeafPredicateInfo
for (RexNode ce : conjuctiveElements) {
// 2.1 Construct JoinLeafPredicateInfo
jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(inputs, systemFieldList, ce);
// 2.2 Classify leaf predicate as Equi vs Non Equi
if (jlpi.comparisonType.equals(SqlKind.EQUALS)) {
equiLPIList.add(jlpi);
// 2.2.1 Maintain join keys (in child & Join Schema)
// 2.2.2 Update Join Key to JoinLeafPredicateInfo map with keys
for (int i = 0; i < inputs.size(); i++) {
projsJoinKeys.get(i).addAll(jlpi.getProjsJoinKeysInChildSchema(i));
projsJoinKeysInJoinSchema.get(i).addAll(jlpi.getProjsJoinKeysInJoinSchema(i));
for (Integer projIndx : jlpi.getProjsJoinKeysInJoinSchema(i)) {
tmpJLPILst = tmpMapOfProjIndxInJoinSchemaToLeafPInfo.get(projIndx);
if (tmpJLPILst == null) {
tmpJLPILst = new ArrayList<JoinLeafPredicateInfo>();
}
tmpJLPILst.add(jlpi);
tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndx, tmpJLPILst);
}
}
} else {
nonEquiLPIList.add(jlpi);
}
}
// 3. Update Update Join Key to List<JoinLeafPredicateInfo> to use
// ImmutableList
for (Entry<Integer, List<JoinLeafPredicateInfo>> e :
tmpMapOfProjIndxInJoinSchemaToLeafPInfo.entrySet()) {
mapOfProjIndxInJoinSchemaToLeafPInfo.put(e.getKey(), ImmutableList.copyOf(e.getValue()));
}
// 4. Construct JoinPredicateInfo
jpi =
new JoinPredicateInfo(
nonEquiLPIList,
equiLPIList,
projsJoinKeys,
projsJoinKeysInJoinSchema,
mapOfProjIndxInJoinSchemaToLeafPInfo);
return jpi;
}
protected static void log(final String name, final RelNode node, final Logger logger) {
if (logger.isDebugEnabled()) {
logger.debug(name + " : \n" + RelOptUtil.toString(node, SqlExplainLevel.ALL_ATTRIBUTES));
}
}