/** Translates a call to a binary operator. Returns whether successful. */
private boolean translateBinary2(String op, RexNode left, RexNode right) {
switch (right.getKind()) {
case LITERAL:
break;
default:
return false;
}
final RexLiteral rightLiteral = (RexLiteral) right;
switch (left.getKind()) {
case INPUT_REF:
final RexInputRef left1 = (RexInputRef) left;
String name = fieldNames.get(left1.getIndex());
translateOp2(op, name, rightLiteral);
return true;
case CAST:
return translateBinary2(op, ((RexCall) left).operands.get(0), right);
case OTHER_FUNCTION:
String itemName = MongoRules.isItem((RexCall) left);
if (itemName != null) {
translateOp2(op, itemName, rightLiteral);
return true;
}
// fall through
default:
return false;
}
}
public static List<ExprNodeDesc> getExprNodes(
List<Integer> inputRefs, RelNode inputRel, String inputTabAlias) {
List<ExprNodeDesc> exprNodes = new ArrayList<ExprNodeDesc>();
List<RexNode> rexInputRefs = getInputRef(inputRefs, inputRel);
List<RexNode> exprs = inputRel.getChildExps();
// TODO: Change ExprNodeConverter to be independent of Partition Expr
ExprNodeConverter exprConv =
new ExprNodeConverter(
inputTabAlias,
inputRel.getRowType(),
new HashSet<Integer>(),
inputRel.getCluster().getTypeFactory());
for (int index = 0; index < rexInputRefs.size(); index++) {
// The following check is only a guard against failures.
// TODO: Knowing which expr is constant in GBY's aggregation function
// arguments could be better done using Metadata provider of Calcite.
if (exprs != null && index < exprs.size() && exprs.get(index) instanceof RexLiteral) {
ExprNodeDesc exprNodeDesc = exprConv.visitLiteral((RexLiteral) exprs.get(index));
exprNodes.add(exprNodeDesc);
} else {
RexNode iRef = rexInputRefs.get(index);
exprNodes.add(iRef.accept(exprConv));
}
}
return exprNodes;
}
/**
* Given a list of predicates to push down, this methods returns the set of predicates that still
* need to be pushed. Predicates need to be pushed because 1) their String representation is not
* included in input set of predicates to exclude, or 2) they are already in the subtree rooted at
* the input node. This method updates the set of predicates to exclude with the String
* representation of the predicates in the output and in the subtree.
*
* @param predicatesToExclude String representation of predicates that should be excluded
* @param inp root of the subtree
* @param predsToPushDown candidate predicates to push down through the subtree
* @return list of predicates to push down
*/
public static ImmutableList<RexNode> getPredsNotPushedAlready(
Set<String> predicatesToExclude, RelNode inp, List<RexNode> predsToPushDown) {
// Bail out if there is nothing to push
if (predsToPushDown.isEmpty()) {
return ImmutableList.of();
}
// Build map to not convert multiple times, further remove already included predicates
Map<String, RexNode> stringToRexNode = Maps.newLinkedHashMap();
for (RexNode r : predsToPushDown) {
String rexNodeString = r.toString();
if (predicatesToExclude.add(rexNodeString)) {
stringToRexNode.put(rexNodeString, r);
}
}
if (stringToRexNode.isEmpty()) {
return ImmutableList.of();
}
// Finally exclude preds that are already in the subtree as given by the metadata provider
// Note: this is the last step, trying to avoid the expensive call to the metadata provider
// if possible
Set<String> predicatesInSubtree = Sets.newHashSet();
for (RexNode pred : RelMetadataQuery.instance().getPulledUpPredicates(inp).pulledUpPredicates) {
predicatesInSubtree.add(pred.toString());
predicatesInSubtree.addAll(Lists.transform(RelOptUtil.conjunctions(pred), REX_STR_FN));
}
final ImmutableList.Builder<RexNode> newConjuncts = ImmutableList.builder();
for (Entry<String, RexNode> e : stringToRexNode.entrySet()) {
if (predicatesInSubtree.add(e.getKey())) {
newConjuncts.add(e.getValue());
}
}
predicatesToExclude.addAll(predicatesInSubtree);
return newConjuncts.build();
}
/**
* Infers predicates for an Aggregate.
*
* <p>Pulls up predicates that only contains references to columns in the GroupSet. For e.g.
*
* <pre>
* childPullUpExprs : { a > 7, b + c < 10, a + e = 9}
* groupSet : { a, b}
* pulledUpExprs : { a > 7}
* </pre>
*/
public RelOptPredicateList getPredicates(Aggregate agg) {
RelNode child = agg.getInput();
RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child);
List<RexNode> aggPullUpPredicates = new ArrayList<RexNode>();
ImmutableBitSet groupKeys = agg.getGroupSet();
Mapping m =
Mappings.create(
MappingType.PARTIAL_FUNCTION,
child.getRowType().getFieldCount(),
agg.getRowType().getFieldCount());
int i = 0;
for (int j : groupKeys) {
m.set(j, i++);
}
for (RexNode r : childInfo.pulledUpPredicates) {
ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r);
if (groupKeys.contains(rCols)) {
r = r.accept(new RexPermuteInputsShuttle(m, child));
aggPullUpPredicates.add(r);
}
}
return RelOptPredicateList.of(aggPullUpPredicates);
}
public static ExprNodeDesc getExprNode(
Integer inputRefIndx, RelNode inputRel, ExprNodeConverter exprConv) {
ExprNodeDesc exprNode = null;
RexNode rexInputRef =
new RexInputRef(
inputRefIndx, inputRel.getRowType().getFieldList().get(inputRefIndx).getType());
exprNode = rexInputRef.accept(exprConv);
return exprNode;
}
@Override
public String translate(ExprCompiler compiler, RexCall call) {
String val = compiler.reserveName();
PrintWriter pw = compiler.pw;
RexNode op = call.getOperands().get(0);
String lhs = op.accept(compiler);
pw.print(
String.format(
"final %1$s %2$s = (%1$s) %3$s;\n", compiler.javaTypeName(call), val, lhs));
return val;
}
private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticException {
RexNode rexNode = convert(fieldDesc.getDesc());
if (rexNode instanceof RexCall) {
// regular case of accessing nested field in a column
return cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true);
} else {
// This may happen for schema-less tables, where columns are dynamically
// supplied by serdes.
throw new CalciteSemanticException(
"Unexpected rexnode : " + rexNode.getClass().getCanonicalName(),
UnsupportedFeature.Schema_less_table);
}
}
/**
* Infers predicates for a project.
*
* <ol>
* <li>create a mapping from input to projection. Map only positions that directly reference an
* input column.
* <li>Expressions that only contain above columns are retained in the Project's pullExpressions
* list.
* <li>For e.g. expression 'a + e = 9' below will not be pulled up because 'e' is not in the
* projection list.
* <pre>
* childPullUpExprs: {a > 7, b + c < 10, a + e = 9}
* projectionExprs: {a, b, c, e / 2}
* projectionPullupExprs: {a > 7, b + c < 10}
* </pre>
* </ol>
*/
public RelOptPredicateList getPredicates(Project project) {
RelNode child = project.getInput();
final RexBuilder rexBuilder = project.getCluster().getRexBuilder();
RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child);
List<RexNode> projectPullUpPredicates = new ArrayList<RexNode>();
ImmutableBitSet.Builder columnsMappedBuilder = ImmutableBitSet.builder();
Mapping m =
Mappings.create(
MappingType.PARTIAL_FUNCTION,
child.getRowType().getFieldCount(),
project.getRowType().getFieldCount());
for (Ord<RexNode> o : Ord.zip(project.getProjects())) {
if (o.e instanceof RexInputRef) {
int sIdx = ((RexInputRef) o.e).getIndex();
m.set(sIdx, o.i);
columnsMappedBuilder.set(sIdx);
}
}
// Go over childPullUpPredicates. If a predicate only contains columns in
// 'columnsMapped' construct a new predicate based on mapping.
final ImmutableBitSet columnsMapped = columnsMappedBuilder.build();
for (RexNode r : childInfo.pulledUpPredicates) {
ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r);
if (columnsMapped.contains(rCols)) {
r = r.accept(new RexPermuteInputsShuttle(m, child));
projectPullUpPredicates.add(r);
}
}
// Project can also generate constants. We need to include them.
for (Ord<RexNode> expr : Ord.zip(project.getProjects())) {
if (RexLiteral.isNullLiteral(expr.e)) {
projectPullUpPredicates.add(
rexBuilder.makeCall(
SqlStdOperatorTable.IS_NULL, rexBuilder.makeInputRef(project, expr.i)));
} else if (RexUtil.isConstant(expr.e)) {
final List<RexNode> args =
ImmutableList.of(rexBuilder.makeInputRef(project, expr.i), expr.e);
final SqlOperator op =
args.get(0).getType().isNullable() || args.get(1).getType().isNullable()
? SqlStdOperatorTable.IS_NOT_DISTINCT_FROM
: SqlStdOperatorTable.EQUALS;
projectPullUpPredicates.add(rexBuilder.makeCall(op, args));
}
}
return RelOptPredicateList.of(projectPullUpPredicates);
}
public Double getDistinctRowCount(Aggregate rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
// determine which predicates can be applied on the child of the
// aggregate
List<RexNode> notPushable = new ArrayList<RexNode>();
List<RexNode> pushable = new ArrayList<RexNode>();
RelOptUtil.splitFilters(rel.getGroupSet(), predicate, pushable, notPushable);
final RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
RexNode childPreds = RexUtil.composeConjunction(rexBuilder, pushable, true);
// set the bits as they correspond to the child input
ImmutableBitSet.Builder childKey = ImmutableBitSet.builder();
RelMdUtil.setAggChildKeys(groupKey, rel, childKey);
Double distinctRowCount =
RelMetadataQuery.getDistinctRowCount(rel.getInput(), childKey.build(), childPreds);
if (distinctRowCount == null) {
return null;
} else if (notPushable.isEmpty()) {
return distinctRowCount;
} else {
RexNode preds = RexUtil.composeConjunction(rexBuilder, notPushable, true);
return distinctRowCount * RelMdUtil.guessSelectivity(preds);
}
}
/**
* Returns whether a condition is supported by {@link JdbcJoin}.
*
* <p>Corresponds to the capabilities of {@link SqlImplementor#convertConditionToSqlNode}.
*
* @param node Condition
* @return Whether condition is supported
*/
private boolean canJoinOnCondition(RexNode node) {
final List<RexNode> operands;
switch (node.getKind()) {
case AND:
case OR:
operands = ((RexCall) node).getOperands();
for (RexNode operand : operands) {
if (!canJoinOnCondition(operand)) {
return false;
}
}
return true;
case EQUALS:
case IS_NOT_DISTINCT_FROM:
case NOT_EQUALS:
case GREATER_THAN:
case GREATER_THAN_OR_EQUAL:
case LESS_THAN:
case LESS_THAN_OR_EQUAL:
operands = ((RexCall) node).getOperands();
if ((operands.get(0) instanceof RexInputRef)
&& (operands.get(1) instanceof RexInputRef)) {
return true;
}
// fall through
default:
return false;
}
}
@Override
public String translate(ExprCompiler compiler, RexCall call) {
String val = compiler.reserveName();
PrintWriter pw = compiler.pw;
RexNode op = call.getOperands().get(0);
String lhs = op.accept(compiler);
boolean nullable = call.getType().isNullable();
pw.print(String.format("final %s %s;\n", compiler.javaTypeName(call), val));
if (!nullable) {
pw.print(String.format("%1$s = !(%2$s);\n", val, lhs));
} else {
String s =
foldNullExpr(String.format("%1$s == null ? null : !(%1$s)", lhs), "null", op);
pw.print(String.format("%1$s = %2$s;\n", val, s));
}
return val;
}
public Double getDistinctRowCount(Join rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
return RelMdUtil.getJoinDistinctRowCount(rel, rel.getJoinType(), groupKey, predicate, false);
}
private boolean isAlwaysTrue(RexNode predicate) {
if (predicate instanceof RexCall) {
RexCall c = (RexCall) predicate;
if (c.getOperator().getKind() == SqlKind.EQUALS) {
int lPos = pos(c.getOperands().get(0));
int rPos = pos(c.getOperands().get(1));
return lPos != -1 && lPos == rPos;
}
}
return predicate.isAlwaysTrue();
}
public Double getDistinctRowCount(Project rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
ImmutableBitSet.Builder baseCols = ImmutableBitSet.builder();
ImmutableBitSet.Builder projCols = ImmutableBitSet.builder();
List<RexNode> projExprs = rel.getProjects();
RelMdUtil.splitCols(projExprs, groupKey, baseCols, projCols);
List<RexNode> notPushable = new ArrayList<RexNode>();
List<RexNode> pushable = new ArrayList<RexNode>();
RelOptUtil.splitFilters(
ImmutableBitSet.range(rel.getRowType().getFieldCount()), predicate, pushable, notPushable);
final RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
// get the distinct row count of the child input, passing in the
// columns and filters that only reference the child; convert the
// filter to reference the children projection expressions
RexNode childPred = RexUtil.composeConjunction(rexBuilder, pushable, true);
RexNode modifiedPred;
if (childPred == null) {
modifiedPred = null;
} else {
modifiedPred = RelOptUtil.pushPastProject(childPred, rel);
}
Double distinctRowCount =
RelMetadataQuery.getDistinctRowCount(rel.getInput(), baseCols.build(), modifiedPred);
if (distinctRowCount == null) {
return null;
} else if (!notPushable.isEmpty()) {
RexNode preds = RexUtil.composeConjunction(rexBuilder, notPushable, true);
distinctRowCount *= RelMdUtil.guessSelectivity(preds);
}
// No further computation required if the projection expressions
// are all column references
if (projCols.cardinality() == 0) {
return distinctRowCount;
}
// multiply by the cardinality of the non-child projection expressions
for (int bit : projCols.build()) {
Double subRowCount = RelMdUtil.cardOfProjExpr(rel, projExprs.get(bit));
if (subRowCount == null) {
return null;
}
distinctRowCount *= subRowCount;
}
return RelMdUtil.numDistinctVals(distinctRowCount, RelMetadataQuery.getRowCount(rel));
}
public Double getDistinctRowCount(Values rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
Double selectivity = RelMdUtil.guessSelectivity(predicate);
// assume half the rows are duplicates
Double nRows = rel.getRows() / 2;
return RelMdUtil.numDistinctVals(nRows, nRows * selectivity);
}
public static boolean isDeterministicFuncOnLiterals(RexNode expr) {
boolean deterministicFuncOnLiterals = true;
RexVisitor<Void> visitor =
new RexVisitorImpl<Void>(true) {
@Override
public Void visitCall(org.apache.calcite.rex.RexCall call) {
if (!call.getOperator().isDeterministic()) {
throw new Util.FoundOne(call);
}
return super.visitCall(call);
}
@Override
public Void visitInputRef(RexInputRef inputRef) {
throw new Util.FoundOne(inputRef);
}
@Override
public Void visitLocalRef(RexLocalRef localRef) {
throw new Util.FoundOne(localRef);
}
@Override
public Void visitOver(RexOver over) {
throw new Util.FoundOne(over);
}
@Override
public Void visitDynamicParam(RexDynamicParam dynamicParam) {
throw new Util.FoundOne(dynamicParam);
}
@Override
public Void visitRangeRef(RexRangeRef rangeRef) {
throw new Util.FoundOne(rangeRef);
}
@Override
public Void visitFieldAccess(RexFieldAccess fieldAccess) {
throw new Util.FoundOne(fieldAccess);
}
};
try {
expr.accept(visitor);
} catch (Util.FoundOne e) {
deterministicFuncOnLiterals = false;
}
return deterministicFuncOnLiterals;
}
public Double getDistinctRowCount(Filter rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
// REVIEW zfong 4/18/06 - In the Broadbase code, duplicates are not
// removed from the two filter lists. However, the code below is
// doing so.
RexNode unionPreds =
RelMdUtil.unionPreds(rel.getCluster().getRexBuilder(), predicate, rel.getCondition());
return RelMetadataQuery.getDistinctRowCount(rel.getInput(), groupKey, unionPreds);
}
public Double getDistinctRowCount(SemiJoin rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
// create a RexNode representing the selectivity of the
// semijoin filter and pass it to getDistinctRowCount
RexNode newPred = RelMdUtil.makeSemiJoinSelectivityRexNode(rel);
if (predicate != null) {
RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
newPred = rexBuilder.makeCall(SqlStdOperatorTable.AND, newPred, predicate);
}
return RelMetadataQuery.getDistinctRowCount(rel.getLeft(), groupKey, newPred);
}
// Catch-all rule when none of the others apply.
public Double getDistinctRowCount(RelNode rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
// REVIEW zfong 4/19/06 - Broadbase code does not take into
// consideration selectivity of predicates passed in. Also, they
// assume the rows are unique even if the table is not
boolean uniq = RelMdUtil.areColumnsDefinitelyUnique(rel, groupKey);
if (uniq) {
return NumberUtil.multiply(
RelMetadataQuery.getRowCount(rel), RelMetadataQuery.getSelectivity(rel, predicate));
}
return null;
}
private Void translateMatch2(RexNode node) {
switch (node.getKind()) {
case EQUALS:
return translateBinary(null, null, (RexCall) node);
case LESS_THAN:
return translateBinary("$lt", "$gt", (RexCall) node);
case LESS_THAN_OR_EQUAL:
return translateBinary("$lte", "$gte", (RexCall) node);
case NOT_EQUALS:
return translateBinary("$ne", "$ne", (RexCall) node);
case GREATER_THAN:
return translateBinary("$gt", "$lt", (RexCall) node);
case GREATER_THAN_OR_EQUAL:
return translateBinary("$gte", "$lte", (RexCall) node);
default:
throw new AssertionError("cannot translate " + node);
}
}
public static boolean isDeterministic(RexNode expr) {
boolean deterministic = true;
RexVisitor<Void> visitor =
new RexVisitorImpl<Void>(true) {
@Override
public Void visitCall(org.apache.calcite.rex.RexCall call) {
if (!call.getOperator().isDeterministic()) {
throw new Util.FoundOne(call);
}
return super.visitCall(call);
}
};
try {
expr.accept(visitor);
} catch (Util.FoundOne e) {
deterministic = false;
}
return deterministic;
}
public Double getDistinctRowCount(Union rel, ImmutableBitSet groupKey, RexNode predicate) {
Double rowCount = 0.0;
int[] adjustments = new int[rel.getRowType().getFieldCount()];
RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
for (RelNode input : rel.getInputs()) {
// convert the predicate to reference the types of the union child
RexNode modifiedPred;
if (predicate == null) {
modifiedPred = null;
} else {
modifiedPred =
predicate.accept(
new RelOptUtil.RexInputConverter(
rexBuilder, null, input.getRowType().getFieldList(), adjustments));
}
Double partialRowCount = RelMetadataQuery.getDistinctRowCount(input, groupKey, modifiedPred);
if (partialRowCount == null) {
return null;
}
rowCount += partialRowCount;
}
return rowCount;
}
@Override
public String translate(ExprCompiler compiler, RexCall call) {
String val = compiler.reserveName();
PrintWriter pw = compiler.pw;
pw.print(String.format("final %s %s;\n", compiler.javaTypeName(call), val));
RexNode op0 = call.getOperands().get(0);
RexNode op1 = call.getOperands().get(1);
boolean lhsNullable = op0.getType().isNullable();
boolean rhsNullable = op1.getType().isNullable();
String lhs = op0.accept(compiler);
if (!lhsNullable) {
pw.print(String.format("if (%2$s) { %1$s = true; }\n", val, lhs));
pw.print("else {\n");
String rhs = op1.accept(compiler);
pw.print(String.format(" %1$s = %2$s;\n}\n", val, rhs));
} else {
String foldedLHS =
foldNullExpr(String.format("%1$s == null || !(%1$s)", lhs), "true", op0);
pw.print(String.format("if (%s) {\n", foldedLHS));
String rhs = op1.accept(compiler);
String s;
if (rhsNullable) {
s =
foldNullExpr(
String.format(
"(%2$s != null && %2$s) ? Boolean.TRUE : ((%1$s == null || %2$s == null) ? null : Boolean.FALSE)",
lhs, rhs),
"null",
op1);
} else {
s = String.format("%2$s ? Boolean.valueOf(%2$s) : %1$s", lhs, rhs);
}
pw.print(String.format(" %1$s = %2$s;\n", val, s));
pw.print(String.format("} else { %1$s = true; }\n", val));
}
return val;
}
public Double averageRexSize(RexNode node, List<Double> inputColumnSizes) {
switch (node.getKind()) {
case INPUT_REF:
return inputColumnSizes.get(((RexInputRef) node).getIndex());
case LITERAL:
return typeValueSize(node.getType(), ((RexLiteral) node).getValue());
default:
if (node instanceof RexCall) {
RexCall call = (RexCall) node;
for (RexNode operand : call.getOperands()) {
// It's a reasonable assumption that a function's result will have
// similar size to its argument of a similar type. For example,
// UPPER(c) has the same average size as c.
if (operand.getType().getSqlTypeName() == node.getType().getSqlTypeName()) {
return averageRexSize(operand, inputColumnSizes);
}
}
}
return averageTypeValueSize(node.getType());
}
}
private void infer(
RexNode predicates,
Set<String> allExprsDigests,
List<RexNode> inferedPredicates,
boolean includeEqualityInference,
ImmutableBitSet inferringFields) {
for (RexNode r : RelOptUtil.conjunctions(predicates)) {
if (!includeEqualityInference && equalityPredicates.contains(r.toString())) {
continue;
}
for (Mapping m : mappings(r)) {
RexNode tr =
r.accept(new RexPermuteInputsShuttle(m, joinRel.getInput(0), joinRel.getInput(1)));
if (inferringFields.contains(RelOptUtil.InputFinder.bits(tr))
&& !allExprsDigests.contains(tr.toString())
&& !isAlwaysTrue(tr)) {
inferedPredicates.add(tr);
allExprsDigests.add(tr.toString());
}
}
}
}
@Test
public void testSplitFilter() {
final RexLiteral i1 = rexBuilder.makeExactLiteral(BigDecimal.ONE);
final RexLiteral i2 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(2));
final RexLiteral i3 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(3));
final RelDataType intType = typeFactory.createType(int.class);
final RexInputRef x = rexBuilder.makeInputRef(intType, 0); // $0
final RexInputRef y = rexBuilder.makeInputRef(intType, 1); // $1
final RexInputRef z = rexBuilder.makeInputRef(intType, 2); // $2
final RexNode x_eq_1 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1
final RexNode x_eq_1_b = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1 again
final RexNode y_eq_2 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, y, i2); // $1 = 2
final RexNode z_eq_3 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, z, i3); // $2 = 3
RexNode newFilter;
// Example 1.
// TODO:
// Example 2.
// condition: x = 1,
// target: x = 1 or z = 3
// yields
// residue: not (z = 3)
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder, x_eq_1, rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, z_eq_3));
assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))"));
// 2b.
// condition: x = 1 or y = 2
// target: x = 1 or y = 2 or z = 3
// yields
// residue: not (z = 3)
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder,
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2),
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3));
assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))"));
// 2c.
// condition: x = 1
// target: x = 1 or y = 2 or z = 3
// yields
// residue: not (y = 2) and not (z = 3)
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder,
x_eq_1,
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3));
assertThat(newFilter.toString(), equalTo("AND(NOT(=($1, 2)), NOT(=($2, 3)))"));
// 2d.
// condition: x = 1 or y = 2
// target: y = 2 or x = 1
// yields
// residue: true
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder,
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2),
rexBuilder.makeCall(SqlStdOperatorTable.OR, y_eq_2, x_eq_1));
assertThat(newFilter.isAlwaysTrue(), equalTo(true));
// 2e.
// condition: x = 1
// target: x = 1 (different object)
// yields
// residue: true
newFilter = SubstitutionVisitor.splitFilter(rexBuilder, x_eq_1, x_eq_1_b);
assertThat(newFilter.isAlwaysTrue(), equalTo(true));
// 2f.
// condition: x = 1 or y = 2
// target: x = 1
// yields
// residue: null
// TODO:
// Example 3.
// Condition [x = 1 and y = 2],
// target [y = 2 and x = 1] yields
// residue [true].
// TODO:
// Example 4.
// TODO:
}
private void checkSatisfiable(RexNode e, String s) {
assertTrue(SubstitutionVisitor.mayBeSatisfiable(e));
final RexNode simple = SubstitutionVisitor.simplify(rexBuilder, e);
assertEquals(s, simple.toString());
}
/**
* The PullUp Strategy is sound but not complete.
*
* <ol>
* <li>We only pullUp inferred predicates for now. Pulling up existing predicates causes an
* explosion of duplicates. The existing predicates are pushed back down as new
* predicates. Once we have rules to eliminate duplicate Filter conditions, we should
* pullUp all predicates.
* <li>For Left Outer: we infer new predicates from the left and set them as applicable on the
* Right side. No predicates are pulledUp.
* <li>Right Outer Joins are handled in an analogous manner.
* <li>For Full Outer Joins no predicates are pulledUp or inferred.
* </ol>
*/
public RelOptPredicateList inferPredicates(boolean includeEqualityInference) {
List<RexNode> inferredPredicates = new ArrayList<RexNode>();
Set<String> allExprsDigests = new HashSet<String>(this.allExprsDigests);
final JoinRelType joinType = joinRel.getJoinType();
switch (joinType) {
case INNER:
case LEFT:
infer(
leftChildPredicates,
allExprsDigests,
inferredPredicates,
includeEqualityInference,
joinType == JoinRelType.LEFT ? rightFieldsBitSet : allFieldsBitSet);
break;
}
switch (joinType) {
case INNER:
case RIGHT:
infer(
rightChildPredicates,
allExprsDigests,
inferredPredicates,
includeEqualityInference,
joinType == JoinRelType.RIGHT ? leftFieldsBitSet : allFieldsBitSet);
break;
}
Mappings.TargetMapping rightMapping =
Mappings.createShiftMapping(
nSysFields + nFieldsLeft + nFieldsRight, 0, nSysFields + nFieldsLeft, nFieldsRight);
final RexPermuteInputsShuttle rightPermute =
new RexPermuteInputsShuttle(rightMapping, joinRel);
Mappings.TargetMapping leftMapping =
Mappings.createShiftMapping(nSysFields + nFieldsLeft, 0, nSysFields, nFieldsLeft);
final RexPermuteInputsShuttle leftPermute = new RexPermuteInputsShuttle(leftMapping, joinRel);
List<RexNode> leftInferredPredicates = new ArrayList<RexNode>();
List<RexNode> rightInferredPredicates = new ArrayList<RexNode>();
for (RexNode iP : inferredPredicates) {
ImmutableBitSet iPBitSet = RelOptUtil.InputFinder.bits(iP);
if (leftFieldsBitSet.contains(iPBitSet)) {
leftInferredPredicates.add(iP.accept(leftPermute));
} else if (rightFieldsBitSet.contains(iPBitSet)) {
rightInferredPredicates.add(iP.accept(rightPermute));
}
}
switch (joinType) {
case INNER:
Iterable<RexNode> pulledUpPredicates;
if (isSemiJoin) {
pulledUpPredicates =
Iterables.concat(
RelOptUtil.conjunctions(leftChildPredicates), leftInferredPredicates);
} else {
pulledUpPredicates =
Iterables.concat(
RelOptUtil.conjunctions(leftChildPredicates),
RelOptUtil.conjunctions(rightChildPredicates),
RelOptUtil.conjunctions(joinRel.getCondition()),
inferredPredicates);
}
return RelOptPredicateList.of(
pulledUpPredicates, leftInferredPredicates, rightInferredPredicates);
case LEFT:
return RelOptPredicateList.of(
RelOptUtil.conjunctions(leftChildPredicates),
leftInferredPredicates,
rightInferredPredicates);
case RIGHT:
return RelOptPredicateList.of(
RelOptUtil.conjunctions(rightChildPredicates), inferredPredicates, EMPTY_LIST);
default:
assert inferredPredicates.size() == 0;
return RelOptPredicateList.EMPTY;
}
}
private JoinConditionBasedPredicateInference(
Join joinRel, boolean isSemiJoin, RexNode lPreds, RexNode rPreds) {
super();
this.joinRel = joinRel;
this.isSemiJoin = isSemiJoin;
nFieldsLeft = joinRel.getLeft().getRowType().getFieldList().size();
nFieldsRight = joinRel.getRight().getRowType().getFieldList().size();
nSysFields = joinRel.getSystemFieldList().size();
leftFieldsBitSet = ImmutableBitSet.range(nSysFields, nSysFields + nFieldsLeft);
rightFieldsBitSet =
ImmutableBitSet.range(nSysFields + nFieldsLeft, nSysFields + nFieldsLeft + nFieldsRight);
allFieldsBitSet = ImmutableBitSet.range(0, nSysFields + nFieldsLeft + nFieldsRight);
exprFields = Maps.newHashMap();
allExprsDigests = new HashSet<String>();
if (lPreds == null) {
leftChildPredicates = null;
} else {
Mappings.TargetMapping leftMapping =
Mappings.createShiftMapping(nSysFields + nFieldsLeft, nSysFields, 0, nFieldsLeft);
leftChildPredicates =
lPreds.accept(new RexPermuteInputsShuttle(leftMapping, joinRel.getInput(0)));
for (RexNode r : RelOptUtil.conjunctions(leftChildPredicates)) {
exprFields.put(r.toString(), RelOptUtil.InputFinder.bits(r));
allExprsDigests.add(r.toString());
}
}
if (rPreds == null) {
rightChildPredicates = null;
} else {
Mappings.TargetMapping rightMapping =
Mappings.createShiftMapping(
nSysFields + nFieldsLeft + nFieldsRight, nSysFields + nFieldsLeft, 0, nFieldsRight);
rightChildPredicates =
rPreds.accept(new RexPermuteInputsShuttle(rightMapping, joinRel.getInput(1)));
for (RexNode r : RelOptUtil.conjunctions(rightChildPredicates)) {
exprFields.put(r.toString(), RelOptUtil.InputFinder.bits(r));
allExprsDigests.add(r.toString());
}
}
equivalence = Maps.newTreeMap();
equalityPredicates = new HashSet<String>();
for (int i = 0; i < nSysFields + nFieldsLeft + nFieldsRight; i++) {
equivalence.put(i, BitSets.of(i));
}
// Only process equivalences found in the join conditions. Processing
// Equivalences from the left or right side infer predicates that are
// already present in the Tree below the join.
RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder();
List<RexNode> exprs =
RelOptUtil.conjunctions(compose(rexBuilder, ImmutableList.of(joinRel.getCondition())));
final EquivalenceFinder eF = new EquivalenceFinder();
new ArrayList<Void>(
Lists.transform(
exprs,
new Function<RexNode, Void>() {
public Void apply(RexNode input) {
return input.accept(eF);
}
}));
equivalence = BitSets.closure(equivalence);
}
private static void splitJoinCondition(
List<RelDataTypeField> sysFieldList,
List<RelNode> inputs,
RexNode condition,
List<List<RexNode>> joinKeys,
List<Integer> filterNulls,
List<SqlOperator> rangeOp,
List<RexNode> nonEquiList)
throws CalciteSemanticException {
final int sysFieldCount = sysFieldList.size();
final RelOptCluster cluster = inputs.get(0).getCluster();
final RexBuilder rexBuilder = cluster.getRexBuilder();
if (condition instanceof RexCall) {
RexCall call = (RexCall) condition;
if (call.getOperator() == SqlStdOperatorTable.AND) {
for (RexNode operand : call.getOperands()) {
splitJoinCondition(
sysFieldList, inputs, operand, joinKeys, filterNulls, rangeOp, nonEquiList);
}
return;
}
RexNode leftKey = null;
RexNode rightKey = null;
int leftInput = 0;
int rightInput = 0;
List<RelDataTypeField> leftFields = null;
List<RelDataTypeField> rightFields = null;
boolean reverse = false;
SqlKind kind = call.getKind();
// Only consider range operators if we haven't already seen one
if ((kind == SqlKind.EQUALS)
|| (filterNulls != null && kind == SqlKind.IS_NOT_DISTINCT_FROM)
|| (rangeOp != null
&& rangeOp.isEmpty()
&& (kind == SqlKind.GREATER_THAN
|| kind == SqlKind.GREATER_THAN_OR_EQUAL
|| kind == SqlKind.LESS_THAN
|| kind == SqlKind.LESS_THAN_OR_EQUAL))) {
final List<RexNode> operands = call.getOperands();
RexNode op0 = operands.get(0);
RexNode op1 = operands.get(1);
final ImmutableBitSet projRefs0 = InputFinder.bits(op0);
final ImmutableBitSet projRefs1 = InputFinder.bits(op1);
final ImmutableBitSet[] inputsRange = new ImmutableBitSet[inputs.size()];
int totalFieldCount = 0;
for (int i = 0; i < inputs.size(); i++) {
final int firstField = totalFieldCount + sysFieldCount;
totalFieldCount = firstField + inputs.get(i).getRowType().getFieldCount();
inputsRange[i] = ImmutableBitSet.range(firstField, totalFieldCount);
}
boolean foundBothInputs = false;
for (int i = 0; i < inputs.size() && !foundBothInputs; i++) {
if (projRefs0.intersects(inputsRange[i])
&& projRefs0.union(inputsRange[i]).equals(inputsRange[i])) {
if (leftKey == null) {
leftKey = op0;
leftInput = i;
leftFields = inputs.get(leftInput).getRowType().getFieldList();
} else {
rightKey = op0;
rightInput = i;
rightFields = inputs.get(rightInput).getRowType().getFieldList();
reverse = true;
foundBothInputs = true;
}
} else if (projRefs1.intersects(inputsRange[i])
&& projRefs1.union(inputsRange[i]).equals(inputsRange[i])) {
if (leftKey == null) {
leftKey = op1;
leftInput = i;
leftFields = inputs.get(leftInput).getRowType().getFieldList();
} else {
rightKey = op1;
rightInput = i;
rightFields = inputs.get(rightInput).getRowType().getFieldList();
foundBothInputs = true;
}
}
}
if ((leftKey != null) && (rightKey != null)) {
// adjustment array
int[] adjustments = new int[totalFieldCount];
for (int i = 0; i < inputs.size(); i++) {
final int adjustment = inputsRange[i].nextSetBit(0);
for (int j = adjustment; j < inputsRange[i].length(); j++) {
adjustments[j] = -adjustment;
}
}
// replace right Key input ref
rightKey =
rightKey.accept(
new RelOptUtil.RexInputConverter(
rexBuilder, rightFields, rightFields, adjustments));
// left key only needs to be adjusted if there are system
// fields, but do it for uniformity
leftKey =
leftKey.accept(
new RelOptUtil.RexInputConverter(
rexBuilder, leftFields, leftFields, adjustments));
RelDataType leftKeyType = leftKey.getType();
RelDataType rightKeyType = rightKey.getType();
if (leftKeyType != rightKeyType) {
// perform casting using Hive rules
TypeInfo rType = TypeConverter.convert(rightKeyType);
TypeInfo lType = TypeConverter.convert(leftKeyType);
TypeInfo tgtType = FunctionRegistry.getCommonClassForComparison(lType, rType);
if (tgtType == null) {
throw new CalciteSemanticException(
"Cannot find common type for join keys "
+ leftKey
+ " (type "
+ leftKeyType
+ ") and "
+ rightKey
+ " (type "
+ rightKeyType
+ ")");
}
RelDataType targetKeyType = TypeConverter.convert(tgtType, rexBuilder.getTypeFactory());
if (leftKeyType != targetKeyType
&& TypeInfoUtils.isConversionRequiredForComparison(tgtType, lType)) {
leftKey = rexBuilder.makeCast(targetKeyType, leftKey);
}
if (rightKeyType != targetKeyType
&& TypeInfoUtils.isConversionRequiredForComparison(tgtType, rType)) {
rightKey = rexBuilder.makeCast(targetKeyType, rightKey);
}
}
}
}
if ((leftKey != null) && (rightKey != null)) {
// found suitable join keys
// add them to key list, ensuring that if there is a
// non-equi join predicate, it appears at the end of the
// key list; also mark the null filtering property
addJoinKey(joinKeys.get(leftInput), leftKey, (rangeOp != null) && !rangeOp.isEmpty());
addJoinKey(joinKeys.get(rightInput), rightKey, (rangeOp != null) && !rangeOp.isEmpty());
if (filterNulls != null && kind == SqlKind.EQUALS) {
// nulls are considered not matching for equality comparison
// add the position of the most recently inserted key
filterNulls.add(joinKeys.get(leftInput).size() - 1);
}
if (rangeOp != null && kind != SqlKind.EQUALS && kind != SqlKind.IS_DISTINCT_FROM) {
if (reverse) {
kind = reverse(kind);
}
rangeOp.add(op(kind, call.getOperator()));
}
return;
} // else fall through and add this condition as nonEqui condition
}
// The operator is not of RexCall type
// So we fail. Fall through.
// Add this condition to the list of non-equi-join conditions.
nonEquiList.add(condition);
}
