/** Translates a call to a binary operator. Returns whether successful. */ private boolean translateBinary2(String op, RexNode left, RexNode right) { switch (right.getKind()) { case LITERAL: break; default: return false; } final RexLiteral rightLiteral = (RexLiteral) right; switch (left.getKind()) { case INPUT_REF: final RexInputRef left1 = (RexInputRef) left; String name = fieldNames.get(left1.getIndex()); translateOp2(op, name, rightLiteral); return true; case CAST: return translateBinary2(op, ((RexCall) left).operands.get(0), right); case OTHER_FUNCTION: String itemName = MongoRules.isItem((RexCall) left); if (itemName != null) { translateOp2(op, itemName, rightLiteral); return true; } // fall through default: return false; } }
public static List<ExprNodeDesc> getExprNodes( List<Integer> inputRefs, RelNode inputRel, String inputTabAlias) { List<ExprNodeDesc> exprNodes = new ArrayList<ExprNodeDesc>(); List<RexNode> rexInputRefs = getInputRef(inputRefs, inputRel); List<RexNode> exprs = inputRel.getChildExps(); // TODO: Change ExprNodeConverter to be independent of Partition Expr ExprNodeConverter exprConv = new ExprNodeConverter( inputTabAlias, inputRel.getRowType(), new HashSet<Integer>(), inputRel.getCluster().getTypeFactory()); for (int index = 0; index < rexInputRefs.size(); index++) { // The following check is only a guard against failures. // TODO: Knowing which expr is constant in GBY's aggregation function // arguments could be better done using Metadata provider of Calcite. if (exprs != null && index < exprs.size() && exprs.get(index) instanceof RexLiteral) { ExprNodeDesc exprNodeDesc = exprConv.visitLiteral((RexLiteral) exprs.get(index)); exprNodes.add(exprNodeDesc); } else { RexNode iRef = rexInputRefs.get(index); exprNodes.add(iRef.accept(exprConv)); } } return exprNodes; }
/** * Given a list of predicates to push down, this methods returns the set of predicates that still * need to be pushed. Predicates need to be pushed because 1) their String representation is not * included in input set of predicates to exclude, or 2) they are already in the subtree rooted at * the input node. This method updates the set of predicates to exclude with the String * representation of the predicates in the output and in the subtree. * * @param predicatesToExclude String representation of predicates that should be excluded * @param inp root of the subtree * @param predsToPushDown candidate predicates to push down through the subtree * @return list of predicates to push down */ public static ImmutableList<RexNode> getPredsNotPushedAlready( Set<String> predicatesToExclude, RelNode inp, List<RexNode> predsToPushDown) { // Bail out if there is nothing to push if (predsToPushDown.isEmpty()) { return ImmutableList.of(); } // Build map to not convert multiple times, further remove already included predicates Map<String, RexNode> stringToRexNode = Maps.newLinkedHashMap(); for (RexNode r : predsToPushDown) { String rexNodeString = r.toString(); if (predicatesToExclude.add(rexNodeString)) { stringToRexNode.put(rexNodeString, r); } } if (stringToRexNode.isEmpty()) { return ImmutableList.of(); } // Finally exclude preds that are already in the subtree as given by the metadata provider // Note: this is the last step, trying to avoid the expensive call to the metadata provider // if possible Set<String> predicatesInSubtree = Sets.newHashSet(); for (RexNode pred : RelMetadataQuery.instance().getPulledUpPredicates(inp).pulledUpPredicates) { predicatesInSubtree.add(pred.toString()); predicatesInSubtree.addAll(Lists.transform(RelOptUtil.conjunctions(pred), REX_STR_FN)); } final ImmutableList.Builder<RexNode> newConjuncts = ImmutableList.builder(); for (Entry<String, RexNode> e : stringToRexNode.entrySet()) { if (predicatesInSubtree.add(e.getKey())) { newConjuncts.add(e.getValue()); } } predicatesToExclude.addAll(predicatesInSubtree); return newConjuncts.build(); }
/** * Infers predicates for an Aggregate. * * <p>Pulls up predicates that only contains references to columns in the GroupSet. For e.g. * * <pre> * childPullUpExprs : { a > 7, b + c < 10, a + e = 9} * groupSet : { a, b} * pulledUpExprs : { a > 7} * </pre> */ public RelOptPredicateList getPredicates(Aggregate agg) { RelNode child = agg.getInput(); RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child); List<RexNode> aggPullUpPredicates = new ArrayList<RexNode>(); ImmutableBitSet groupKeys = agg.getGroupSet(); Mapping m = Mappings.create( MappingType.PARTIAL_FUNCTION, child.getRowType().getFieldCount(), agg.getRowType().getFieldCount()); int i = 0; for (int j : groupKeys) { m.set(j, i++); } for (RexNode r : childInfo.pulledUpPredicates) { ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r); if (groupKeys.contains(rCols)) { r = r.accept(new RexPermuteInputsShuttle(m, child)); aggPullUpPredicates.add(r); } } return RelOptPredicateList.of(aggPullUpPredicates); }
public static ExprNodeDesc getExprNode( Integer inputRefIndx, RelNode inputRel, ExprNodeConverter exprConv) { ExprNodeDesc exprNode = null; RexNode rexInputRef = new RexInputRef( inputRefIndx, inputRel.getRowType().getFieldList().get(inputRefIndx).getType()); exprNode = rexInputRef.accept(exprConv); return exprNode; }
@Override public String translate(ExprCompiler compiler, RexCall call) { String val = compiler.reserveName(); PrintWriter pw = compiler.pw; RexNode op = call.getOperands().get(0); String lhs = op.accept(compiler); pw.print( String.format( "final %1$s %2$s = (%1$s) %3$s;\n", compiler.javaTypeName(call), val, lhs)); return val; }
private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticException { RexNode rexNode = convert(fieldDesc.getDesc()); if (rexNode instanceof RexCall) { // regular case of accessing nested field in a column return cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true); } else { // This may happen for schema-less tables, where columns are dynamically // supplied by serdes. throw new CalciteSemanticException( "Unexpected rexnode : " + rexNode.getClass().getCanonicalName(), UnsupportedFeature.Schema_less_table); } }
/** * Infers predicates for a project. * * <ol> * <li>create a mapping from input to projection. Map only positions that directly reference an * input column. * <li>Expressions that only contain above columns are retained in the Project's pullExpressions * list. * <li>For e.g. expression 'a + e = 9' below will not be pulled up because 'e' is not in the * projection list. * <pre> * childPullUpExprs: {a > 7, b + c < 10, a + e = 9} * projectionExprs: {a, b, c, e / 2} * projectionPullupExprs: {a > 7, b + c < 10} * </pre> * </ol> */ public RelOptPredicateList getPredicates(Project project) { RelNode child = project.getInput(); final RexBuilder rexBuilder = project.getCluster().getRexBuilder(); RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child); List<RexNode> projectPullUpPredicates = new ArrayList<RexNode>(); ImmutableBitSet.Builder columnsMappedBuilder = ImmutableBitSet.builder(); Mapping m = Mappings.create( MappingType.PARTIAL_FUNCTION, child.getRowType().getFieldCount(), project.getRowType().getFieldCount()); for (Ord<RexNode> o : Ord.zip(project.getProjects())) { if (o.e instanceof RexInputRef) { int sIdx = ((RexInputRef) o.e).getIndex(); m.set(sIdx, o.i); columnsMappedBuilder.set(sIdx); } } // Go over childPullUpPredicates. If a predicate only contains columns in // 'columnsMapped' construct a new predicate based on mapping. final ImmutableBitSet columnsMapped = columnsMappedBuilder.build(); for (RexNode r : childInfo.pulledUpPredicates) { ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r); if (columnsMapped.contains(rCols)) { r = r.accept(new RexPermuteInputsShuttle(m, child)); projectPullUpPredicates.add(r); } } // Project can also generate constants. We need to include them. for (Ord<RexNode> expr : Ord.zip(project.getProjects())) { if (RexLiteral.isNullLiteral(expr.e)) { projectPullUpPredicates.add( rexBuilder.makeCall( SqlStdOperatorTable.IS_NULL, rexBuilder.makeInputRef(project, expr.i))); } else if (RexUtil.isConstant(expr.e)) { final List<RexNode> args = ImmutableList.of(rexBuilder.makeInputRef(project, expr.i), expr.e); final SqlOperator op = args.get(0).getType().isNullable() || args.get(1).getType().isNullable() ? SqlStdOperatorTable.IS_NOT_DISTINCT_FROM : SqlStdOperatorTable.EQUALS; projectPullUpPredicates.add(rexBuilder.makeCall(op, args)); } } return RelOptPredicateList.of(projectPullUpPredicates); }
public Double getDistinctRowCount(Aggregate rel, ImmutableBitSet groupKey, RexNode predicate) { if (predicate == null || predicate.isAlwaysTrue()) { if (groupKey.isEmpty()) { return 1D; } } // determine which predicates can be applied on the child of the // aggregate List<RexNode> notPushable = new ArrayList<RexNode>(); List<RexNode> pushable = new ArrayList<RexNode>(); RelOptUtil.splitFilters(rel.getGroupSet(), predicate, pushable, notPushable); final RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); RexNode childPreds = RexUtil.composeConjunction(rexBuilder, pushable, true); // set the bits as they correspond to the child input ImmutableBitSet.Builder childKey = ImmutableBitSet.builder(); RelMdUtil.setAggChildKeys(groupKey, rel, childKey); Double distinctRowCount = RelMetadataQuery.getDistinctRowCount(rel.getInput(), childKey.build(), childPreds); if (distinctRowCount == null) { return null; } else if (notPushable.isEmpty()) { return distinctRowCount; } else { RexNode preds = RexUtil.composeConjunction(rexBuilder, notPushable, true); return distinctRowCount * RelMdUtil.guessSelectivity(preds); } }
/** * Returns whether a condition is supported by {@link JdbcJoin}. * * <p>Corresponds to the capabilities of {@link SqlImplementor#convertConditionToSqlNode}. * * @param node Condition * @return Whether condition is supported */ private boolean canJoinOnCondition(RexNode node) { final List<RexNode> operands; switch (node.getKind()) { case AND: case OR: operands = ((RexCall) node).getOperands(); for (RexNode operand : operands) { if (!canJoinOnCondition(operand)) { return false; } } return true; case EQUALS: case IS_NOT_DISTINCT_FROM: case NOT_EQUALS: case GREATER_THAN: case GREATER_THAN_OR_EQUAL: case LESS_THAN: case LESS_THAN_OR_EQUAL: operands = ((RexCall) node).getOperands(); if ((operands.get(0) instanceof RexInputRef) && (operands.get(1) instanceof RexInputRef)) { return true; } // fall through default: return false; } }
@Override public String translate(ExprCompiler compiler, RexCall call) { String val = compiler.reserveName(); PrintWriter pw = compiler.pw; RexNode op = call.getOperands().get(0); String lhs = op.accept(compiler); boolean nullable = call.getType().isNullable(); pw.print(String.format("final %s %s;\n", compiler.javaTypeName(call), val)); if (!nullable) { pw.print(String.format("%1$s = !(%2$s);\n", val, lhs)); } else { String s = foldNullExpr(String.format("%1$s == null ? null : !(%1$s)", lhs), "null", op); pw.print(String.format("%1$s = %2$s;\n", val, s)); } return val; }
public Double getDistinctRowCount(Join rel, ImmutableBitSet groupKey, RexNode predicate) { if (predicate == null || predicate.isAlwaysTrue()) { if (groupKey.isEmpty()) { return 1D; } } return RelMdUtil.getJoinDistinctRowCount(rel, rel.getJoinType(), groupKey, predicate, false); }
private boolean isAlwaysTrue(RexNode predicate) { if (predicate instanceof RexCall) { RexCall c = (RexCall) predicate; if (c.getOperator().getKind() == SqlKind.EQUALS) { int lPos = pos(c.getOperands().get(0)); int rPos = pos(c.getOperands().get(1)); return lPos != -1 && lPos == rPos; } } return predicate.isAlwaysTrue(); }
public Double getDistinctRowCount(Project rel, ImmutableBitSet groupKey, RexNode predicate) { if (predicate == null || predicate.isAlwaysTrue()) { if (groupKey.isEmpty()) { return 1D; } } ImmutableBitSet.Builder baseCols = ImmutableBitSet.builder(); ImmutableBitSet.Builder projCols = ImmutableBitSet.builder(); List<RexNode> projExprs = rel.getProjects(); RelMdUtil.splitCols(projExprs, groupKey, baseCols, projCols); List<RexNode> notPushable = new ArrayList<RexNode>(); List<RexNode> pushable = new ArrayList<RexNode>(); RelOptUtil.splitFilters( ImmutableBitSet.range(rel.getRowType().getFieldCount()), predicate, pushable, notPushable); final RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); // get the distinct row count of the child input, passing in the // columns and filters that only reference the child; convert the // filter to reference the children projection expressions RexNode childPred = RexUtil.composeConjunction(rexBuilder, pushable, true); RexNode modifiedPred; if (childPred == null) { modifiedPred = null; } else { modifiedPred = RelOptUtil.pushPastProject(childPred, rel); } Double distinctRowCount = RelMetadataQuery.getDistinctRowCount(rel.getInput(), baseCols.build(), modifiedPred); if (distinctRowCount == null) { return null; } else if (!notPushable.isEmpty()) { RexNode preds = RexUtil.composeConjunction(rexBuilder, notPushable, true); distinctRowCount *= RelMdUtil.guessSelectivity(preds); } // No further computation required if the projection expressions // are all column references if (projCols.cardinality() == 0) { return distinctRowCount; } // multiply by the cardinality of the non-child projection expressions for (int bit : projCols.build()) { Double subRowCount = RelMdUtil.cardOfProjExpr(rel, projExprs.get(bit)); if (subRowCount == null) { return null; } distinctRowCount *= subRowCount; } return RelMdUtil.numDistinctVals(distinctRowCount, RelMetadataQuery.getRowCount(rel)); }
public Double getDistinctRowCount(Values rel, ImmutableBitSet groupKey, RexNode predicate) { if (predicate == null || predicate.isAlwaysTrue()) { if (groupKey.isEmpty()) { return 1D; } } Double selectivity = RelMdUtil.guessSelectivity(predicate); // assume half the rows are duplicates Double nRows = rel.getRows() / 2; return RelMdUtil.numDistinctVals(nRows, nRows * selectivity); }
public static boolean isDeterministicFuncOnLiterals(RexNode expr) { boolean deterministicFuncOnLiterals = true; RexVisitor<Void> visitor = new RexVisitorImpl<Void>(true) { @Override public Void visitCall(org.apache.calcite.rex.RexCall call) { if (!call.getOperator().isDeterministic()) { throw new Util.FoundOne(call); } return super.visitCall(call); } @Override public Void visitInputRef(RexInputRef inputRef) { throw new Util.FoundOne(inputRef); } @Override public Void visitLocalRef(RexLocalRef localRef) { throw new Util.FoundOne(localRef); } @Override public Void visitOver(RexOver over) { throw new Util.FoundOne(over); } @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { throw new Util.FoundOne(dynamicParam); } @Override public Void visitRangeRef(RexRangeRef rangeRef) { throw new Util.FoundOne(rangeRef); } @Override public Void visitFieldAccess(RexFieldAccess fieldAccess) { throw new Util.FoundOne(fieldAccess); } }; try { expr.accept(visitor); } catch (Util.FoundOne e) { deterministicFuncOnLiterals = false; } return deterministicFuncOnLiterals; }
public Double getDistinctRowCount(Filter rel, ImmutableBitSet groupKey, RexNode predicate) { if (predicate == null || predicate.isAlwaysTrue()) { if (groupKey.isEmpty()) { return 1D; } } // REVIEW zfong 4/18/06 - In the Broadbase code, duplicates are not // removed from the two filter lists. However, the code below is // doing so. RexNode unionPreds = RelMdUtil.unionPreds(rel.getCluster().getRexBuilder(), predicate, rel.getCondition()); return RelMetadataQuery.getDistinctRowCount(rel.getInput(), groupKey, unionPreds); }
public Double getDistinctRowCount(SemiJoin rel, ImmutableBitSet groupKey, RexNode predicate) { if (predicate == null || predicate.isAlwaysTrue()) { if (groupKey.isEmpty()) { return 1D; } } // create a RexNode representing the selectivity of the // semijoin filter and pass it to getDistinctRowCount RexNode newPred = RelMdUtil.makeSemiJoinSelectivityRexNode(rel); if (predicate != null) { RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); newPred = rexBuilder.makeCall(SqlStdOperatorTable.AND, newPred, predicate); } return RelMetadataQuery.getDistinctRowCount(rel.getLeft(), groupKey, newPred); }
// Catch-all rule when none of the others apply. public Double getDistinctRowCount(RelNode rel, ImmutableBitSet groupKey, RexNode predicate) { if (predicate == null || predicate.isAlwaysTrue()) { if (groupKey.isEmpty()) { return 1D; } } // REVIEW zfong 4/19/06 - Broadbase code does not take into // consideration selectivity of predicates passed in. Also, they // assume the rows are unique even if the table is not boolean uniq = RelMdUtil.areColumnsDefinitelyUnique(rel, groupKey); if (uniq) { return NumberUtil.multiply( RelMetadataQuery.getRowCount(rel), RelMetadataQuery.getSelectivity(rel, predicate)); } return null; }
private Void translateMatch2(RexNode node) { switch (node.getKind()) { case EQUALS: return translateBinary(null, null, (RexCall) node); case LESS_THAN: return translateBinary("$lt", "$gt", (RexCall) node); case LESS_THAN_OR_EQUAL: return translateBinary("$lte", "$gte", (RexCall) node); case NOT_EQUALS: return translateBinary("$ne", "$ne", (RexCall) node); case GREATER_THAN: return translateBinary("$gt", "$lt", (RexCall) node); case GREATER_THAN_OR_EQUAL: return translateBinary("$gte", "$lte", (RexCall) node); default: throw new AssertionError("cannot translate " + node); } }
public static boolean isDeterministic(RexNode expr) { boolean deterministic = true; RexVisitor<Void> visitor = new RexVisitorImpl<Void>(true) { @Override public Void visitCall(org.apache.calcite.rex.RexCall call) { if (!call.getOperator().isDeterministic()) { throw new Util.FoundOne(call); } return super.visitCall(call); } }; try { expr.accept(visitor); } catch (Util.FoundOne e) { deterministic = false; } return deterministic; }
public Double getDistinctRowCount(Union rel, ImmutableBitSet groupKey, RexNode predicate) { Double rowCount = 0.0; int[] adjustments = new int[rel.getRowType().getFieldCount()]; RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); for (RelNode input : rel.getInputs()) { // convert the predicate to reference the types of the union child RexNode modifiedPred; if (predicate == null) { modifiedPred = null; } else { modifiedPred = predicate.accept( new RelOptUtil.RexInputConverter( rexBuilder, null, input.getRowType().getFieldList(), adjustments)); } Double partialRowCount = RelMetadataQuery.getDistinctRowCount(input, groupKey, modifiedPred); if (partialRowCount == null) { return null; } rowCount += partialRowCount; } return rowCount; }
@Override public String translate(ExprCompiler compiler, RexCall call) { String val = compiler.reserveName(); PrintWriter pw = compiler.pw; pw.print(String.format("final %s %s;\n", compiler.javaTypeName(call), val)); RexNode op0 = call.getOperands().get(0); RexNode op1 = call.getOperands().get(1); boolean lhsNullable = op0.getType().isNullable(); boolean rhsNullable = op1.getType().isNullable(); String lhs = op0.accept(compiler); if (!lhsNullable) { pw.print(String.format("if (%2$s) { %1$s = true; }\n", val, lhs)); pw.print("else {\n"); String rhs = op1.accept(compiler); pw.print(String.format(" %1$s = %2$s;\n}\n", val, rhs)); } else { String foldedLHS = foldNullExpr(String.format("%1$s == null || !(%1$s)", lhs), "true", op0); pw.print(String.format("if (%s) {\n", foldedLHS)); String rhs = op1.accept(compiler); String s; if (rhsNullable) { s = foldNullExpr( String.format( "(%2$s != null && %2$s) ? Boolean.TRUE : ((%1$s == null || %2$s == null) ? null : Boolean.FALSE)", lhs, rhs), "null", op1); } else { s = String.format("%2$s ? Boolean.valueOf(%2$s) : %1$s", lhs, rhs); } pw.print(String.format(" %1$s = %2$s;\n", val, s)); pw.print(String.format("} else { %1$s = true; }\n", val)); } return val; }
public Double averageRexSize(RexNode node, List<Double> inputColumnSizes) { switch (node.getKind()) { case INPUT_REF: return inputColumnSizes.get(((RexInputRef) node).getIndex()); case LITERAL: return typeValueSize(node.getType(), ((RexLiteral) node).getValue()); default: if (node instanceof RexCall) { RexCall call = (RexCall) node; for (RexNode operand : call.getOperands()) { // It's a reasonable assumption that a function's result will have // similar size to its argument of a similar type. For example, // UPPER(c) has the same average size as c. if (operand.getType().getSqlTypeName() == node.getType().getSqlTypeName()) { return averageRexSize(operand, inputColumnSizes); } } } return averageTypeValueSize(node.getType()); } }
private void infer( RexNode predicates, Set<String> allExprsDigests, List<RexNode> inferedPredicates, boolean includeEqualityInference, ImmutableBitSet inferringFields) { for (RexNode r : RelOptUtil.conjunctions(predicates)) { if (!includeEqualityInference && equalityPredicates.contains(r.toString())) { continue; } for (Mapping m : mappings(r)) { RexNode tr = r.accept(new RexPermuteInputsShuttle(m, joinRel.getInput(0), joinRel.getInput(1))); if (inferringFields.contains(RelOptUtil.InputFinder.bits(tr)) && !allExprsDigests.contains(tr.toString()) && !isAlwaysTrue(tr)) { inferedPredicates.add(tr); allExprsDigests.add(tr.toString()); } } } }
@Test public void testSplitFilter() { final RexLiteral i1 = rexBuilder.makeExactLiteral(BigDecimal.ONE); final RexLiteral i2 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(2)); final RexLiteral i3 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(3)); final RelDataType intType = typeFactory.createType(int.class); final RexInputRef x = rexBuilder.makeInputRef(intType, 0); // $0 final RexInputRef y = rexBuilder.makeInputRef(intType, 1); // $1 final RexInputRef z = rexBuilder.makeInputRef(intType, 2); // $2 final RexNode x_eq_1 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1 final RexNode x_eq_1_b = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1 again final RexNode y_eq_2 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, y, i2); // $1 = 2 final RexNode z_eq_3 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, z, i3); // $2 = 3 RexNode newFilter; // Example 1. // TODO: // Example 2. // condition: x = 1, // target: x = 1 or z = 3 // yields // residue: not (z = 3) newFilter = SubstitutionVisitor.splitFilter( rexBuilder, x_eq_1, rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, z_eq_3)); assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))")); // 2b. // condition: x = 1 or y = 2 // target: x = 1 or y = 2 or z = 3 // yields // residue: not (z = 3) newFilter = SubstitutionVisitor.splitFilter( rexBuilder, rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2), rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3)); assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))")); // 2c. // condition: x = 1 // target: x = 1 or y = 2 or z = 3 // yields // residue: not (y = 2) and not (z = 3) newFilter = SubstitutionVisitor.splitFilter( rexBuilder, x_eq_1, rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3)); assertThat(newFilter.toString(), equalTo("AND(NOT(=($1, 2)), NOT(=($2, 3)))")); // 2d. // condition: x = 1 or y = 2 // target: y = 2 or x = 1 // yields // residue: true newFilter = SubstitutionVisitor.splitFilter( rexBuilder, rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2), rexBuilder.makeCall(SqlStdOperatorTable.OR, y_eq_2, x_eq_1)); assertThat(newFilter.isAlwaysTrue(), equalTo(true)); // 2e. // condition: x = 1 // target: x = 1 (different object) // yields // residue: true newFilter = SubstitutionVisitor.splitFilter(rexBuilder, x_eq_1, x_eq_1_b); assertThat(newFilter.isAlwaysTrue(), equalTo(true)); // 2f. // condition: x = 1 or y = 2 // target: x = 1 // yields // residue: null // TODO: // Example 3. // Condition [x = 1 and y = 2], // target [y = 2 and x = 1] yields // residue [true]. // TODO: // Example 4. // TODO: }
private void checkSatisfiable(RexNode e, String s) { assertTrue(SubstitutionVisitor.mayBeSatisfiable(e)); final RexNode simple = SubstitutionVisitor.simplify(rexBuilder, e); assertEquals(s, simple.toString()); }
/** * The PullUp Strategy is sound but not complete. * * <ol> * <li>We only pullUp inferred predicates for now. Pulling up existing predicates causes an * explosion of duplicates. The existing predicates are pushed back down as new * predicates. Once we have rules to eliminate duplicate Filter conditions, we should * pullUp all predicates. * <li>For Left Outer: we infer new predicates from the left and set them as applicable on the * Right side. No predicates are pulledUp. * <li>Right Outer Joins are handled in an analogous manner. * <li>For Full Outer Joins no predicates are pulledUp or inferred. * </ol> */ public RelOptPredicateList inferPredicates(boolean includeEqualityInference) { List<RexNode> inferredPredicates = new ArrayList<RexNode>(); Set<String> allExprsDigests = new HashSet<String>(this.allExprsDigests); final JoinRelType joinType = joinRel.getJoinType(); switch (joinType) { case INNER: case LEFT: infer( leftChildPredicates, allExprsDigests, inferredPredicates, includeEqualityInference, joinType == JoinRelType.LEFT ? rightFieldsBitSet : allFieldsBitSet); break; } switch (joinType) { case INNER: case RIGHT: infer( rightChildPredicates, allExprsDigests, inferredPredicates, includeEqualityInference, joinType == JoinRelType.RIGHT ? leftFieldsBitSet : allFieldsBitSet); break; } Mappings.TargetMapping rightMapping = Mappings.createShiftMapping( nSysFields + nFieldsLeft + nFieldsRight, 0, nSysFields + nFieldsLeft, nFieldsRight); final RexPermuteInputsShuttle rightPermute = new RexPermuteInputsShuttle(rightMapping, joinRel); Mappings.TargetMapping leftMapping = Mappings.createShiftMapping(nSysFields + nFieldsLeft, 0, nSysFields, nFieldsLeft); final RexPermuteInputsShuttle leftPermute = new RexPermuteInputsShuttle(leftMapping, joinRel); List<RexNode> leftInferredPredicates = new ArrayList<RexNode>(); List<RexNode> rightInferredPredicates = new ArrayList<RexNode>(); for (RexNode iP : inferredPredicates) { ImmutableBitSet iPBitSet = RelOptUtil.InputFinder.bits(iP); if (leftFieldsBitSet.contains(iPBitSet)) { leftInferredPredicates.add(iP.accept(leftPermute)); } else if (rightFieldsBitSet.contains(iPBitSet)) { rightInferredPredicates.add(iP.accept(rightPermute)); } } switch (joinType) { case INNER: Iterable<RexNode> pulledUpPredicates; if (isSemiJoin) { pulledUpPredicates = Iterables.concat( RelOptUtil.conjunctions(leftChildPredicates), leftInferredPredicates); } else { pulledUpPredicates = Iterables.concat( RelOptUtil.conjunctions(leftChildPredicates), RelOptUtil.conjunctions(rightChildPredicates), RelOptUtil.conjunctions(joinRel.getCondition()), inferredPredicates); } return RelOptPredicateList.of( pulledUpPredicates, leftInferredPredicates, rightInferredPredicates); case LEFT: return RelOptPredicateList.of( RelOptUtil.conjunctions(leftChildPredicates), leftInferredPredicates, rightInferredPredicates); case RIGHT: return RelOptPredicateList.of( RelOptUtil.conjunctions(rightChildPredicates), inferredPredicates, EMPTY_LIST); default: assert inferredPredicates.size() == 0; return RelOptPredicateList.EMPTY; } }
private JoinConditionBasedPredicateInference( Join joinRel, boolean isSemiJoin, RexNode lPreds, RexNode rPreds) { super(); this.joinRel = joinRel; this.isSemiJoin = isSemiJoin; nFieldsLeft = joinRel.getLeft().getRowType().getFieldList().size(); nFieldsRight = joinRel.getRight().getRowType().getFieldList().size(); nSysFields = joinRel.getSystemFieldList().size(); leftFieldsBitSet = ImmutableBitSet.range(nSysFields, nSysFields + nFieldsLeft); rightFieldsBitSet = ImmutableBitSet.range(nSysFields + nFieldsLeft, nSysFields + nFieldsLeft + nFieldsRight); allFieldsBitSet = ImmutableBitSet.range(0, nSysFields + nFieldsLeft + nFieldsRight); exprFields = Maps.newHashMap(); allExprsDigests = new HashSet<String>(); if (lPreds == null) { leftChildPredicates = null; } else { Mappings.TargetMapping leftMapping = Mappings.createShiftMapping(nSysFields + nFieldsLeft, nSysFields, 0, nFieldsLeft); leftChildPredicates = lPreds.accept(new RexPermuteInputsShuttle(leftMapping, joinRel.getInput(0))); for (RexNode r : RelOptUtil.conjunctions(leftChildPredicates)) { exprFields.put(r.toString(), RelOptUtil.InputFinder.bits(r)); allExprsDigests.add(r.toString()); } } if (rPreds == null) { rightChildPredicates = null; } else { Mappings.TargetMapping rightMapping = Mappings.createShiftMapping( nSysFields + nFieldsLeft + nFieldsRight, nSysFields + nFieldsLeft, 0, nFieldsRight); rightChildPredicates = rPreds.accept(new RexPermuteInputsShuttle(rightMapping, joinRel.getInput(1))); for (RexNode r : RelOptUtil.conjunctions(rightChildPredicates)) { exprFields.put(r.toString(), RelOptUtil.InputFinder.bits(r)); allExprsDigests.add(r.toString()); } } equivalence = Maps.newTreeMap(); equalityPredicates = new HashSet<String>(); for (int i = 0; i < nSysFields + nFieldsLeft + nFieldsRight; i++) { equivalence.put(i, BitSets.of(i)); } // Only process equivalences found in the join conditions. Processing // Equivalences from the left or right side infer predicates that are // already present in the Tree below the join. RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder(); List<RexNode> exprs = RelOptUtil.conjunctions(compose(rexBuilder, ImmutableList.of(joinRel.getCondition()))); final EquivalenceFinder eF = new EquivalenceFinder(); new ArrayList<Void>( Lists.transform( exprs, new Function<RexNode, Void>() { public Void apply(RexNode input) { return input.accept(eF); } })); equivalence = BitSets.closure(equivalence); }
private static void splitJoinCondition( List<RelDataTypeField> sysFieldList, List<RelNode> inputs, RexNode condition, List<List<RexNode>> joinKeys, List<Integer> filterNulls, List<SqlOperator> rangeOp, List<RexNode> nonEquiList) throws CalciteSemanticException { final int sysFieldCount = sysFieldList.size(); final RelOptCluster cluster = inputs.get(0).getCluster(); final RexBuilder rexBuilder = cluster.getRexBuilder(); if (condition instanceof RexCall) { RexCall call = (RexCall) condition; if (call.getOperator() == SqlStdOperatorTable.AND) { for (RexNode operand : call.getOperands()) { splitJoinCondition( sysFieldList, inputs, operand, joinKeys, filterNulls, rangeOp, nonEquiList); } return; } RexNode leftKey = null; RexNode rightKey = null; int leftInput = 0; int rightInput = 0; List<RelDataTypeField> leftFields = null; List<RelDataTypeField> rightFields = null; boolean reverse = false; SqlKind kind = call.getKind(); // Only consider range operators if we haven't already seen one if ((kind == SqlKind.EQUALS) || (filterNulls != null && kind == SqlKind.IS_NOT_DISTINCT_FROM) || (rangeOp != null && rangeOp.isEmpty() && (kind == SqlKind.GREATER_THAN || kind == SqlKind.GREATER_THAN_OR_EQUAL || kind == SqlKind.LESS_THAN || kind == SqlKind.LESS_THAN_OR_EQUAL))) { final List<RexNode> operands = call.getOperands(); RexNode op0 = operands.get(0); RexNode op1 = operands.get(1); final ImmutableBitSet projRefs0 = InputFinder.bits(op0); final ImmutableBitSet projRefs1 = InputFinder.bits(op1); final ImmutableBitSet[] inputsRange = new ImmutableBitSet[inputs.size()]; int totalFieldCount = 0; for (int i = 0; i < inputs.size(); i++) { final int firstField = totalFieldCount + sysFieldCount; totalFieldCount = firstField + inputs.get(i).getRowType().getFieldCount(); inputsRange[i] = ImmutableBitSet.range(firstField, totalFieldCount); } boolean foundBothInputs = false; for (int i = 0; i < inputs.size() && !foundBothInputs; i++) { if (projRefs0.intersects(inputsRange[i]) && projRefs0.union(inputsRange[i]).equals(inputsRange[i])) { if (leftKey == null) { leftKey = op0; leftInput = i; leftFields = inputs.get(leftInput).getRowType().getFieldList(); } else { rightKey = op0; rightInput = i; rightFields = inputs.get(rightInput).getRowType().getFieldList(); reverse = true; foundBothInputs = true; } } else if (projRefs1.intersects(inputsRange[i]) && projRefs1.union(inputsRange[i]).equals(inputsRange[i])) { if (leftKey == null) { leftKey = op1; leftInput = i; leftFields = inputs.get(leftInput).getRowType().getFieldList(); } else { rightKey = op1; rightInput = i; rightFields = inputs.get(rightInput).getRowType().getFieldList(); foundBothInputs = true; } } } if ((leftKey != null) && (rightKey != null)) { // adjustment array int[] adjustments = new int[totalFieldCount]; for (int i = 0; i < inputs.size(); i++) { final int adjustment = inputsRange[i].nextSetBit(0); for (int j = adjustment; j < inputsRange[i].length(); j++) { adjustments[j] = -adjustment; } } // replace right Key input ref rightKey = rightKey.accept( new RelOptUtil.RexInputConverter( rexBuilder, rightFields, rightFields, adjustments)); // left key only needs to be adjusted if there are system // fields, but do it for uniformity leftKey = leftKey.accept( new RelOptUtil.RexInputConverter( rexBuilder, leftFields, leftFields, adjustments)); RelDataType leftKeyType = leftKey.getType(); RelDataType rightKeyType = rightKey.getType(); if (leftKeyType != rightKeyType) { // perform casting using Hive rules TypeInfo rType = TypeConverter.convert(rightKeyType); TypeInfo lType = TypeConverter.convert(leftKeyType); TypeInfo tgtType = FunctionRegistry.getCommonClassForComparison(lType, rType); if (tgtType == null) { throw new CalciteSemanticException( "Cannot find common type for join keys " + leftKey + " (type " + leftKeyType + ") and " + rightKey + " (type " + rightKeyType + ")"); } RelDataType targetKeyType = TypeConverter.convert(tgtType, rexBuilder.getTypeFactory()); if (leftKeyType != targetKeyType && TypeInfoUtils.isConversionRequiredForComparison(tgtType, lType)) { leftKey = rexBuilder.makeCast(targetKeyType, leftKey); } if (rightKeyType != targetKeyType && TypeInfoUtils.isConversionRequiredForComparison(tgtType, rType)) { rightKey = rexBuilder.makeCast(targetKeyType, rightKey); } } } } if ((leftKey != null) && (rightKey != null)) { // found suitable join keys // add them to key list, ensuring that if there is a // non-equi join predicate, it appears at the end of the // key list; also mark the null filtering property addJoinKey(joinKeys.get(leftInput), leftKey, (rangeOp != null) && !rangeOp.isEmpty()); addJoinKey(joinKeys.get(rightInput), rightKey, (rangeOp != null) && !rangeOp.isEmpty()); if (filterNulls != null && kind == SqlKind.EQUALS) { // nulls are considered not matching for equality comparison // add the position of the most recently inserted key filterNulls.add(joinKeys.get(leftInput).size() - 1); } if (rangeOp != null && kind != SqlKind.EQUALS && kind != SqlKind.IS_DISTINCT_FROM) { if (reverse) { kind = reverse(kind); } rangeOp.add(op(kind, call.getOperator())); } return; } // else fall through and add this condition as nonEqui condition } // The operator is not of RexCall type // So we fail. Fall through. // Add this condition to the list of non-equi-join conditions. nonEquiList.add(condition); }