/**
* Infers predicates for a project.
*
* <ol>
* <li>create a mapping from input to projection. Map only positions that directly reference an
* input column.
* <li>Expressions that only contain above columns are retained in the Project's pullExpressions
* list.
* <li>For e.g. expression 'a + e = 9' below will not be pulled up because 'e' is not in the
* projection list.
* <pre>
* childPullUpExprs: {a > 7, b + c < 10, a + e = 9}
* projectionExprs: {a, b, c, e / 2}
* projectionPullupExprs: {a > 7, b + c < 10}
* </pre>
* </ol>
*/
public RelOptPredicateList getPredicates(Project project) {
RelNode child = project.getInput();
final RexBuilder rexBuilder = project.getCluster().getRexBuilder();
RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child);
List<RexNode> projectPullUpPredicates = new ArrayList<RexNode>();
ImmutableBitSet.Builder columnsMappedBuilder = ImmutableBitSet.builder();
Mapping m =
Mappings.create(
MappingType.PARTIAL_FUNCTION,
child.getRowType().getFieldCount(),
project.getRowType().getFieldCount());
for (Ord<RexNode> o : Ord.zip(project.getProjects())) {
if (o.e instanceof RexInputRef) {
int sIdx = ((RexInputRef) o.e).getIndex();
m.set(sIdx, o.i);
columnsMappedBuilder.set(sIdx);
}
}
// Go over childPullUpPredicates. If a predicate only contains columns in
// 'columnsMapped' construct a new predicate based on mapping.
final ImmutableBitSet columnsMapped = columnsMappedBuilder.build();
for (RexNode r : childInfo.pulledUpPredicates) {
ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r);
if (columnsMapped.contains(rCols)) {
r = r.accept(new RexPermuteInputsShuttle(m, child));
projectPullUpPredicates.add(r);
}
}
// Project can also generate constants. We need to include them.
for (Ord<RexNode> expr : Ord.zip(project.getProjects())) {
if (RexLiteral.isNullLiteral(expr.e)) {
projectPullUpPredicates.add(
rexBuilder.makeCall(
SqlStdOperatorTable.IS_NULL, rexBuilder.makeInputRef(project, expr.i)));
} else if (RexUtil.isConstant(expr.e)) {
final List<RexNode> args =
ImmutableList.of(rexBuilder.makeInputRef(project, expr.i), expr.e);
final SqlOperator op =
args.get(0).getType().isNullable() || args.get(1).getType().isNullable()
? SqlStdOperatorTable.IS_NOT_DISTINCT_FROM
: SqlStdOperatorTable.EQUALS;
projectPullUpPredicates.add(rexBuilder.makeCall(op, args));
}
}
return RelOptPredicateList.of(projectPullUpPredicates);
}
protected DrillRel addRenamedProject(DrillRel rel, RelDataType validatedRowType) {
RelDataType t = rel.getRowType();
RexBuilder b = rel.getCluster().getRexBuilder();
List<RexNode> projections = Lists.newArrayList();
int projectCount = t.getFieldList().size();
for (int i = 0; i < projectCount; i++) {
projections.add(b.makeInputRef(rel, i));
}
final List<String> fieldNames2 =
SqlValidatorUtil.uniquify(validatedRowType.getFieldNames(), SqlValidatorUtil.F_SUGGESTER2);
RelDataType newRowType =
RexUtil.createStructType(rel.getCluster().getTypeFactory(), projections, fieldNames2);
DrillProjectRel topProj =
DrillProjectRel.create(rel.getCluster(), rel.getTraitSet(), rel, projections, newRowType);
// Add a final non-trivial Project to get the validatedRowType, if child is not project.
if (rel instanceof Project && DrillRelOptUtil.isTrivialProject(topProj, true)) {
return rel;
} else {
return topProj;
}
}
@Test
public void testSplitFilter() {
final RexLiteral i1 = rexBuilder.makeExactLiteral(BigDecimal.ONE);
final RexLiteral i2 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(2));
final RexLiteral i3 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(3));
final RelDataType intType = typeFactory.createType(int.class);
final RexInputRef x = rexBuilder.makeInputRef(intType, 0); // $0
final RexInputRef y = rexBuilder.makeInputRef(intType, 1); // $1
final RexInputRef z = rexBuilder.makeInputRef(intType, 2); // $2
final RexNode x_eq_1 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1
final RexNode x_eq_1_b = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1 again
final RexNode y_eq_2 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, y, i2); // $1 = 2
final RexNode z_eq_3 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, z, i3); // $2 = 3
RexNode newFilter;
// Example 1.
// TODO:
// Example 2.
// condition: x = 1,
// target: x = 1 or z = 3
// yields
// residue: not (z = 3)
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder, x_eq_1, rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, z_eq_3));
assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))"));
// 2b.
// condition: x = 1 or y = 2
// target: x = 1 or y = 2 or z = 3
// yields
// residue: not (z = 3)
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder,
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2),
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3));
assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))"));
// 2c.
// condition: x = 1
// target: x = 1 or y = 2 or z = 3
// yields
// residue: not (y = 2) and not (z = 3)
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder,
x_eq_1,
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3));
assertThat(newFilter.toString(), equalTo("AND(NOT(=($1, 2)), NOT(=($2, 3)))"));
// 2d.
// condition: x = 1 or y = 2
// target: y = 2 or x = 1
// yields
// residue: true
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder,
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2),
rexBuilder.makeCall(SqlStdOperatorTable.OR, y_eq_2, x_eq_1));
assertThat(newFilter.isAlwaysTrue(), equalTo(true));
// 2e.
// condition: x = 1
// target: x = 1 (different object)
// yields
// residue: true
newFilter = SubstitutionVisitor.splitFilter(rexBuilder, x_eq_1, x_eq_1_b);
assertThat(newFilter.isAlwaysTrue(), equalTo(true));
// 2f.
// condition: x = 1 or y = 2
// target: x = 1
// yields
// residue: null
// TODO:
// Example 3.
// Condition [x = 1 and y = 2],
// target [y = 2 and x = 1] yields
// residue [true].
// TODO:
// Example 4.
// TODO:
}
/**
* Unit test for logic functions {@link
* org.apache.calcite.plan.SubstitutionVisitor#mayBeSatisfiable} and {@link
* org.apache.calcite.plan.SubstitutionVisitor#simplify}.
*/
@Test
public void testSatisfiable() {
// TRUE may be satisfiable
checkSatisfiable(rexBuilder.makeLiteral(true), "true");
// FALSE is not satisfiable
checkNotSatisfiable(rexBuilder.makeLiteral(false));
// The expression "$0 = 1".
final RexNode i0_eq_0 =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(typeFactory.createType(int.class), 0),
rexBuilder.makeExactLiteral(BigDecimal.ZERO));
// "$0 = 1" may be satisfiable
checkSatisfiable(i0_eq_0, "=($0, 0)");
// "$0 = 1 AND TRUE" may be satisfiable
final RexNode e0 =
rexBuilder.makeCall(SqlStdOperatorTable.AND, i0_eq_0, rexBuilder.makeLiteral(true));
checkSatisfiable(e0, "=($0, 0)");
// "$0 = 1 AND FALSE" is not satisfiable
final RexNode e1 =
rexBuilder.makeCall(SqlStdOperatorTable.AND, i0_eq_0, rexBuilder.makeLiteral(false));
checkNotSatisfiable(e1);
// "$0 = 0 AND NOT $0 = 0" is not satisfiable
final RexNode e2 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0));
checkNotSatisfiable(e2);
// "TRUE AND NOT $0 = 0" may be satisfiable. Can simplify.
final RexNode e3 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
rexBuilder.makeLiteral(true),
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0));
checkSatisfiable(e3, "NOT(=($0, 0))");
// The expression "$1 = 1".
final RexNode i1_eq_1 =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(typeFactory.createType(int.class), 1),
rexBuilder.makeExactLiteral(BigDecimal.ONE));
// "$0 = 0 AND $1 = 1 AND NOT $0 = 0" is not satisfiable
final RexNode e4 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i1_eq_1,
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0)));
checkNotSatisfiable(e4);
// "$0 = 0 AND NOT $1 = 1" may be satisfiable. Can't simplify.
final RexNode e5 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i1_eq_1));
checkSatisfiable(e5, "AND(=($0, 0), NOT(=($1, 1)))");
// "$0 = 0 AND NOT ($0 = 0 AND $1 = 1)" may be satisfiable. Can simplify.
final RexNode e6 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(
SqlStdOperatorTable.NOT,
rexBuilder.makeCall(SqlStdOperatorTable.AND, i0_eq_0, i1_eq_1)));
checkSatisfiable(e6, "AND(=($0, 0), NOT(AND(=($0, 0), =($1, 1))))");
// "$0 = 0 AND ($1 = 1 AND NOT ($0 = 0))" is not satisfiable.
final RexNode e7 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i1_eq_1,
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0)));
checkNotSatisfiable(e7);
// The expression "$2".
final RexInputRef i2 = rexBuilder.makeInputRef(typeFactory.createType(boolean.class), 2);
// The expression "$3".
final RexInputRef i3 = rexBuilder.makeInputRef(typeFactory.createType(boolean.class), 3);
// The expression "$4".
final RexInputRef i4 = rexBuilder.makeInputRef(typeFactory.createType(boolean.class), 4);
// "$0 = 0 AND $2 AND $3 AND NOT ($2 AND $3 AND $4) AND NOT ($2 AND $4)" may
// be satisfiable. Can't simplify.
final RexNode e8 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i2,
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i3,
rexBuilder.makeCall(
SqlStdOperatorTable.NOT,
rexBuilder.makeCall(SqlStdOperatorTable.AND, i2, i3, i4)),
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i4))));
checkSatisfiable(e8, "AND(=($0, 0), $2, $3, NOT(AND($2, $3, $4)), NOT($4))");
}
/**
* Push any equi join conditions that are not column references as Projections on top of the
* children.
*
* @param factory Project factory to use.
* @param inputRels inputs to a join
* @param leftJoinKeys expressions for LHS of join key
* @param rightJoinKeys expressions for RHS of join key
* @param systemColCount number of system columns, usually zero. These columns are projected at
* the leading edge of the output row.
* @param leftKeys on return this contains the join key positions from the new project rel on the
* LHS.
* @param rightKeys on return this contains the join key positions from the new project rel on the
* RHS.
* @return the join condition after the equi expressions pushed down.
*/
public static RexNode projectNonColumnEquiConditions(
ProjectFactory factory,
RelNode[] inputRels,
List<RexNode> leftJoinKeys,
List<RexNode> rightJoinKeys,
int systemColCount,
List<Integer> leftKeys,
List<Integer> rightKeys) {
RelNode leftRel = inputRels[0];
RelNode rightRel = inputRels[1];
RexBuilder rexBuilder = leftRel.getCluster().getRexBuilder();
RexNode outJoinCond = null;
int origLeftInputSize = leftRel.getRowType().getFieldCount();
int origRightInputSize = rightRel.getRowType().getFieldCount();
List<RexNode> newLeftFields = new ArrayList<RexNode>();
List<String> newLeftFieldNames = new ArrayList<String>();
List<RexNode> newRightFields = new ArrayList<RexNode>();
List<String> newRightFieldNames = new ArrayList<String>();
int leftKeyCount = leftJoinKeys.size();
int i;
for (i = 0; i < origLeftInputSize; i++) {
final RelDataTypeField field = leftRel.getRowType().getFieldList().get(i);
newLeftFields.add(rexBuilder.makeInputRef(field.getType(), i));
newLeftFieldNames.add(field.getName());
}
for (i = 0; i < origRightInputSize; i++) {
final RelDataTypeField field = rightRel.getRowType().getFieldList().get(i);
newRightFields.add(rexBuilder.makeInputRef(field.getType(), i));
newRightFieldNames.add(field.getName());
}
int newKeyCount = 0;
List<Pair<Integer, Integer>> origColEqConds = new ArrayList<Pair<Integer, Integer>>();
for (i = 0; i < leftKeyCount; i++) {
RexNode leftKey = leftJoinKeys.get(i);
RexNode rightKey = rightJoinKeys.get(i);
if (leftKey instanceof RexInputRef && rightKey instanceof RexInputRef) {
origColEqConds.add(
Pair.of(((RexInputRef) leftKey).getIndex(), ((RexInputRef) rightKey).getIndex()));
} else {
newLeftFields.add(leftKey);
newLeftFieldNames.add(null);
newRightFields.add(rightKey);
newRightFieldNames.add(null);
newKeyCount++;
}
}
for (i = 0; i < origColEqConds.size(); i++) {
Pair<Integer, Integer> p = origColEqConds.get(i);
RexNode leftKey = leftJoinKeys.get(i);
RexNode rightKey = rightJoinKeys.get(i);
leftKeys.add(p.left);
rightKeys.add(p.right);
RexNode cond =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(leftKey.getType(), systemColCount + p.left),
rexBuilder.makeInputRef(
rightKey.getType(), systemColCount + origLeftInputSize + newKeyCount + p.right));
if (outJoinCond == null) {
outJoinCond = cond;
} else {
outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
}
}
if (newKeyCount == 0) {
return outJoinCond;
}
int newLeftOffset = systemColCount + origLeftInputSize;
int newRightOffset = systemColCount + origLeftInputSize + origRightInputSize + newKeyCount;
for (i = 0; i < newKeyCount; i++) {
leftKeys.add(origLeftInputSize + i);
rightKeys.add(origRightInputSize + i);
RexNode cond =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(
newLeftFields.get(origLeftInputSize + i).getType(), newLeftOffset + i),
rexBuilder.makeInputRef(
newRightFields.get(origRightInputSize + i).getType(), newRightOffset + i));
if (outJoinCond == null) {
outJoinCond = cond;
} else {
outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
}
}
// added project if need to produce new keys than the original input
// fields
if (newKeyCount > 0) {
leftRel =
factory.createProject(
leftRel, newLeftFields, SqlValidatorUtil.uniquify(newLeftFieldNames));
rightRel =
factory.createProject(
rightRel, newRightFields, SqlValidatorUtil.uniquify(newRightFieldNames));
}
inputRels[0] = leftRel;
inputRels[1] = rightRel;
return outJoinCond;
}
@Override
public Prel visitProject(ProjectPrel project, Object unused) throws RelConversionException {
// Apply the rule to the child
RelNode originalInput = ((Prel) project.getInput(0)).accept(this, null);
project = (ProjectPrel) project.copy(project.getTraitSet(), Lists.newArrayList(originalInput));
List<RexNode> exprList = new ArrayList<>();
List<RelDataTypeField> relDataTypes = new ArrayList();
List<RelDataTypeField> origRelDataTypes = new ArrayList();
int i = 0;
final int lastColumnReferenced = PrelUtil.getLastUsedColumnReference(project.getProjects());
if (lastColumnReferenced == -1) {
return project;
}
final int lastRexInput = lastColumnReferenced + 1;
RexVisitorComplexExprSplitter exprSplitter =
new RexVisitorComplexExprSplitter(factory, funcReg, lastRexInput);
for (RexNode rex : project.getChildExps()) {
origRelDataTypes.add(project.getRowType().getFieldList().get(i));
i++;
exprList.add(rex.accept(exprSplitter));
}
List<RexNode> complexExprs = exprSplitter.getComplexExprs();
if (complexExprs.size() == 1 && findTopComplexFunc(project.getChildExps()).size() == 1) {
return project;
}
ProjectPrel childProject;
List<RexNode> allExprs = new ArrayList();
int exprIndex = 0;
List<String> fieldNames = originalInput.getRowType().getFieldNames();
for (int index = 0; index < lastRexInput; index++) {
RexBuilder builder = new RexBuilder(factory);
allExprs.add(
builder.makeInputRef(new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
if (fieldNames.get(index).contains(StarColumnHelper.STAR_COLUMN)) {
relDataTypes.add(
new RelDataTypeFieldImpl(
fieldNames.get(index), allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
} else {
relDataTypes.add(
new RelDataTypeFieldImpl(
"EXPR$" + exprIndex, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
exprIndex++;
}
}
RexNode currRexNode;
int index = lastRexInput - 1;
// if the projection expressions contained complex outputs, split them into their own individual
// projects
if (complexExprs.size() > 0) {
while (complexExprs.size() > 0) {
if (index >= lastRexInput) {
allExprs.remove(allExprs.size() - 1);
RexBuilder builder = new RexBuilder(factory);
allExprs.add(
builder.makeInputRef(
new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
}
index++;
exprIndex++;
currRexNode = complexExprs.remove(0);
allExprs.add(currRexNode);
relDataTypes.add(
new RelDataTypeFieldImpl(
"EXPR$" + exprIndex, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
childProject =
new ProjectPrel(
project.getCluster(),
project.getTraitSet(),
originalInput,
ImmutableList.copyOf(allExprs),
new RelRecordType(relDataTypes));
originalInput = childProject;
}
// copied from above, find a better way to do this
allExprs.remove(allExprs.size() - 1);
RexBuilder builder = new RexBuilder(factory);
allExprs.add(
builder.makeInputRef(new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
relDataTypes.add(
new RelDataTypeFieldImpl(
"EXPR$" + index, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
}
return (Prel)
project.copy(
project.getTraitSet(), originalInput, exprList, new RelRecordType(origRelDataTypes));
}