protected DrillRel addRenamedProject(DrillRel rel, RelDataType validatedRowType) {
RelDataType t = rel.getRowType();
RexBuilder b = rel.getCluster().getRexBuilder();
List<RexNode> projections = Lists.newArrayList();
int projectCount = t.getFieldList().size();
for (int i = 0; i < projectCount; i++) {
projections.add(b.makeInputRef(rel, i));
}
final List<String> fieldNames2 =
SqlValidatorUtil.uniquify(validatedRowType.getFieldNames(), SqlValidatorUtil.F_SUGGESTER2);
RelDataType newRowType =
RexUtil.createStructType(rel.getCluster().getTypeFactory(), projections, fieldNames2);
DrillProjectRel topProj =
DrillProjectRel.create(rel.getCluster(), rel.getTraitSet(), rel, projections, newRowType);
// Add a final non-trivial Project to get the validatedRowType, if child is not project.
if (rel instanceof Project && DrillRelOptUtil.isTrivialProject(topProj, true)) {
return rel;
} else {
return topProj;
}
}
/**
* Infers predicates for a project.
*
* <ol>
* <li>create a mapping from input to projection. Map only positions that directly reference an
* input column.
* <li>Expressions that only contain above columns are retained in the Project's pullExpressions
* list.
* <li>For e.g. expression 'a + e = 9' below will not be pulled up because 'e' is not in the
* projection list.
* <pre>
* childPullUpExprs: {a > 7, b + c < 10, a + e = 9}
* projectionExprs: {a, b, c, e / 2}
* projectionPullupExprs: {a > 7, b + c < 10}
* </pre>
* </ol>
*/
public RelOptPredicateList getPredicates(Project project) {
RelNode child = project.getInput();
final RexBuilder rexBuilder = project.getCluster().getRexBuilder();
RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child);
List<RexNode> projectPullUpPredicates = new ArrayList<RexNode>();
ImmutableBitSet.Builder columnsMappedBuilder = ImmutableBitSet.builder();
Mapping m =
Mappings.create(
MappingType.PARTIAL_FUNCTION,
child.getRowType().getFieldCount(),
project.getRowType().getFieldCount());
for (Ord<RexNode> o : Ord.zip(project.getProjects())) {
if (o.e instanceof RexInputRef) {
int sIdx = ((RexInputRef) o.e).getIndex();
m.set(sIdx, o.i);
columnsMappedBuilder.set(sIdx);
}
}
// Go over childPullUpPredicates. If a predicate only contains columns in
// 'columnsMapped' construct a new predicate based on mapping.
final ImmutableBitSet columnsMapped = columnsMappedBuilder.build();
for (RexNode r : childInfo.pulledUpPredicates) {
ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r);
if (columnsMapped.contains(rCols)) {
r = r.accept(new RexPermuteInputsShuttle(m, child));
projectPullUpPredicates.add(r);
}
}
// Project can also generate constants. We need to include them.
for (Ord<RexNode> expr : Ord.zip(project.getProjects())) {
if (RexLiteral.isNullLiteral(expr.e)) {
projectPullUpPredicates.add(
rexBuilder.makeCall(
SqlStdOperatorTable.IS_NULL, rexBuilder.makeInputRef(project, expr.i)));
} else if (RexUtil.isConstant(expr.e)) {
final List<RexNode> args =
ImmutableList.of(rexBuilder.makeInputRef(project, expr.i), expr.e);
final SqlOperator op =
args.get(0).getType().isNullable() || args.get(1).getType().isNullable()
? SqlStdOperatorTable.IS_NOT_DISTINCT_FROM
: SqlStdOperatorTable.EQUALS;
projectPullUpPredicates.add(rexBuilder.makeCall(op, args));
}
}
return RelOptPredicateList.of(projectPullUpPredicates);
}
public static RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) {
RexNode typeSafeRex = rex;
if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) {
RexBuilder rb = cluster.getRexBuilder();
List<RexNode> fixedPredElems = new ArrayList<RexNode>();
RelDataType commonType =
cluster.getTypeFactory().leastRestrictive(RexUtil.types(((RexCall) rex).getOperands()));
for (RexNode rn : ((RexCall) rex).getOperands()) {
fixedPredElems.add(rb.ensureType(commonType, rn, true));
}
typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems);
}
return typeSafeRex;
}
public Double getDistinctRowCount(SemiJoin rel, ImmutableBitSet groupKey, RexNode predicate) {
if (predicate == null || predicate.isAlwaysTrue()) {
if (groupKey.isEmpty()) {
return 1D;
}
}
// create a RexNode representing the selectivity of the
// semijoin filter and pass it to getDistinctRowCount
RexNode newPred = RelMdUtil.makeSemiJoinSelectivityRexNode(rel);
if (predicate != null) {
RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
newPred = rexBuilder.makeCall(SqlStdOperatorTable.AND, newPred, predicate);
}
return RelMetadataQuery.getDistinctRowCount(rel.getLeft(), groupKey, newPred);
}
/** Creates a cluster. */
RelOptCluster(
RelOptQuery query,
RelOptPlanner planner,
RelDataTypeFactory typeFactory,
RexBuilder rexBuilder) {
assert planner != null;
assert typeFactory != null;
this.query = query;
this.planner = planner;
this.typeFactory = typeFactory;
this.rexBuilder = rexBuilder;
this.originalExpression = rexBuilder.makeLiteral("?");
// set up a default rel metadata provider,
// giving the planner first crack at everything
setMetadataProvider(new DefaultRelMetadataProvider());
this.emptyTraitSet = planner.emptyTraitSet();
}
@Test
public void testSplitFilter() {
final RexLiteral i1 = rexBuilder.makeExactLiteral(BigDecimal.ONE);
final RexLiteral i2 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(2));
final RexLiteral i3 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(3));
final RelDataType intType = typeFactory.createType(int.class);
final RexInputRef x = rexBuilder.makeInputRef(intType, 0); // $0
final RexInputRef y = rexBuilder.makeInputRef(intType, 1); // $1
final RexInputRef z = rexBuilder.makeInputRef(intType, 2); // $2
final RexNode x_eq_1 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1
final RexNode x_eq_1_b = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1 again
final RexNode y_eq_2 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, y, i2); // $1 = 2
final RexNode z_eq_3 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, z, i3); // $2 = 3
RexNode newFilter;
// Example 1.
// TODO:
// Example 2.
// condition: x = 1,
// target: x = 1 or z = 3
// yields
// residue: not (z = 3)
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder, x_eq_1, rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, z_eq_3));
assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))"));
// 2b.
// condition: x = 1 or y = 2
// target: x = 1 or y = 2 or z = 3
// yields
// residue: not (z = 3)
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder,
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2),
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3));
assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))"));
// 2c.
// condition: x = 1
// target: x = 1 or y = 2 or z = 3
// yields
// residue: not (y = 2) and not (z = 3)
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder,
x_eq_1,
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3));
assertThat(newFilter.toString(), equalTo("AND(NOT(=($1, 2)), NOT(=($2, 3)))"));
// 2d.
// condition: x = 1 or y = 2
// target: y = 2 or x = 1
// yields
// residue: true
newFilter =
SubstitutionVisitor.splitFilter(
rexBuilder,
rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2),
rexBuilder.makeCall(SqlStdOperatorTable.OR, y_eq_2, x_eq_1));
assertThat(newFilter.isAlwaysTrue(), equalTo(true));
// 2e.
// condition: x = 1
// target: x = 1 (different object)
// yields
// residue: true
newFilter = SubstitutionVisitor.splitFilter(rexBuilder, x_eq_1, x_eq_1_b);
assertThat(newFilter.isAlwaysTrue(), equalTo(true));
// 2f.
// condition: x = 1 or y = 2
// target: x = 1
// yields
// residue: null
// TODO:
// Example 3.
// Condition [x = 1 and y = 2],
// target [y = 2 and x = 1] yields
// residue [true].
// TODO:
// Example 4.
// TODO:
}
/**
* Unit test for logic functions {@link
* org.apache.calcite.plan.SubstitutionVisitor#mayBeSatisfiable} and {@link
* org.apache.calcite.plan.SubstitutionVisitor#simplify}.
*/
@Test
public void testSatisfiable() {
// TRUE may be satisfiable
checkSatisfiable(rexBuilder.makeLiteral(true), "true");
// FALSE is not satisfiable
checkNotSatisfiable(rexBuilder.makeLiteral(false));
// The expression "$0 = 1".
final RexNode i0_eq_0 =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(typeFactory.createType(int.class), 0),
rexBuilder.makeExactLiteral(BigDecimal.ZERO));
// "$0 = 1" may be satisfiable
checkSatisfiable(i0_eq_0, "=($0, 0)");
// "$0 = 1 AND TRUE" may be satisfiable
final RexNode e0 =
rexBuilder.makeCall(SqlStdOperatorTable.AND, i0_eq_0, rexBuilder.makeLiteral(true));
checkSatisfiable(e0, "=($0, 0)");
// "$0 = 1 AND FALSE" is not satisfiable
final RexNode e1 =
rexBuilder.makeCall(SqlStdOperatorTable.AND, i0_eq_0, rexBuilder.makeLiteral(false));
checkNotSatisfiable(e1);
// "$0 = 0 AND NOT $0 = 0" is not satisfiable
final RexNode e2 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0));
checkNotSatisfiable(e2);
// "TRUE AND NOT $0 = 0" may be satisfiable. Can simplify.
final RexNode e3 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
rexBuilder.makeLiteral(true),
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0));
checkSatisfiable(e3, "NOT(=($0, 0))");
// The expression "$1 = 1".
final RexNode i1_eq_1 =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(typeFactory.createType(int.class), 1),
rexBuilder.makeExactLiteral(BigDecimal.ONE));
// "$0 = 0 AND $1 = 1 AND NOT $0 = 0" is not satisfiable
final RexNode e4 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i1_eq_1,
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0)));
checkNotSatisfiable(e4);
// "$0 = 0 AND NOT $1 = 1" may be satisfiable. Can't simplify.
final RexNode e5 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i1_eq_1));
checkSatisfiable(e5, "AND(=($0, 0), NOT(=($1, 1)))");
// "$0 = 0 AND NOT ($0 = 0 AND $1 = 1)" may be satisfiable. Can simplify.
final RexNode e6 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(
SqlStdOperatorTable.NOT,
rexBuilder.makeCall(SqlStdOperatorTable.AND, i0_eq_0, i1_eq_1)));
checkSatisfiable(e6, "AND(=($0, 0), NOT(AND(=($0, 0), =($1, 1))))");
// "$0 = 0 AND ($1 = 1 AND NOT ($0 = 0))" is not satisfiable.
final RexNode e7 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i1_eq_1,
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0)));
checkNotSatisfiable(e7);
// The expression "$2".
final RexInputRef i2 = rexBuilder.makeInputRef(typeFactory.createType(boolean.class), 2);
// The expression "$3".
final RexInputRef i3 = rexBuilder.makeInputRef(typeFactory.createType(boolean.class), 3);
// The expression "$4".
final RexInputRef i4 = rexBuilder.makeInputRef(typeFactory.createType(boolean.class), 4);
// "$0 = 0 AND $2 AND $3 AND NOT ($2 AND $3 AND $4) AND NOT ($2 AND $4)" may
// be satisfiable. Can't simplify.
final RexNode e8 =
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i0_eq_0,
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i2,
rexBuilder.makeCall(
SqlStdOperatorTable.AND,
i3,
rexBuilder.makeCall(
SqlStdOperatorTable.NOT,
rexBuilder.makeCall(SqlStdOperatorTable.AND, i2, i3, i4)),
rexBuilder.makeCall(SqlStdOperatorTable.NOT, i4))));
checkSatisfiable(e8, "AND(=($0, 0), $2, $3, NOT(AND($2, $3, $4)), NOT($4))");
}
private static void splitJoinCondition(
List<RelDataTypeField> sysFieldList,
List<RelNode> inputs,
RexNode condition,
List<List<RexNode>> joinKeys,
List<Integer> filterNulls,
List<SqlOperator> rangeOp,
List<RexNode> nonEquiList)
throws CalciteSemanticException {
final int sysFieldCount = sysFieldList.size();
final RelOptCluster cluster = inputs.get(0).getCluster();
final RexBuilder rexBuilder = cluster.getRexBuilder();
if (condition instanceof RexCall) {
RexCall call = (RexCall) condition;
if (call.getOperator() == SqlStdOperatorTable.AND) {
for (RexNode operand : call.getOperands()) {
splitJoinCondition(
sysFieldList, inputs, operand, joinKeys, filterNulls, rangeOp, nonEquiList);
}
return;
}
RexNode leftKey = null;
RexNode rightKey = null;
int leftInput = 0;
int rightInput = 0;
List<RelDataTypeField> leftFields = null;
List<RelDataTypeField> rightFields = null;
boolean reverse = false;
SqlKind kind = call.getKind();
// Only consider range operators if we haven't already seen one
if ((kind == SqlKind.EQUALS)
|| (filterNulls != null && kind == SqlKind.IS_NOT_DISTINCT_FROM)
|| (rangeOp != null
&& rangeOp.isEmpty()
&& (kind == SqlKind.GREATER_THAN
|| kind == SqlKind.GREATER_THAN_OR_EQUAL
|| kind == SqlKind.LESS_THAN
|| kind == SqlKind.LESS_THAN_OR_EQUAL))) {
final List<RexNode> operands = call.getOperands();
RexNode op0 = operands.get(0);
RexNode op1 = operands.get(1);
final ImmutableBitSet projRefs0 = InputFinder.bits(op0);
final ImmutableBitSet projRefs1 = InputFinder.bits(op1);
final ImmutableBitSet[] inputsRange = new ImmutableBitSet[inputs.size()];
int totalFieldCount = 0;
for (int i = 0; i < inputs.size(); i++) {
final int firstField = totalFieldCount + sysFieldCount;
totalFieldCount = firstField + inputs.get(i).getRowType().getFieldCount();
inputsRange[i] = ImmutableBitSet.range(firstField, totalFieldCount);
}
boolean foundBothInputs = false;
for (int i = 0; i < inputs.size() && !foundBothInputs; i++) {
if (projRefs0.intersects(inputsRange[i])
&& projRefs0.union(inputsRange[i]).equals(inputsRange[i])) {
if (leftKey == null) {
leftKey = op0;
leftInput = i;
leftFields = inputs.get(leftInput).getRowType().getFieldList();
} else {
rightKey = op0;
rightInput = i;
rightFields = inputs.get(rightInput).getRowType().getFieldList();
reverse = true;
foundBothInputs = true;
}
} else if (projRefs1.intersects(inputsRange[i])
&& projRefs1.union(inputsRange[i]).equals(inputsRange[i])) {
if (leftKey == null) {
leftKey = op1;
leftInput = i;
leftFields = inputs.get(leftInput).getRowType().getFieldList();
} else {
rightKey = op1;
rightInput = i;
rightFields = inputs.get(rightInput).getRowType().getFieldList();
foundBothInputs = true;
}
}
}
if ((leftKey != null) && (rightKey != null)) {
// adjustment array
int[] adjustments = new int[totalFieldCount];
for (int i = 0; i < inputs.size(); i++) {
final int adjustment = inputsRange[i].nextSetBit(0);
for (int j = adjustment; j < inputsRange[i].length(); j++) {
adjustments[j] = -adjustment;
}
}
// replace right Key input ref
rightKey =
rightKey.accept(
new RelOptUtil.RexInputConverter(
rexBuilder, rightFields, rightFields, adjustments));
// left key only needs to be adjusted if there are system
// fields, but do it for uniformity
leftKey =
leftKey.accept(
new RelOptUtil.RexInputConverter(
rexBuilder, leftFields, leftFields, adjustments));
RelDataType leftKeyType = leftKey.getType();
RelDataType rightKeyType = rightKey.getType();
if (leftKeyType != rightKeyType) {
// perform casting using Hive rules
TypeInfo rType = TypeConverter.convert(rightKeyType);
TypeInfo lType = TypeConverter.convert(leftKeyType);
TypeInfo tgtType = FunctionRegistry.getCommonClassForComparison(lType, rType);
if (tgtType == null) {
throw new CalciteSemanticException(
"Cannot find common type for join keys "
+ leftKey
+ " (type "
+ leftKeyType
+ ") and "
+ rightKey
+ " (type "
+ rightKeyType
+ ")");
}
RelDataType targetKeyType = TypeConverter.convert(tgtType, rexBuilder.getTypeFactory());
if (leftKeyType != targetKeyType
&& TypeInfoUtils.isConversionRequiredForComparison(tgtType, lType)) {
leftKey = rexBuilder.makeCast(targetKeyType, leftKey);
}
if (rightKeyType != targetKeyType
&& TypeInfoUtils.isConversionRequiredForComparison(tgtType, rType)) {
rightKey = rexBuilder.makeCast(targetKeyType, rightKey);
}
}
}
}
if ((leftKey != null) && (rightKey != null)) {
// found suitable join keys
// add them to key list, ensuring that if there is a
// non-equi join predicate, it appears at the end of the
// key list; also mark the null filtering property
addJoinKey(joinKeys.get(leftInput), leftKey, (rangeOp != null) && !rangeOp.isEmpty());
addJoinKey(joinKeys.get(rightInput), rightKey, (rangeOp != null) && !rangeOp.isEmpty());
if (filterNulls != null && kind == SqlKind.EQUALS) {
// nulls are considered not matching for equality comparison
// add the position of the most recently inserted key
filterNulls.add(joinKeys.get(leftInput).size() - 1);
}
if (rangeOp != null && kind != SqlKind.EQUALS && kind != SqlKind.IS_DISTINCT_FROM) {
if (reverse) {
kind = reverse(kind);
}
rangeOp.add(op(kind, call.getOperator()));
}
return;
} // else fall through and add this condition as nonEqui condition
}
// The operator is not of RexCall type
// So we fail. Fall through.
// Add this condition to the list of non-equi-join conditions.
nonEquiList.add(condition);
}
/**
* Push any equi join conditions that are not column references as Projections on top of the
* children.
*
* @param factory Project factory to use.
* @param inputRels inputs to a join
* @param leftJoinKeys expressions for LHS of join key
* @param rightJoinKeys expressions for RHS of join key
* @param systemColCount number of system columns, usually zero. These columns are projected at
* the leading edge of the output row.
* @param leftKeys on return this contains the join key positions from the new project rel on the
* LHS.
* @param rightKeys on return this contains the join key positions from the new project rel on the
* RHS.
* @return the join condition after the equi expressions pushed down.
*/
public static RexNode projectNonColumnEquiConditions(
ProjectFactory factory,
RelNode[] inputRels,
List<RexNode> leftJoinKeys,
List<RexNode> rightJoinKeys,
int systemColCount,
List<Integer> leftKeys,
List<Integer> rightKeys) {
RelNode leftRel = inputRels[0];
RelNode rightRel = inputRels[1];
RexBuilder rexBuilder = leftRel.getCluster().getRexBuilder();
RexNode outJoinCond = null;
int origLeftInputSize = leftRel.getRowType().getFieldCount();
int origRightInputSize = rightRel.getRowType().getFieldCount();
List<RexNode> newLeftFields = new ArrayList<RexNode>();
List<String> newLeftFieldNames = new ArrayList<String>();
List<RexNode> newRightFields = new ArrayList<RexNode>();
List<String> newRightFieldNames = new ArrayList<String>();
int leftKeyCount = leftJoinKeys.size();
int i;
for (i = 0; i < origLeftInputSize; i++) {
final RelDataTypeField field = leftRel.getRowType().getFieldList().get(i);
newLeftFields.add(rexBuilder.makeInputRef(field.getType(), i));
newLeftFieldNames.add(field.getName());
}
for (i = 0; i < origRightInputSize; i++) {
final RelDataTypeField field = rightRel.getRowType().getFieldList().get(i);
newRightFields.add(rexBuilder.makeInputRef(field.getType(), i));
newRightFieldNames.add(field.getName());
}
int newKeyCount = 0;
List<Pair<Integer, Integer>> origColEqConds = new ArrayList<Pair<Integer, Integer>>();
for (i = 0; i < leftKeyCount; i++) {
RexNode leftKey = leftJoinKeys.get(i);
RexNode rightKey = rightJoinKeys.get(i);
if (leftKey instanceof RexInputRef && rightKey instanceof RexInputRef) {
origColEqConds.add(
Pair.of(((RexInputRef) leftKey).getIndex(), ((RexInputRef) rightKey).getIndex()));
} else {
newLeftFields.add(leftKey);
newLeftFieldNames.add(null);
newRightFields.add(rightKey);
newRightFieldNames.add(null);
newKeyCount++;
}
}
for (i = 0; i < origColEqConds.size(); i++) {
Pair<Integer, Integer> p = origColEqConds.get(i);
RexNode leftKey = leftJoinKeys.get(i);
RexNode rightKey = rightJoinKeys.get(i);
leftKeys.add(p.left);
rightKeys.add(p.right);
RexNode cond =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(leftKey.getType(), systemColCount + p.left),
rexBuilder.makeInputRef(
rightKey.getType(), systemColCount + origLeftInputSize + newKeyCount + p.right));
if (outJoinCond == null) {
outJoinCond = cond;
} else {
outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
}
}
if (newKeyCount == 0) {
return outJoinCond;
}
int newLeftOffset = systemColCount + origLeftInputSize;
int newRightOffset = systemColCount + origLeftInputSize + origRightInputSize + newKeyCount;
for (i = 0; i < newKeyCount; i++) {
leftKeys.add(origLeftInputSize + i);
rightKeys.add(origRightInputSize + i);
RexNode cond =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(
newLeftFields.get(origLeftInputSize + i).getType(), newLeftOffset + i),
rexBuilder.makeInputRef(
newRightFields.get(origRightInputSize + i).getType(), newRightOffset + i));
if (outJoinCond == null) {
outJoinCond = cond;
} else {
outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
}
}
// added project if need to produce new keys than the original input
// fields
if (newKeyCount > 0) {
leftRel =
factory.createProject(
leftRel, newLeftFields, SqlValidatorUtil.uniquify(newLeftFieldNames));
rightRel =
factory.createProject(
rightRel, newRightFields, SqlValidatorUtil.uniquify(newRightFieldNames));
}
inputRels[0] = leftRel;
inputRels[1] = rightRel;
return outJoinCond;
}
protected RexNode convert(ExprNodeConstantDesc literal) throws CalciteSemanticException {
RexBuilder rexBuilder = cluster.getRexBuilder();
RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
PrimitiveTypeInfo hiveType = (PrimitiveTypeInfo) literal.getTypeInfo();
RelDataType calciteDataType = TypeConverter.convert(hiveType, dtFactory);
PrimitiveCategory hiveTypeCategory = hiveType.getPrimitiveCategory();
ConstantObjectInspector coi = literal.getWritableObjectInspector();
Object value =
ObjectInspectorUtils.copyToStandardJavaObject(coi.getWritableConstantValue(), coi);
RexNode calciteLiteral = null;
// TODO: Verify if we need to use ConstantObjectInspector to unwrap data
switch (hiveTypeCategory) {
case BOOLEAN:
calciteLiteral = rexBuilder.makeLiteral(((Boolean) value).booleanValue());
break;
case BYTE:
calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Byte) value), calciteDataType);
break;
case SHORT:
calciteLiteral =
rexBuilder.makeExactLiteral(new BigDecimal((Short) value), calciteDataType);
break;
case INT:
calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Integer) value));
break;
case LONG:
calciteLiteral = rexBuilder.makeBigintLiteral(new BigDecimal((Long) value));
break;
// TODO: is Decimal an exact numeric or approximate numeric?
case DECIMAL:
if (value instanceof HiveDecimal) {
value = ((HiveDecimal) value).bigDecimalValue();
} else if (value instanceof Decimal128) {
value = ((Decimal128) value).toBigDecimal();
}
if (value == null) {
// We have found an invalid decimal value while enforcing precision and
// scale. Ideally,
// we would replace it with null here, which is what Hive does. However,
// we need to plumb
// this thru up somehow, because otherwise having different expression
// type in AST causes
// the plan generation to fail after CBO, probably due to some residual
// state in SA/QB.
// For now, we will not run CBO in the presence of invalid decimal
// literals.
throw new CalciteSemanticException(
"Expression " + literal.getExprString() + " is not a valid decimal",
UnsupportedFeature.Invalid_decimal);
// TODO: return createNullLiteral(literal);
}
BigDecimal bd = (BigDecimal) value;
BigInteger unscaled = bd.unscaledValue();
if (unscaled.compareTo(MIN_LONG_BI) >= 0 && unscaled.compareTo(MAX_LONG_BI) <= 0) {
calciteLiteral = rexBuilder.makeExactLiteral(bd);
} else {
// CBO doesn't support unlimited precision decimals. In practice, this
// will work...
// An alternative would be to throw CboSemanticException and fall back
// to no CBO.
RelDataType relType =
cluster
.getTypeFactory()
.createSqlType(SqlTypeName.DECIMAL, bd.scale(), unscaled.toString().length());
calciteLiteral = rexBuilder.makeExactLiteral(bd, relType);
}
break;
case FLOAT:
calciteLiteral =
rexBuilder.makeApproxLiteral(new BigDecimal((Float) value), calciteDataType);
break;
case DOUBLE:
calciteLiteral =
rexBuilder.makeApproxLiteral(new BigDecimal((Double) value), calciteDataType);
break;
case CHAR:
if (value instanceof HiveChar) {
value = ((HiveChar) value).getValue();
}
calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
break;
case VARCHAR:
if (value instanceof HiveVarchar) {
value = ((HiveVarchar) value).getValue();
}
calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
break;
case STRING:
calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
break;
case DATE:
Calendar cal = new GregorianCalendar();
cal.setTime((Date) value);
calciteLiteral = rexBuilder.makeDateLiteral(cal);
break;
case TIMESTAMP:
Calendar c = null;
if (value instanceof Calendar) {
c = (Calendar) value;
} else {
c = Calendar.getInstance();
c.setTimeInMillis(((Timestamp) value).getTime());
}
calciteLiteral = rexBuilder.makeTimestampLiteral(c, RelDataType.PRECISION_NOT_SPECIFIED);
break;
case INTERVAL_YEAR_MONTH:
// Calcite year-month literal value is months as BigDecimal
BigDecimal totalMonths =
BigDecimal.valueOf(((HiveIntervalYearMonth) value).getTotalMonths());
calciteLiteral =
rexBuilder.makeIntervalLiteral(
totalMonths,
new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
break;
case INTERVAL_DAY_TIME:
// Calcite day-time interval is millis value as BigDecimal
// Seconds converted to millis
BigDecimal secsValueBd =
BigDecimal.valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000);
// Nanos converted to millis
BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getNanos(), 6);
calciteLiteral =
rexBuilder.makeIntervalLiteral(
secsValueBd.add(nanosValueBd),
new SqlIntervalQualifier(TimeUnit.DAY, TimeUnit.SECOND, new SqlParserPos(1, 1)));
break;
case VOID:
calciteLiteral =
cluster
.getRexBuilder()
.makeLiteral(null, cluster.getTypeFactory().createSqlType(SqlTypeName.NULL), true);
break;
case BINARY:
case UNKNOWN:
default:
throw new RuntimeException("UnSupported Literal");
}
return calciteLiteral;
}
@Override
public Prel visitProject(ProjectPrel project, Object unused) throws RelConversionException {
// Apply the rule to the child
RelNode originalInput = ((Prel) project.getInput(0)).accept(this, null);
project = (ProjectPrel) project.copy(project.getTraitSet(), Lists.newArrayList(originalInput));
List<RexNode> exprList = new ArrayList<>();
List<RelDataTypeField> relDataTypes = new ArrayList();
List<RelDataTypeField> origRelDataTypes = new ArrayList();
int i = 0;
final int lastColumnReferenced = PrelUtil.getLastUsedColumnReference(project.getProjects());
if (lastColumnReferenced == -1) {
return project;
}
final int lastRexInput = lastColumnReferenced + 1;
RexVisitorComplexExprSplitter exprSplitter =
new RexVisitorComplexExprSplitter(factory, funcReg, lastRexInput);
for (RexNode rex : project.getChildExps()) {
origRelDataTypes.add(project.getRowType().getFieldList().get(i));
i++;
exprList.add(rex.accept(exprSplitter));
}
List<RexNode> complexExprs = exprSplitter.getComplexExprs();
if (complexExprs.size() == 1 && findTopComplexFunc(project.getChildExps()).size() == 1) {
return project;
}
ProjectPrel childProject;
List<RexNode> allExprs = new ArrayList();
int exprIndex = 0;
List<String> fieldNames = originalInput.getRowType().getFieldNames();
for (int index = 0; index < lastRexInput; index++) {
RexBuilder builder = new RexBuilder(factory);
allExprs.add(
builder.makeInputRef(new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
if (fieldNames.get(index).contains(StarColumnHelper.STAR_COLUMN)) {
relDataTypes.add(
new RelDataTypeFieldImpl(
fieldNames.get(index), allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
} else {
relDataTypes.add(
new RelDataTypeFieldImpl(
"EXPR$" + exprIndex, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
exprIndex++;
}
}
RexNode currRexNode;
int index = lastRexInput - 1;
// if the projection expressions contained complex outputs, split them into their own individual
// projects
if (complexExprs.size() > 0) {
while (complexExprs.size() > 0) {
if (index >= lastRexInput) {
allExprs.remove(allExprs.size() - 1);
RexBuilder builder = new RexBuilder(factory);
allExprs.add(
builder.makeInputRef(
new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
}
index++;
exprIndex++;
currRexNode = complexExprs.remove(0);
allExprs.add(currRexNode);
relDataTypes.add(
new RelDataTypeFieldImpl(
"EXPR$" + exprIndex, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
childProject =
new ProjectPrel(
project.getCluster(),
project.getTraitSet(),
originalInput,
ImmutableList.copyOf(allExprs),
new RelRecordType(relDataTypes));
originalInput = childProject;
}
// copied from above, find a better way to do this
allExprs.remove(allExprs.size() - 1);
RexBuilder builder = new RexBuilder(factory);
allExprs.add(
builder.makeInputRef(new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
relDataTypes.add(
new RelDataTypeFieldImpl(
"EXPR$" + index, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
}
return (Prel)
project.copy(
project.getTraitSet(), originalInput, exprList, new RelRecordType(origRelDataTypes));
}
