public static List<ExprNodeDesc> getExprNodes(
List<Integer> inputRefs, RelNode inputRel, String inputTabAlias) {
List<ExprNodeDesc> exprNodes = new ArrayList<ExprNodeDesc>();
List<RexNode> rexInputRefs = getInputRef(inputRefs, inputRel);
List<RexNode> exprs = inputRel.getChildExps();
// TODO: Change ExprNodeConverter to be independent of Partition Expr
ExprNodeConverter exprConv =
new ExprNodeConverter(
inputTabAlias,
inputRel.getRowType(),
new HashSet<Integer>(),
inputRel.getCluster().getTypeFactory());
for (int index = 0; index < rexInputRefs.size(); index++) {
// The following check is only a guard against failures.
// TODO: Knowing which expr is constant in GBY's aggregation function
// arguments could be better done using Metadata provider of Calcite.
if (exprs != null && index < exprs.size() && exprs.get(index) instanceof RexLiteral) {
ExprNodeDesc exprNodeDesc = exprConv.visitLiteral((RexLiteral) exprs.get(index));
exprNodes.add(exprNodeDesc);
} else {
RexNode iRef = rexInputRefs.get(index);
exprNodes.add(iRef.accept(exprConv));
}
}
return exprNodes;
}
public static ImmutableList<RexNode> getInputRef(List<Integer> inputRefs, RelNode inputRel) {
ImmutableList.Builder<RexNode> bldr = ImmutableList.<RexNode>builder();
for (int i : inputRefs) {
bldr.add(new RexInputRef(i, inputRel.getRowType().getFieldList().get(i).getType()));
}
return bldr.build();
}
/**
* Creates a relational expression that projects the given fields of the input.
*
* <p>Optimizes if the fields are the identity projection.
*
* @param relBuilder RelBuilder
* @param child Input relational expression
* @param posList Source of each projected field
* @return Relational expression that projects given fields
*/
public static RelNode createProject(
final RelBuilder relBuilder, final RelNode child, final List<Integer> posList) {
RelDataType rowType = child.getRowType();
final List<String> fieldNames = rowType.getFieldNames();
final RexBuilder rexBuilder = child.getCluster().getRexBuilder();
return createProject(
child,
new AbstractList<RexNode>() {
public int size() {
return posList.size();
}
public RexNode get(int index) {
final int pos = posList.get(index);
return rexBuilder.makeInputRef(child, pos);
}
},
new AbstractList<String>() {
public int size() {
return posList.size();
}
public String get(int index) {
final int pos = posList.get(index);
return fieldNames.get(pos);
}
},
true,
relBuilder);
}
/**
* Infers predicates for an Aggregate.
*
* <p>Pulls up predicates that only contains references to columns in the GroupSet. For e.g.
*
* <pre>
* childPullUpExprs : { a > 7, b + c < 10, a + e = 9}
* groupSet : { a, b}
* pulledUpExprs : { a > 7}
* </pre>
*/
public RelOptPredicateList getPredicates(Aggregate agg) {
RelNode child = agg.getInput();
RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child);
List<RexNode> aggPullUpPredicates = new ArrayList<RexNode>();
ImmutableBitSet groupKeys = agg.getGroupSet();
Mapping m =
Mappings.create(
MappingType.PARTIAL_FUNCTION,
child.getRowType().getFieldCount(),
agg.getRowType().getFieldCount());
int i = 0;
for (int j : groupKeys) {
m.set(j, i++);
}
for (RexNode r : childInfo.pulledUpPredicates) {
ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r);
if (groupKeys.contains(rCols)) {
r = r.accept(new RexPermuteInputsShuttle(m, child));
aggPullUpPredicates.add(r);
}
}
return RelOptPredicateList.of(aggPullUpPredicates);
}
public static List<String> getFieldNames(List<Integer> inputRefs, RelNode inputRel) {
List<String> fieldNames = new ArrayList<String>();
List<String> schemaNames = inputRel.getRowType().getFieldNames();
for (Integer iRef : inputRefs) {
fieldNames.add(schemaNames.get(iRef));
}
return fieldNames;
}
public static ExprNodeDesc getExprNode(
Integer inputRefIndx, RelNode inputRel, ExprNodeConverter exprConv) {
ExprNodeDesc exprNode = null;
RexNode rexInputRef =
new RexInputRef(
inputRefIndx, inputRel.getRowType().getFieldList().get(inputRefIndx).getType());
exprNode = rexInputRef.accept(exprConv);
return exprNode;
}
public static RexNode convert(
RelOptCluster cluster,
ExprNodeDesc joinCondnExprNode,
List<RelNode> inputRels,
LinkedHashMap<RelNode, RowResolver> relToHiveRR,
Map<RelNode, ImmutableMap<String, Integer>> relToHiveColNameCalcitePosMap,
boolean flattenExpr)
throws SemanticException {
List<InputCtx> inputCtxLst = new ArrayList<InputCtx>();
int offSet = 0;
for (RelNode r : inputRels) {
inputCtxLst.add(
new InputCtx(
r.getRowType(), relToHiveColNameCalcitePosMap.get(r), relToHiveRR.get(r), offSet));
offSet += r.getRowType().getFieldCount();
}
return (new RexNodeConverter(cluster, inputCtxLst, flattenExpr)).convert(joinCondnExprNode);
}
/**
* Infers predicates for a project.
*
* <ol>
* <li>create a mapping from input to projection. Map only positions that directly reference an
* input column.
* <li>Expressions that only contain above columns are retained in the Project's pullExpressions
* list.
* <li>For e.g. expression 'a + e = 9' below will not be pulled up because 'e' is not in the
* projection list.
* <pre>
* childPullUpExprs: {a > 7, b + c < 10, a + e = 9}
* projectionExprs: {a, b, c, e / 2}
* projectionPullupExprs: {a > 7, b + c < 10}
* </pre>
* </ol>
*/
public RelOptPredicateList getPredicates(Project project) {
RelNode child = project.getInput();
final RexBuilder rexBuilder = project.getCluster().getRexBuilder();
RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child);
List<RexNode> projectPullUpPredicates = new ArrayList<RexNode>();
ImmutableBitSet.Builder columnsMappedBuilder = ImmutableBitSet.builder();
Mapping m =
Mappings.create(
MappingType.PARTIAL_FUNCTION,
child.getRowType().getFieldCount(),
project.getRowType().getFieldCount());
for (Ord<RexNode> o : Ord.zip(project.getProjects())) {
if (o.e instanceof RexInputRef) {
int sIdx = ((RexInputRef) o.e).getIndex();
m.set(sIdx, o.i);
columnsMappedBuilder.set(sIdx);
}
}
// Go over childPullUpPredicates. If a predicate only contains columns in
// 'columnsMapped' construct a new predicate based on mapping.
final ImmutableBitSet columnsMapped = columnsMappedBuilder.build();
for (RexNode r : childInfo.pulledUpPredicates) {
ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r);
if (columnsMapped.contains(rCols)) {
r = r.accept(new RexPermuteInputsShuttle(m, child));
projectPullUpPredicates.add(r);
}
}
// Project can also generate constants. We need to include them.
for (Ord<RexNode> expr : Ord.zip(project.getProjects())) {
if (RexLiteral.isNullLiteral(expr.e)) {
projectPullUpPredicates.add(
rexBuilder.makeCall(
SqlStdOperatorTable.IS_NULL, rexBuilder.makeInputRef(project, expr.i)));
} else if (RexUtil.isConstant(expr.e)) {
final List<RexNode> args =
ImmutableList.of(rexBuilder.makeInputRef(project, expr.i), expr.e);
final SqlOperator op =
args.get(0).getType().isNullable() || args.get(1).getType().isNullable()
? SqlStdOperatorTable.IS_NOT_DISTINCT_FROM
: SqlStdOperatorTable.EQUALS;
projectPullUpPredicates.add(rexBuilder.makeCall(op, args));
}
}
return RelOptPredicateList.of(projectPullUpPredicates);
}
public static List<RexNode> getProjsFromBelowAsInputRef(final RelNode rel) {
List<RexNode> projectList =
Lists.transform(
rel.getRowType().getFieldList(),
new Function<RelDataTypeField, RexNode>() {
@Override
public RexNode apply(RelDataTypeField field) {
return rel.getCluster()
.getRexBuilder()
.makeInputRef(field.getType(), field.getIndex());
}
});
return projectList;
}
/**
* Catch-all implementation for {@link BuiltInMetadata.Size#averageRowSize()}, invoked using
* reflection.
*
* @see org.apache.calcite.rel.metadata.RelMetadataQuery#getAverageRowSize
*/
public Double averageRowSize(RelNode rel, RelMetadataQuery mq) {
final List<Double> averageColumnSizes = mq.getAverageColumnSizes(rel);
if (averageColumnSizes == null) {
return null;
}
Double d = 0d;
final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
for (Pair<Double, RelDataTypeField> p : Pair.zip(averageColumnSizes, fields)) {
if (p.left == null) {
d += averageFieldValueSize(p.right);
} else {
d += p.left;
}
}
return d;
}
private List<Double> averageJoinColumnSizes(Join rel, RelMetadataQuery mq, boolean semijoin) {
final RelNode left = rel.getLeft();
final RelNode right = rel.getRight();
final List<Double> lefts = mq.getAverageColumnSizes(left);
final List<Double> rights = semijoin ? null : mq.getAverageColumnSizes(right);
if (lefts == null && rights == null) {
return null;
}
final int fieldCount = rel.getRowType().getFieldCount();
Double[] sizes = new Double[fieldCount];
if (lefts != null) {
lefts.toArray(sizes);
}
if (rights != null) {
final int leftCount = left.getRowType().getFieldCount();
for (int i = 0; i < rights.size(); i++) {
sizes[leftCount + i] = rights.get(i);
}
}
return ImmutableNullableList.copyOf(sizes);
}
public Double getDistinctRowCount(Union rel, ImmutableBitSet groupKey, RexNode predicate) {
Double rowCount = 0.0;
int[] adjustments = new int[rel.getRowType().getFieldCount()];
RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
for (RelNode input : rel.getInputs()) {
// convert the predicate to reference the types of the union child
RexNode modifiedPred;
if (predicate == null) {
modifiedPred = null;
} else {
modifiedPred =
predicate.accept(
new RelOptUtil.RexInputConverter(
rexBuilder, null, input.getRowType().getFieldList(), adjustments));
}
Double partialRowCount = RelMetadataQuery.getDistinctRowCount(input, groupKey, modifiedPred);
if (partialRowCount == null) {
return null;
}
rowCount += partialRowCount;
}
return rowCount;
}
@Override
public Prel visitProject(ProjectPrel project, Object unused) throws RelConversionException {
// Apply the rule to the child
RelNode originalInput = ((Prel) project.getInput(0)).accept(this, null);
project = (ProjectPrel) project.copy(project.getTraitSet(), Lists.newArrayList(originalInput));
List<RexNode> exprList = new ArrayList<>();
List<RelDataTypeField> relDataTypes = new ArrayList();
List<RelDataTypeField> origRelDataTypes = new ArrayList();
int i = 0;
final int lastColumnReferenced = PrelUtil.getLastUsedColumnReference(project.getProjects());
if (lastColumnReferenced == -1) {
return project;
}
final int lastRexInput = lastColumnReferenced + 1;
RexVisitorComplexExprSplitter exprSplitter =
new RexVisitorComplexExprSplitter(factory, funcReg, lastRexInput);
for (RexNode rex : project.getChildExps()) {
origRelDataTypes.add(project.getRowType().getFieldList().get(i));
i++;
exprList.add(rex.accept(exprSplitter));
}
List<RexNode> complexExprs = exprSplitter.getComplexExprs();
if (complexExprs.size() == 1 && findTopComplexFunc(project.getChildExps()).size() == 1) {
return project;
}
ProjectPrel childProject;
List<RexNode> allExprs = new ArrayList();
int exprIndex = 0;
List<String> fieldNames = originalInput.getRowType().getFieldNames();
for (int index = 0; index < lastRexInput; index++) {
RexBuilder builder = new RexBuilder(factory);
allExprs.add(
builder.makeInputRef(new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
if (fieldNames.get(index).contains(StarColumnHelper.STAR_COLUMN)) {
relDataTypes.add(
new RelDataTypeFieldImpl(
fieldNames.get(index), allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
} else {
relDataTypes.add(
new RelDataTypeFieldImpl(
"EXPR$" + exprIndex, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
exprIndex++;
}
}
RexNode currRexNode;
int index = lastRexInput - 1;
// if the projection expressions contained complex outputs, split them into their own individual
// projects
if (complexExprs.size() > 0) {
while (complexExprs.size() > 0) {
if (index >= lastRexInput) {
allExprs.remove(allExprs.size() - 1);
RexBuilder builder = new RexBuilder(factory);
allExprs.add(
builder.makeInputRef(
new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
}
index++;
exprIndex++;
currRexNode = complexExprs.remove(0);
allExprs.add(currRexNode);
relDataTypes.add(
new RelDataTypeFieldImpl(
"EXPR$" + exprIndex, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
childProject =
new ProjectPrel(
project.getCluster(),
project.getTraitSet(),
originalInput,
ImmutableList.copyOf(allExprs),
new RelRecordType(relDataTypes));
originalInput = childProject;
}
// copied from above, find a better way to do this
allExprs.remove(allExprs.size() - 1);
RexBuilder builder = new RexBuilder(factory);
allExprs.add(
builder.makeInputRef(new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
relDataTypes.add(
new RelDataTypeFieldImpl(
"EXPR$" + index, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
}
return (Prel)
project.copy(
project.getTraitSet(), originalInput, exprList, new RelRecordType(origRelDataTypes));
}
/**
* Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for {@link
* org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin}.
*/
public TrimResult trimFields(
HiveMultiJoin join, ImmutableBitSet fieldsUsed, Set<RelDataTypeField> extraFields) {
final int fieldCount = join.getRowType().getFieldCount();
final RexNode conditionExpr = join.getCondition();
// Add in fields used in the condition.
final Set<RelDataTypeField> combinedInputExtraFields =
new LinkedHashSet<RelDataTypeField>(extraFields);
RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(combinedInputExtraFields);
inputFinder.inputBitSet.addAll(fieldsUsed);
conditionExpr.accept(inputFinder);
final ImmutableBitSet fieldsUsedPlus = inputFinder.inputBitSet.build();
int inputStartPos = 0;
int changeCount = 0;
int newFieldCount = 0;
List<RelNode> newInputs = new ArrayList<RelNode>();
List<Mapping> inputMappings = new ArrayList<Mapping>();
for (RelNode input : join.getInputs()) {
final RelDataType inputRowType = input.getRowType();
final int inputFieldCount = inputRowType.getFieldCount();
// Compute required mapping.
ImmutableBitSet.Builder inputFieldsUsed = ImmutableBitSet.builder();
for (int bit : fieldsUsedPlus) {
if (bit >= inputStartPos && bit < inputStartPos + inputFieldCount) {
inputFieldsUsed.set(bit - inputStartPos);
}
}
Set<RelDataTypeField> inputExtraFields = Collections.<RelDataTypeField>emptySet();
TrimResult trimResult = trimChild(join, input, inputFieldsUsed.build(), inputExtraFields);
newInputs.add(trimResult.left);
if (trimResult.left != input) {
++changeCount;
}
final Mapping inputMapping = trimResult.right;
inputMappings.add(inputMapping);
// Move offset to point to start of next input.
inputStartPos += inputFieldCount;
newFieldCount += inputMapping.getTargetCount();
}
Mapping mapping = Mappings.create(MappingType.INVERSE_SURJECTION, fieldCount, newFieldCount);
int offset = 0;
int newOffset = 0;
for (int i = 0; i < inputMappings.size(); i++) {
Mapping inputMapping = inputMappings.get(i);
for (IntPair pair : inputMapping) {
mapping.set(pair.source + offset, pair.target + newOffset);
}
offset += inputMapping.getSourceCount();
newOffset += inputMapping.getTargetCount();
}
if (changeCount == 0 && mapping.isIdentity()) {
return new TrimResult(join, Mappings.createIdentity(fieldCount));
}
// Build new join.
final RexVisitor<RexNode> shuttle =
new RexPermuteInputsShuttle(mapping, newInputs.toArray(new RelNode[newInputs.size()]));
RexNode newConditionExpr = conditionExpr.accept(shuttle);
final RelDataType newRowType =
RelOptUtil.permute(join.getCluster().getTypeFactory(), join.getRowType(), mapping);
final RelNode newJoin =
new HiveMultiJoin(
join.getCluster(),
newInputs,
newConditionExpr,
newRowType,
join.getJoinInputs(),
join.getJoinTypes(),
join.getJoinFilters());
return new TrimResult(newJoin, mapping);
}
/**
* Creates a reference to a given field of the input relational expression.
*
* @param input Input relational expression
* @param i Ordinal of field
* @return Reference to field
*/
public RexInputRef makeInputRef(RelNode input, int i) {
return makeInputRef(input.getRowType().getFieldList().get(i).getType(), i);
}
/**
* Creates a reference to all the fields in the row. That is, the whole row as a single record
* object.
*
* @param input Input relational expression
*/
public RexNode makeRangeReference(RelNode input) {
return new RexRangeRef(input.getRowType(), 0);
}
/**
* Push any equi join conditions that are not column references as Projections on top of the
* children.
*
* @param factory Project factory to use.
* @param inputRels inputs to a join
* @param leftJoinKeys expressions for LHS of join key
* @param rightJoinKeys expressions for RHS of join key
* @param systemColCount number of system columns, usually zero. These columns are projected at
* the leading edge of the output row.
* @param leftKeys on return this contains the join key positions from the new project rel on the
* LHS.
* @param rightKeys on return this contains the join key positions from the new project rel on the
* RHS.
* @return the join condition after the equi expressions pushed down.
*/
public static RexNode projectNonColumnEquiConditions(
ProjectFactory factory,
RelNode[] inputRels,
List<RexNode> leftJoinKeys,
List<RexNode> rightJoinKeys,
int systemColCount,
List<Integer> leftKeys,
List<Integer> rightKeys) {
RelNode leftRel = inputRels[0];
RelNode rightRel = inputRels[1];
RexBuilder rexBuilder = leftRel.getCluster().getRexBuilder();
RexNode outJoinCond = null;
int origLeftInputSize = leftRel.getRowType().getFieldCount();
int origRightInputSize = rightRel.getRowType().getFieldCount();
List<RexNode> newLeftFields = new ArrayList<RexNode>();
List<String> newLeftFieldNames = new ArrayList<String>();
List<RexNode> newRightFields = new ArrayList<RexNode>();
List<String> newRightFieldNames = new ArrayList<String>();
int leftKeyCount = leftJoinKeys.size();
int i;
for (i = 0; i < origLeftInputSize; i++) {
final RelDataTypeField field = leftRel.getRowType().getFieldList().get(i);
newLeftFields.add(rexBuilder.makeInputRef(field.getType(), i));
newLeftFieldNames.add(field.getName());
}
for (i = 0; i < origRightInputSize; i++) {
final RelDataTypeField field = rightRel.getRowType().getFieldList().get(i);
newRightFields.add(rexBuilder.makeInputRef(field.getType(), i));
newRightFieldNames.add(field.getName());
}
int newKeyCount = 0;
List<Pair<Integer, Integer>> origColEqConds = new ArrayList<Pair<Integer, Integer>>();
for (i = 0; i < leftKeyCount; i++) {
RexNode leftKey = leftJoinKeys.get(i);
RexNode rightKey = rightJoinKeys.get(i);
if (leftKey instanceof RexInputRef && rightKey instanceof RexInputRef) {
origColEqConds.add(
Pair.of(((RexInputRef) leftKey).getIndex(), ((RexInputRef) rightKey).getIndex()));
} else {
newLeftFields.add(leftKey);
newLeftFieldNames.add(null);
newRightFields.add(rightKey);
newRightFieldNames.add(null);
newKeyCount++;
}
}
for (i = 0; i < origColEqConds.size(); i++) {
Pair<Integer, Integer> p = origColEqConds.get(i);
RexNode leftKey = leftJoinKeys.get(i);
RexNode rightKey = rightJoinKeys.get(i);
leftKeys.add(p.left);
rightKeys.add(p.right);
RexNode cond =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(leftKey.getType(), systemColCount + p.left),
rexBuilder.makeInputRef(
rightKey.getType(), systemColCount + origLeftInputSize + newKeyCount + p.right));
if (outJoinCond == null) {
outJoinCond = cond;
} else {
outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
}
}
if (newKeyCount == 0) {
return outJoinCond;
}
int newLeftOffset = systemColCount + origLeftInputSize;
int newRightOffset = systemColCount + origLeftInputSize + origRightInputSize + newKeyCount;
for (i = 0; i < newKeyCount; i++) {
leftKeys.add(origLeftInputSize + i);
rightKeys.add(origRightInputSize + i);
RexNode cond =
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(
newLeftFields.get(origLeftInputSize + i).getType(), newLeftOffset + i),
rexBuilder.makeInputRef(
newRightFields.get(origRightInputSize + i).getType(), newRightOffset + i));
if (outJoinCond == null) {
outJoinCond = cond;
} else {
outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
}
}
// added project if need to produce new keys than the original input
// fields
if (newKeyCount > 0) {
leftRel =
factory.createProject(
leftRel, newLeftFields, SqlValidatorUtil.uniquify(newLeftFieldNames));
rightRel =
factory.createProject(
rightRel, newRightFields, SqlValidatorUtil.uniquify(newRightFieldNames));
}
inputRels[0] = leftRel;
inputRels[1] = rightRel;
return outJoinCond;
}