Ejemplo n.º 1
0
  protected DrillRel addRenamedProject(DrillRel rel, RelDataType validatedRowType) {
    RelDataType t = rel.getRowType();

    RexBuilder b = rel.getCluster().getRexBuilder();
    List<RexNode> projections = Lists.newArrayList();
    int projectCount = t.getFieldList().size();

    for (int i = 0; i < projectCount; i++) {
      projections.add(b.makeInputRef(rel, i));
    }

    final List<String> fieldNames2 =
        SqlValidatorUtil.uniquify(validatedRowType.getFieldNames(), SqlValidatorUtil.F_SUGGESTER2);

    RelDataType newRowType =
        RexUtil.createStructType(rel.getCluster().getTypeFactory(), projections, fieldNames2);

    DrillProjectRel topProj =
        DrillProjectRel.create(rel.getCluster(), rel.getTraitSet(), rel, projections, newRowType);

    // Add a final non-trivial Project to get the validatedRowType, if child is not project.
    if (rel instanceof Project && DrillRelOptUtil.isTrivialProject(topProj, true)) {
      return rel;
    } else {
      return topProj;
    }
  }
Ejemplo n.º 2
0
  /**
   * Infers predicates for a project.
   *
   * <ol>
   *   <li>create a mapping from input to projection. Map only positions that directly reference an
   *       input column.
   *   <li>Expressions that only contain above columns are retained in the Project's pullExpressions
   *       list.
   *   <li>For e.g. expression 'a + e = 9' below will not be pulled up because 'e' is not in the
   *       projection list.
   *       <pre>
   * childPullUpExprs:      {a &gt; 7, b + c &lt; 10, a + e = 9}
   * projectionExprs:       {a, b, c, e / 2}
   * projectionPullupExprs: {a &gt; 7, b + c &lt; 10}
   * </pre>
   * </ol>
   */
  public RelOptPredicateList getPredicates(Project project) {
    RelNode child = project.getInput();
    final RexBuilder rexBuilder = project.getCluster().getRexBuilder();
    RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child);

    List<RexNode> projectPullUpPredicates = new ArrayList<RexNode>();

    ImmutableBitSet.Builder columnsMappedBuilder = ImmutableBitSet.builder();
    Mapping m =
        Mappings.create(
            MappingType.PARTIAL_FUNCTION,
            child.getRowType().getFieldCount(),
            project.getRowType().getFieldCount());

    for (Ord<RexNode> o : Ord.zip(project.getProjects())) {
      if (o.e instanceof RexInputRef) {
        int sIdx = ((RexInputRef) o.e).getIndex();
        m.set(sIdx, o.i);
        columnsMappedBuilder.set(sIdx);
      }
    }

    // Go over childPullUpPredicates. If a predicate only contains columns in
    // 'columnsMapped' construct a new predicate based on mapping.
    final ImmutableBitSet columnsMapped = columnsMappedBuilder.build();
    for (RexNode r : childInfo.pulledUpPredicates) {
      ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r);
      if (columnsMapped.contains(rCols)) {
        r = r.accept(new RexPermuteInputsShuttle(m, child));
        projectPullUpPredicates.add(r);
      }
    }

    // Project can also generate constants. We need to include them.
    for (Ord<RexNode> expr : Ord.zip(project.getProjects())) {
      if (RexLiteral.isNullLiteral(expr.e)) {
        projectPullUpPredicates.add(
            rexBuilder.makeCall(
                SqlStdOperatorTable.IS_NULL, rexBuilder.makeInputRef(project, expr.i)));
      } else if (RexUtil.isConstant(expr.e)) {
        final List<RexNode> args =
            ImmutableList.of(rexBuilder.makeInputRef(project, expr.i), expr.e);
        final SqlOperator op =
            args.get(0).getType().isNullable() || args.get(1).getType().isNullable()
                ? SqlStdOperatorTable.IS_NOT_DISTINCT_FROM
                : SqlStdOperatorTable.EQUALS;
        projectPullUpPredicates.add(rexBuilder.makeCall(op, args));
      }
    }
    return RelOptPredicateList.of(projectPullUpPredicates);
  }
Ejemplo n.º 3
0
  public static RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) {
    RexNode typeSafeRex = rex;
    if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) {
      RexBuilder rb = cluster.getRexBuilder();
      List<RexNode> fixedPredElems = new ArrayList<RexNode>();
      RelDataType commonType =
          cluster.getTypeFactory().leastRestrictive(RexUtil.types(((RexCall) rex).getOperands()));
      for (RexNode rn : ((RexCall) rex).getOperands()) {
        fixedPredElems.add(rb.ensureType(commonType, rn, true));
      }

      typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems);
    }

    return typeSafeRex;
  }
  public Double getDistinctRowCount(SemiJoin rel, ImmutableBitSet groupKey, RexNode predicate) {
    if (predicate == null || predicate.isAlwaysTrue()) {
      if (groupKey.isEmpty()) {
        return 1D;
      }
    }
    // create a RexNode representing the selectivity of the
    // semijoin filter and pass it to getDistinctRowCount
    RexNode newPred = RelMdUtil.makeSemiJoinSelectivityRexNode(rel);
    if (predicate != null) {
      RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
      newPred = rexBuilder.makeCall(SqlStdOperatorTable.AND, newPred, predicate);
    }

    return RelMetadataQuery.getDistinctRowCount(rel.getLeft(), groupKey, newPred);
  }
  /** Creates a cluster. */
  RelOptCluster(
      RelOptQuery query,
      RelOptPlanner planner,
      RelDataTypeFactory typeFactory,
      RexBuilder rexBuilder) {
    assert planner != null;
    assert typeFactory != null;
    this.query = query;
    this.planner = planner;
    this.typeFactory = typeFactory;
    this.rexBuilder = rexBuilder;
    this.originalExpression = rexBuilder.makeLiteral("?");

    // set up a default rel metadata provider,
    // giving the planner first crack at everything
    setMetadataProvider(new DefaultRelMetadataProvider());
    this.emptyTraitSet = planner.emptyTraitSet();
  }
  @Test
  public void testSplitFilter() {
    final RexLiteral i1 = rexBuilder.makeExactLiteral(BigDecimal.ONE);
    final RexLiteral i2 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(2));
    final RexLiteral i3 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(3));

    final RelDataType intType = typeFactory.createType(int.class);
    final RexInputRef x = rexBuilder.makeInputRef(intType, 0); // $0
    final RexInputRef y = rexBuilder.makeInputRef(intType, 1); // $1
    final RexInputRef z = rexBuilder.makeInputRef(intType, 2); // $2

    final RexNode x_eq_1 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1
    final RexNode x_eq_1_b = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, x, i1); // $0 = 1 again
    final RexNode y_eq_2 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, y, i2); // $1 = 2
    final RexNode z_eq_3 = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, z, i3); // $2 = 3

    RexNode newFilter;

    // Example 1.
    // TODO:

    // Example 2.
    //   condition: x = 1,
    //   target:    x = 1 or z = 3
    // yields
    //   residue:   not (z = 3)
    newFilter =
        SubstitutionVisitor.splitFilter(
            rexBuilder, x_eq_1, rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, z_eq_3));
    assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))"));

    // 2b.
    //   condition: x = 1 or y = 2
    //   target:    x = 1 or y = 2 or z = 3
    // yields
    //   residue:   not (z = 3)
    newFilter =
        SubstitutionVisitor.splitFilter(
            rexBuilder,
            rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2),
            rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3));
    assertThat(newFilter.toString(), equalTo("NOT(=($2, 3))"));

    // 2c.
    //   condition: x = 1
    //   target:    x = 1 or y = 2 or z = 3
    // yields
    //   residue:   not (y = 2) and not (z = 3)
    newFilter =
        SubstitutionVisitor.splitFilter(
            rexBuilder,
            x_eq_1,
            rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2, z_eq_3));
    assertThat(newFilter.toString(), equalTo("AND(NOT(=($1, 2)), NOT(=($2, 3)))"));

    // 2d.
    //   condition: x = 1 or y = 2
    //   target:    y = 2 or x = 1
    // yields
    //   residue:   true
    newFilter =
        SubstitutionVisitor.splitFilter(
            rexBuilder,
            rexBuilder.makeCall(SqlStdOperatorTable.OR, x_eq_1, y_eq_2),
            rexBuilder.makeCall(SqlStdOperatorTable.OR, y_eq_2, x_eq_1));
    assertThat(newFilter.isAlwaysTrue(), equalTo(true));

    // 2e.
    //   condition: x = 1
    //   target:    x = 1 (different object)
    // yields
    //   residue:   true
    newFilter = SubstitutionVisitor.splitFilter(rexBuilder, x_eq_1, x_eq_1_b);
    assertThat(newFilter.isAlwaysTrue(), equalTo(true));

    // 2f.
    //   condition: x = 1 or y = 2
    //   target:    x = 1
    // yields
    //   residue:   null
    // TODO:

    // Example 3.
    // Condition [x = 1 and y = 2],
    // target [y = 2 and x = 1] yields
    // residue [true].
    // TODO:

    // Example 4.
    // TODO:
  }
  /**
   * Unit test for logic functions {@link
   * org.apache.calcite.plan.SubstitutionVisitor#mayBeSatisfiable} and {@link
   * org.apache.calcite.plan.SubstitutionVisitor#simplify}.
   */
  @Test
  public void testSatisfiable() {
    // TRUE may be satisfiable
    checkSatisfiable(rexBuilder.makeLiteral(true), "true");

    // FALSE is not satisfiable
    checkNotSatisfiable(rexBuilder.makeLiteral(false));

    // The expression "$0 = 1".
    final RexNode i0_eq_0 =
        rexBuilder.makeCall(
            SqlStdOperatorTable.EQUALS,
            rexBuilder.makeInputRef(typeFactory.createType(int.class), 0),
            rexBuilder.makeExactLiteral(BigDecimal.ZERO));

    // "$0 = 1" may be satisfiable
    checkSatisfiable(i0_eq_0, "=($0, 0)");

    // "$0 = 1 AND TRUE" may be satisfiable
    final RexNode e0 =
        rexBuilder.makeCall(SqlStdOperatorTable.AND, i0_eq_0, rexBuilder.makeLiteral(true));
    checkSatisfiable(e0, "=($0, 0)");

    // "$0 = 1 AND FALSE" is not satisfiable
    final RexNode e1 =
        rexBuilder.makeCall(SqlStdOperatorTable.AND, i0_eq_0, rexBuilder.makeLiteral(false));
    checkNotSatisfiable(e1);

    // "$0 = 0 AND NOT $0 = 0" is not satisfiable
    final RexNode e2 =
        rexBuilder.makeCall(
            SqlStdOperatorTable.AND,
            i0_eq_0,
            rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0));
    checkNotSatisfiable(e2);

    // "TRUE AND NOT $0 = 0" may be satisfiable. Can simplify.
    final RexNode e3 =
        rexBuilder.makeCall(
            SqlStdOperatorTable.AND,
            rexBuilder.makeLiteral(true),
            rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0));
    checkSatisfiable(e3, "NOT(=($0, 0))");

    // The expression "$1 = 1".
    final RexNode i1_eq_1 =
        rexBuilder.makeCall(
            SqlStdOperatorTable.EQUALS,
            rexBuilder.makeInputRef(typeFactory.createType(int.class), 1),
            rexBuilder.makeExactLiteral(BigDecimal.ONE));

    // "$0 = 0 AND $1 = 1 AND NOT $0 = 0" is not satisfiable
    final RexNode e4 =
        rexBuilder.makeCall(
            SqlStdOperatorTable.AND,
            i0_eq_0,
            rexBuilder.makeCall(
                SqlStdOperatorTable.AND,
                i1_eq_1,
                rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0)));
    checkNotSatisfiable(e4);

    // "$0 = 0 AND NOT $1 = 1" may be satisfiable. Can't simplify.
    final RexNode e5 =
        rexBuilder.makeCall(
            SqlStdOperatorTable.AND,
            i0_eq_0,
            rexBuilder.makeCall(SqlStdOperatorTable.NOT, i1_eq_1));
    checkSatisfiable(e5, "AND(=($0, 0), NOT(=($1, 1)))");

    // "$0 = 0 AND NOT ($0 = 0 AND $1 = 1)" may be satisfiable. Can simplify.
    final RexNode e6 =
        rexBuilder.makeCall(
            SqlStdOperatorTable.AND,
            i0_eq_0,
            rexBuilder.makeCall(
                SqlStdOperatorTable.NOT,
                rexBuilder.makeCall(SqlStdOperatorTable.AND, i0_eq_0, i1_eq_1)));
    checkSatisfiable(e6, "AND(=($0, 0), NOT(AND(=($0, 0), =($1, 1))))");

    // "$0 = 0 AND ($1 = 1 AND NOT ($0 = 0))" is not satisfiable.
    final RexNode e7 =
        rexBuilder.makeCall(
            SqlStdOperatorTable.AND,
            i0_eq_0,
            rexBuilder.makeCall(
                SqlStdOperatorTable.AND,
                i1_eq_1,
                rexBuilder.makeCall(SqlStdOperatorTable.NOT, i0_eq_0)));
    checkNotSatisfiable(e7);

    // The expression "$2".
    final RexInputRef i2 = rexBuilder.makeInputRef(typeFactory.createType(boolean.class), 2);

    // The expression "$3".
    final RexInputRef i3 = rexBuilder.makeInputRef(typeFactory.createType(boolean.class), 3);

    // The expression "$4".
    final RexInputRef i4 = rexBuilder.makeInputRef(typeFactory.createType(boolean.class), 4);

    // "$0 = 0 AND $2 AND $3 AND NOT ($2 AND $3 AND $4) AND NOT ($2 AND $4)" may
    // be satisfiable. Can't simplify.
    final RexNode e8 =
        rexBuilder.makeCall(
            SqlStdOperatorTable.AND,
            i0_eq_0,
            rexBuilder.makeCall(
                SqlStdOperatorTable.AND,
                i2,
                rexBuilder.makeCall(
                    SqlStdOperatorTable.AND,
                    i3,
                    rexBuilder.makeCall(
                        SqlStdOperatorTable.NOT,
                        rexBuilder.makeCall(SqlStdOperatorTable.AND, i2, i3, i4)),
                    rexBuilder.makeCall(SqlStdOperatorTable.NOT, i4))));
    checkSatisfiable(e8, "AND(=($0, 0), $2, $3, NOT(AND($2, $3, $4)), NOT($4))");
  }
Ejemplo n.º 8
0
  private static void splitJoinCondition(
      List<RelDataTypeField> sysFieldList,
      List<RelNode> inputs,
      RexNode condition,
      List<List<RexNode>> joinKeys,
      List<Integer> filterNulls,
      List<SqlOperator> rangeOp,
      List<RexNode> nonEquiList)
      throws CalciteSemanticException {
    final int sysFieldCount = sysFieldList.size();
    final RelOptCluster cluster = inputs.get(0).getCluster();
    final RexBuilder rexBuilder = cluster.getRexBuilder();

    if (condition instanceof RexCall) {
      RexCall call = (RexCall) condition;
      if (call.getOperator() == SqlStdOperatorTable.AND) {
        for (RexNode operand : call.getOperands()) {
          splitJoinCondition(
              sysFieldList, inputs, operand, joinKeys, filterNulls, rangeOp, nonEquiList);
        }
        return;
      }

      RexNode leftKey = null;
      RexNode rightKey = null;
      int leftInput = 0;
      int rightInput = 0;
      List<RelDataTypeField> leftFields = null;
      List<RelDataTypeField> rightFields = null;
      boolean reverse = false;

      SqlKind kind = call.getKind();

      // Only consider range operators if we haven't already seen one
      if ((kind == SqlKind.EQUALS)
          || (filterNulls != null && kind == SqlKind.IS_NOT_DISTINCT_FROM)
          || (rangeOp != null
              && rangeOp.isEmpty()
              && (kind == SqlKind.GREATER_THAN
                  || kind == SqlKind.GREATER_THAN_OR_EQUAL
                  || kind == SqlKind.LESS_THAN
                  || kind == SqlKind.LESS_THAN_OR_EQUAL))) {
        final List<RexNode> operands = call.getOperands();
        RexNode op0 = operands.get(0);
        RexNode op1 = operands.get(1);

        final ImmutableBitSet projRefs0 = InputFinder.bits(op0);
        final ImmutableBitSet projRefs1 = InputFinder.bits(op1);

        final ImmutableBitSet[] inputsRange = new ImmutableBitSet[inputs.size()];
        int totalFieldCount = 0;
        for (int i = 0; i < inputs.size(); i++) {
          final int firstField = totalFieldCount + sysFieldCount;
          totalFieldCount = firstField + inputs.get(i).getRowType().getFieldCount();
          inputsRange[i] = ImmutableBitSet.range(firstField, totalFieldCount);
        }

        boolean foundBothInputs = false;
        for (int i = 0; i < inputs.size() && !foundBothInputs; i++) {
          if (projRefs0.intersects(inputsRange[i])
              && projRefs0.union(inputsRange[i]).equals(inputsRange[i])) {
            if (leftKey == null) {
              leftKey = op0;
              leftInput = i;
              leftFields = inputs.get(leftInput).getRowType().getFieldList();
            } else {
              rightKey = op0;
              rightInput = i;
              rightFields = inputs.get(rightInput).getRowType().getFieldList();
              reverse = true;
              foundBothInputs = true;
            }
          } else if (projRefs1.intersects(inputsRange[i])
              && projRefs1.union(inputsRange[i]).equals(inputsRange[i])) {
            if (leftKey == null) {
              leftKey = op1;
              leftInput = i;
              leftFields = inputs.get(leftInput).getRowType().getFieldList();
            } else {
              rightKey = op1;
              rightInput = i;
              rightFields = inputs.get(rightInput).getRowType().getFieldList();
              foundBothInputs = true;
            }
          }
        }

        if ((leftKey != null) && (rightKey != null)) {
          // adjustment array
          int[] adjustments = new int[totalFieldCount];
          for (int i = 0; i < inputs.size(); i++) {
            final int adjustment = inputsRange[i].nextSetBit(0);
            for (int j = adjustment; j < inputsRange[i].length(); j++) {
              adjustments[j] = -adjustment;
            }
          }

          // replace right Key input ref
          rightKey =
              rightKey.accept(
                  new RelOptUtil.RexInputConverter(
                      rexBuilder, rightFields, rightFields, adjustments));

          // left key only needs to be adjusted if there are system
          // fields, but do it for uniformity
          leftKey =
              leftKey.accept(
                  new RelOptUtil.RexInputConverter(
                      rexBuilder, leftFields, leftFields, adjustments));

          RelDataType leftKeyType = leftKey.getType();
          RelDataType rightKeyType = rightKey.getType();

          if (leftKeyType != rightKeyType) {
            // perform casting using Hive rules
            TypeInfo rType = TypeConverter.convert(rightKeyType);
            TypeInfo lType = TypeConverter.convert(leftKeyType);
            TypeInfo tgtType = FunctionRegistry.getCommonClassForComparison(lType, rType);

            if (tgtType == null) {
              throw new CalciteSemanticException(
                  "Cannot find common type for join keys "
                      + leftKey
                      + " (type "
                      + leftKeyType
                      + ") and "
                      + rightKey
                      + " (type "
                      + rightKeyType
                      + ")");
            }
            RelDataType targetKeyType = TypeConverter.convert(tgtType, rexBuilder.getTypeFactory());

            if (leftKeyType != targetKeyType
                && TypeInfoUtils.isConversionRequiredForComparison(tgtType, lType)) {
              leftKey = rexBuilder.makeCast(targetKeyType, leftKey);
            }

            if (rightKeyType != targetKeyType
                && TypeInfoUtils.isConversionRequiredForComparison(tgtType, rType)) {
              rightKey = rexBuilder.makeCast(targetKeyType, rightKey);
            }
          }
        }
      }

      if ((leftKey != null) && (rightKey != null)) {
        // found suitable join keys
        // add them to key list, ensuring that if there is a
        // non-equi join predicate, it appears at the end of the
        // key list; also mark the null filtering property
        addJoinKey(joinKeys.get(leftInput), leftKey, (rangeOp != null) && !rangeOp.isEmpty());
        addJoinKey(joinKeys.get(rightInput), rightKey, (rangeOp != null) && !rangeOp.isEmpty());
        if (filterNulls != null && kind == SqlKind.EQUALS) {
          // nulls are considered not matching for equality comparison
          // add the position of the most recently inserted key
          filterNulls.add(joinKeys.get(leftInput).size() - 1);
        }
        if (rangeOp != null && kind != SqlKind.EQUALS && kind != SqlKind.IS_DISTINCT_FROM) {
          if (reverse) {
            kind = reverse(kind);
          }
          rangeOp.add(op(kind, call.getOperator()));
        }
        return;
      } // else fall through and add this condition as nonEqui condition
    }

    // The operator is not of RexCall type
    // So we fail. Fall through.
    // Add this condition to the list of non-equi-join conditions.
    nonEquiList.add(condition);
  }
Ejemplo n.º 9
0
  /**
   * Push any equi join conditions that are not column references as Projections on top of the
   * children.
   *
   * @param factory Project factory to use.
   * @param inputRels inputs to a join
   * @param leftJoinKeys expressions for LHS of join key
   * @param rightJoinKeys expressions for RHS of join key
   * @param systemColCount number of system columns, usually zero. These columns are projected at
   *     the leading edge of the output row.
   * @param leftKeys on return this contains the join key positions from the new project rel on the
   *     LHS.
   * @param rightKeys on return this contains the join key positions from the new project rel on the
   *     RHS.
   * @return the join condition after the equi expressions pushed down.
   */
  public static RexNode projectNonColumnEquiConditions(
      ProjectFactory factory,
      RelNode[] inputRels,
      List<RexNode> leftJoinKeys,
      List<RexNode> rightJoinKeys,
      int systemColCount,
      List<Integer> leftKeys,
      List<Integer> rightKeys) {
    RelNode leftRel = inputRels[0];
    RelNode rightRel = inputRels[1];
    RexBuilder rexBuilder = leftRel.getCluster().getRexBuilder();
    RexNode outJoinCond = null;

    int origLeftInputSize = leftRel.getRowType().getFieldCount();
    int origRightInputSize = rightRel.getRowType().getFieldCount();

    List<RexNode> newLeftFields = new ArrayList<RexNode>();
    List<String> newLeftFieldNames = new ArrayList<String>();

    List<RexNode> newRightFields = new ArrayList<RexNode>();
    List<String> newRightFieldNames = new ArrayList<String>();
    int leftKeyCount = leftJoinKeys.size();
    int i;

    for (i = 0; i < origLeftInputSize; i++) {
      final RelDataTypeField field = leftRel.getRowType().getFieldList().get(i);
      newLeftFields.add(rexBuilder.makeInputRef(field.getType(), i));
      newLeftFieldNames.add(field.getName());
    }

    for (i = 0; i < origRightInputSize; i++) {
      final RelDataTypeField field = rightRel.getRowType().getFieldList().get(i);
      newRightFields.add(rexBuilder.makeInputRef(field.getType(), i));
      newRightFieldNames.add(field.getName());
    }

    int newKeyCount = 0;
    List<Pair<Integer, Integer>> origColEqConds = new ArrayList<Pair<Integer, Integer>>();
    for (i = 0; i < leftKeyCount; i++) {
      RexNode leftKey = leftJoinKeys.get(i);
      RexNode rightKey = rightJoinKeys.get(i);

      if (leftKey instanceof RexInputRef && rightKey instanceof RexInputRef) {
        origColEqConds.add(
            Pair.of(((RexInputRef) leftKey).getIndex(), ((RexInputRef) rightKey).getIndex()));
      } else {
        newLeftFields.add(leftKey);
        newLeftFieldNames.add(null);
        newRightFields.add(rightKey);
        newRightFieldNames.add(null);
        newKeyCount++;
      }
    }

    for (i = 0; i < origColEqConds.size(); i++) {
      Pair<Integer, Integer> p = origColEqConds.get(i);
      RexNode leftKey = leftJoinKeys.get(i);
      RexNode rightKey = rightJoinKeys.get(i);
      leftKeys.add(p.left);
      rightKeys.add(p.right);
      RexNode cond =
          rexBuilder.makeCall(
              SqlStdOperatorTable.EQUALS,
              rexBuilder.makeInputRef(leftKey.getType(), systemColCount + p.left),
              rexBuilder.makeInputRef(
                  rightKey.getType(), systemColCount + origLeftInputSize + newKeyCount + p.right));
      if (outJoinCond == null) {
        outJoinCond = cond;
      } else {
        outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
      }
    }

    if (newKeyCount == 0) {
      return outJoinCond;
    }

    int newLeftOffset = systemColCount + origLeftInputSize;
    int newRightOffset = systemColCount + origLeftInputSize + origRightInputSize + newKeyCount;
    for (i = 0; i < newKeyCount; i++) {
      leftKeys.add(origLeftInputSize + i);
      rightKeys.add(origRightInputSize + i);
      RexNode cond =
          rexBuilder.makeCall(
              SqlStdOperatorTable.EQUALS,
              rexBuilder.makeInputRef(
                  newLeftFields.get(origLeftInputSize + i).getType(), newLeftOffset + i),
              rexBuilder.makeInputRef(
                  newRightFields.get(origRightInputSize + i).getType(), newRightOffset + i));
      if (outJoinCond == null) {
        outJoinCond = cond;
      } else {
        outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
      }
    }

    // added project if need to produce new keys than the original input
    // fields
    if (newKeyCount > 0) {
      leftRel =
          factory.createProject(
              leftRel, newLeftFields, SqlValidatorUtil.uniquify(newLeftFieldNames));
      rightRel =
          factory.createProject(
              rightRel, newRightFields, SqlValidatorUtil.uniquify(newRightFieldNames));
    }

    inputRels[0] = leftRel;
    inputRels[1] = rightRel;

    return outJoinCond;
  }
Ejemplo n.º 10
0
  protected RexNode convert(ExprNodeConstantDesc literal) throws CalciteSemanticException {
    RexBuilder rexBuilder = cluster.getRexBuilder();
    RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
    PrimitiveTypeInfo hiveType = (PrimitiveTypeInfo) literal.getTypeInfo();
    RelDataType calciteDataType = TypeConverter.convert(hiveType, dtFactory);

    PrimitiveCategory hiveTypeCategory = hiveType.getPrimitiveCategory();

    ConstantObjectInspector coi = literal.getWritableObjectInspector();
    Object value =
        ObjectInspectorUtils.copyToStandardJavaObject(coi.getWritableConstantValue(), coi);

    RexNode calciteLiteral = null;
    // TODO: Verify if we need to use ConstantObjectInspector to unwrap data
    switch (hiveTypeCategory) {
      case BOOLEAN:
        calciteLiteral = rexBuilder.makeLiteral(((Boolean) value).booleanValue());
        break;
      case BYTE:
        calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Byte) value), calciteDataType);
        break;
      case SHORT:
        calciteLiteral =
            rexBuilder.makeExactLiteral(new BigDecimal((Short) value), calciteDataType);
        break;
      case INT:
        calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Integer) value));
        break;
      case LONG:
        calciteLiteral = rexBuilder.makeBigintLiteral(new BigDecimal((Long) value));
        break;
        // TODO: is Decimal an exact numeric or approximate numeric?
      case DECIMAL:
        if (value instanceof HiveDecimal) {
          value = ((HiveDecimal) value).bigDecimalValue();
        } else if (value instanceof Decimal128) {
          value = ((Decimal128) value).toBigDecimal();
        }
        if (value == null) {
          // We have found an invalid decimal value while enforcing precision and
          // scale. Ideally,
          // we would replace it with null here, which is what Hive does. However,
          // we need to plumb
          // this thru up somehow, because otherwise having different expression
          // type in AST causes
          // the plan generation to fail after CBO, probably due to some residual
          // state in SA/QB.
          // For now, we will not run CBO in the presence of invalid decimal
          // literals.
          throw new CalciteSemanticException(
              "Expression " + literal.getExprString() + " is not a valid decimal",
              UnsupportedFeature.Invalid_decimal);
          // TODO: return createNullLiteral(literal);
        }
        BigDecimal bd = (BigDecimal) value;
        BigInteger unscaled = bd.unscaledValue();
        if (unscaled.compareTo(MIN_LONG_BI) >= 0 && unscaled.compareTo(MAX_LONG_BI) <= 0) {
          calciteLiteral = rexBuilder.makeExactLiteral(bd);
        } else {
          // CBO doesn't support unlimited precision decimals. In practice, this
          // will work...
          // An alternative would be to throw CboSemanticException and fall back
          // to no CBO.
          RelDataType relType =
              cluster
                  .getTypeFactory()
                  .createSqlType(SqlTypeName.DECIMAL, bd.scale(), unscaled.toString().length());
          calciteLiteral = rexBuilder.makeExactLiteral(bd, relType);
        }
        break;
      case FLOAT:
        calciteLiteral =
            rexBuilder.makeApproxLiteral(new BigDecimal((Float) value), calciteDataType);
        break;
      case DOUBLE:
        calciteLiteral =
            rexBuilder.makeApproxLiteral(new BigDecimal((Double) value), calciteDataType);
        break;
      case CHAR:
        if (value instanceof HiveChar) {
          value = ((HiveChar) value).getValue();
        }
        calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
        break;
      case VARCHAR:
        if (value instanceof HiveVarchar) {
          value = ((HiveVarchar) value).getValue();
        }
        calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
        break;
      case STRING:
        calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
        break;
      case DATE:
        Calendar cal = new GregorianCalendar();
        cal.setTime((Date) value);
        calciteLiteral = rexBuilder.makeDateLiteral(cal);
        break;
      case TIMESTAMP:
        Calendar c = null;
        if (value instanceof Calendar) {
          c = (Calendar) value;
        } else {
          c = Calendar.getInstance();
          c.setTimeInMillis(((Timestamp) value).getTime());
        }
        calciteLiteral = rexBuilder.makeTimestampLiteral(c, RelDataType.PRECISION_NOT_SPECIFIED);
        break;
      case INTERVAL_YEAR_MONTH:
        // Calcite year-month literal value is months as BigDecimal
        BigDecimal totalMonths =
            BigDecimal.valueOf(((HiveIntervalYearMonth) value).getTotalMonths());
        calciteLiteral =
            rexBuilder.makeIntervalLiteral(
                totalMonths,
                new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
        break;
      case INTERVAL_DAY_TIME:
        // Calcite day-time interval is millis value as BigDecimal
        // Seconds converted to millis
        BigDecimal secsValueBd =
            BigDecimal.valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000);
        // Nanos converted to millis
        BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getNanos(), 6);
        calciteLiteral =
            rexBuilder.makeIntervalLiteral(
                secsValueBd.add(nanosValueBd),
                new SqlIntervalQualifier(TimeUnit.DAY, TimeUnit.SECOND, new SqlParserPos(1, 1)));
        break;
      case VOID:
        calciteLiteral =
            cluster
                .getRexBuilder()
                .makeLiteral(null, cluster.getTypeFactory().createSqlType(SqlTypeName.NULL), true);
        break;
      case BINARY:
      case UNKNOWN:
      default:
        throw new RuntimeException("UnSupported Literal");
    }

    return calciteLiteral;
  }
  @Override
  public Prel visitProject(ProjectPrel project, Object unused) throws RelConversionException {

    // Apply the rule to the child
    RelNode originalInput = ((Prel) project.getInput(0)).accept(this, null);
    project = (ProjectPrel) project.copy(project.getTraitSet(), Lists.newArrayList(originalInput));

    List<RexNode> exprList = new ArrayList<>();

    List<RelDataTypeField> relDataTypes = new ArrayList();
    List<RelDataTypeField> origRelDataTypes = new ArrayList();
    int i = 0;
    final int lastColumnReferenced = PrelUtil.getLastUsedColumnReference(project.getProjects());

    if (lastColumnReferenced == -1) {
      return project;
    }

    final int lastRexInput = lastColumnReferenced + 1;
    RexVisitorComplexExprSplitter exprSplitter =
        new RexVisitorComplexExprSplitter(factory, funcReg, lastRexInput);

    for (RexNode rex : project.getChildExps()) {
      origRelDataTypes.add(project.getRowType().getFieldList().get(i));
      i++;
      exprList.add(rex.accept(exprSplitter));
    }
    List<RexNode> complexExprs = exprSplitter.getComplexExprs();

    if (complexExprs.size() == 1 && findTopComplexFunc(project.getChildExps()).size() == 1) {
      return project;
    }

    ProjectPrel childProject;

    List<RexNode> allExprs = new ArrayList();
    int exprIndex = 0;
    List<String> fieldNames = originalInput.getRowType().getFieldNames();
    for (int index = 0; index < lastRexInput; index++) {
      RexBuilder builder = new RexBuilder(factory);
      allExprs.add(
          builder.makeInputRef(new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));

      if (fieldNames.get(index).contains(StarColumnHelper.STAR_COLUMN)) {
        relDataTypes.add(
            new RelDataTypeFieldImpl(
                fieldNames.get(index), allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
      } else {
        relDataTypes.add(
            new RelDataTypeFieldImpl(
                "EXPR$" + exprIndex, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
        exprIndex++;
      }
    }
    RexNode currRexNode;
    int index = lastRexInput - 1;

    // if the projection expressions contained complex outputs, split them into their own individual
    // projects
    if (complexExprs.size() > 0) {
      while (complexExprs.size() > 0) {
        if (index >= lastRexInput) {
          allExprs.remove(allExprs.size() - 1);
          RexBuilder builder = new RexBuilder(factory);
          allExprs.add(
              builder.makeInputRef(
                  new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
        }
        index++;
        exprIndex++;

        currRexNode = complexExprs.remove(0);
        allExprs.add(currRexNode);
        relDataTypes.add(
            new RelDataTypeFieldImpl(
                "EXPR$" + exprIndex, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
        childProject =
            new ProjectPrel(
                project.getCluster(),
                project.getTraitSet(),
                originalInput,
                ImmutableList.copyOf(allExprs),
                new RelRecordType(relDataTypes));
        originalInput = childProject;
      }
      // copied from above, find a better way to do this
      allExprs.remove(allExprs.size() - 1);
      RexBuilder builder = new RexBuilder(factory);
      allExprs.add(
          builder.makeInputRef(new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index));
      relDataTypes.add(
          new RelDataTypeFieldImpl(
              "EXPR$" + index, allExprs.size(), factory.createSqlType(SqlTypeName.ANY)));
    }
    return (Prel)
        project.copy(
            project.getTraitSet(), originalInput, exprList, new RelRecordType(origRelDataTypes));
  }