/** * Get the precision of the operation involving two of the same types. Only meaningful for * decimals, which override this. * * @param operator a string representing the operator, null means no operator, just a type merge * @param leftType the left type * @param rightType the left type * @return the resultant precision */ private int getPrecision( String operator, DataTypeDescriptor leftType, DataTypeDescriptor rightType) { // Only meaningful for decimal if (getStoredFormatIdFromTypeId() != StoredFormatIds.DECIMAL_TYPE_ID) { return leftType.getPrecision(); } long lscale = (long) leftType.getScale(); long rscale = (long) rightType.getScale(); long lprec = (long) leftType.getPrecision(); long rprec = (long) rightType.getPrecision(); long val; /* ** Null means datatype merge. Take the maximum ** left of decimal digits plus the scale. */ if (operator == null) { val = this.getScale(operator, leftType, rightType) + Math.max(lprec - lscale, rprec - rscale); } else if (operator.equals(TypeCompiler.TIMES_OP)) { val = lprec + rprec; } else if (operator.equals(TypeCompiler.SUM_OP)) { val = lprec - lscale + rprec - rscale + this.getScale(operator, leftType, rightType); } else if (operator.equals(TypeCompiler.DIVIDE_OP)) { val = Math.min( NumberDataValue.MAX_DECIMAL_PRECISION_SCALE, this.getScale(operator, leftType, rightType) + lprec - lscale + rprec); } /* ** AVG, -, + */ else { /* ** Take max scale and max left of decimal ** plus one. */ val = this.getScale(operator, leftType, rightType) + Math.max(lprec - lscale, rprec - rscale) + 1; if (val > Limits.DB2_MAX_DECIMAL_PRECISION_SCALE) // then, like DB2, just set it to the max possible. val = Limits.DB2_MAX_DECIMAL_PRECISION_SCALE; } if (val > Integer.MAX_VALUE) { val = Integer.MAX_VALUE; } val = Math.min(NumberDataValue.MAX_DECIMAL_PRECISION_SCALE, val); return (int) val; }
/** * Get the scale of the operation involving two of the same types. Since we don't really have a * good way to pass the resultant scale and precision around at execution time, we will model that * BigDecimal does by default. This is good in most cases, though we would probably like to use * something more sophisticated for division. * * @param operator a string representing the operator, null means no operator, just a type merge * @param leftType the left type * @param rightType the left type * @return the resultant precision */ private int getScale(String operator, DataTypeDescriptor leftType, DataTypeDescriptor rightType) { // Only meaningful for decimal if (getStoredFormatIdFromTypeId() != StoredFormatIds.DECIMAL_TYPE_ID) { return leftType.getScale(); } long val; long lscale = (long) leftType.getScale(); long rscale = (long) rightType.getScale(); long lprec = (long) leftType.getPrecision(); long rprec = (long) rightType.getPrecision(); /* ** Retain greatest scale, take sum of left ** of decimal */ if (TypeCompiler.TIMES_OP.equals(operator)) { val = lscale + rscale; } else if (TypeCompiler.DIVIDE_OP.equals(operator)) { /* ** Take max left scale + right precision - right scale + 1, ** or 4, whichever is biggest */ LanguageConnectionContext lcc = (LanguageConnectionContext) (ContextService.getContext(LanguageConnectionContext.CONTEXT_ID)); // Scale: 31 - left precision + left scale - right scale val = Math.max(NumberDataValue.MAX_DECIMAL_PRECISION_SCALE - lprec + lscale - rscale, 0); } else if (TypeCompiler.AVG_OP.equals(operator)) { val = Math.max(Math.max(lscale, rscale), NumberDataValue.MIN_DECIMAL_DIVIDE_SCALE); } /* ** SUM, -, + all take max(lscale,rscale) */ else { val = Math.max(lscale, rscale); } if (val > Integer.MAX_VALUE) { val = Integer.MAX_VALUE; } val = Math.min(NumberDataValue.MAX_DECIMAL_PRECISION_SCALE, val); return (int) val; }