/**
* Get the precision of the operation involving two of the same types. Only meaningful for
* decimals, which override this.
*
* @param operator a string representing the operator, null means no operator, just a type merge
* @param leftType the left type
* @param rightType the left type
* @return the resultant precision
*/
private int getPrecision(
String operator, DataTypeDescriptor leftType, DataTypeDescriptor rightType) {
// Only meaningful for decimal
if (getStoredFormatIdFromTypeId() != StoredFormatIds.DECIMAL_TYPE_ID) {
return leftType.getPrecision();
}
long lscale = (long) leftType.getScale();
long rscale = (long) rightType.getScale();
long lprec = (long) leftType.getPrecision();
long rprec = (long) rightType.getPrecision();
long val;
/*
** Null means datatype merge. Take the maximum
** left of decimal digits plus the scale.
*/
if (operator == null) {
val = this.getScale(operator, leftType, rightType) + Math.max(lprec - lscale, rprec - rscale);
} else if (operator.equals(TypeCompiler.TIMES_OP)) {
val = lprec + rprec;
} else if (operator.equals(TypeCompiler.SUM_OP)) {
val = lprec - lscale + rprec - rscale + this.getScale(operator, leftType, rightType);
} else if (operator.equals(TypeCompiler.DIVIDE_OP)) {
val =
Math.min(
NumberDataValue.MAX_DECIMAL_PRECISION_SCALE,
this.getScale(operator, leftType, rightType) + lprec - lscale + rprec);
}
/*
** AVG, -, +
*/
else {
/*
** Take max scale and max left of decimal
** plus one.
*/
val =
this.getScale(operator, leftType, rightType)
+ Math.max(lprec - lscale, rprec - rscale)
+ 1;
if (val > Limits.DB2_MAX_DECIMAL_PRECISION_SCALE)
// then, like DB2, just set it to the max possible.
val = Limits.DB2_MAX_DECIMAL_PRECISION_SCALE;
}
if (val > Integer.MAX_VALUE) {
val = Integer.MAX_VALUE;
}
val = Math.min(NumberDataValue.MAX_DECIMAL_PRECISION_SCALE, val);
return (int) val;
}
/**
* Get the scale of the operation involving two of the same types. Since we don't really have a
* good way to pass the resultant scale and precision around at execution time, we will model that
* BigDecimal does by default. This is good in most cases, though we would probably like to use
* something more sophisticated for division.
*
* @param operator a string representing the operator, null means no operator, just a type merge
* @param leftType the left type
* @param rightType the left type
* @return the resultant precision
*/
private int getScale(String operator, DataTypeDescriptor leftType, DataTypeDescriptor rightType) {
// Only meaningful for decimal
if (getStoredFormatIdFromTypeId() != StoredFormatIds.DECIMAL_TYPE_ID) {
return leftType.getScale();
}
long val;
long lscale = (long) leftType.getScale();
long rscale = (long) rightType.getScale();
long lprec = (long) leftType.getPrecision();
long rprec = (long) rightType.getPrecision();
/*
** Retain greatest scale, take sum of left
** of decimal
*/
if (TypeCompiler.TIMES_OP.equals(operator)) {
val = lscale + rscale;
} else if (TypeCompiler.DIVIDE_OP.equals(operator)) {
/*
** Take max left scale + right precision - right scale + 1,
** or 4, whichever is biggest
*/
LanguageConnectionContext lcc =
(LanguageConnectionContext)
(ContextService.getContext(LanguageConnectionContext.CONTEXT_ID));
// Scale: 31 - left precision + left scale - right scale
val = Math.max(NumberDataValue.MAX_DECIMAL_PRECISION_SCALE - lprec + lscale - rscale, 0);
} else if (TypeCompiler.AVG_OP.equals(operator)) {
val = Math.max(Math.max(lscale, rscale), NumberDataValue.MIN_DECIMAL_DIVIDE_SCALE);
}
/*
** SUM, -, + all take max(lscale,rscale)
*/
else {
val = Math.max(lscale, rscale);
}
if (val > Integer.MAX_VALUE) {
val = Integer.MAX_VALUE;
}
val = Math.min(NumberDataValue.MAX_DECIMAL_PRECISION_SCALE, val);
return (int) val;
}
