@Test
public void testDecimalPlusDecimalSameParams() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(5, 2)),
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(5, 2))
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(6, 2), oi.getTypeInfo());
}
private ObjectInspector solveOi(ObjectInspector arg) {
switch (arg.getCategory()) {
case PRIMITIVE:
// VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
// UNKNOWN
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getPrimitiveCategory());
case LIST:
return ObjectInspectorFactory.getStandardListObjectInspector(
solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
case MAP:
return ObjectInspectorFactory.getStandardMapObjectInspector(
solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) arg;
int size = soi.getAllStructFieldRefs().size();
ArrayList<String> fnl = new ArrayList<String>(size);
ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);
for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
fnl.add(sf.getFieldName());
foil.add(solveOi(sf.getFieldObjectInspector()));
}
return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
default:
return arg;
}
}
public static ConstantObjectInspector getConstantObjectInspector(
ObjectInspector oi, Object value) {
if (oi instanceof ConstantObjectInspector) {
return (ConstantObjectInspector) oi;
}
ObjectInspector writableOI = getStandardObjectInspector(oi, ObjectInspectorCopyOption.WRITABLE);
Object writableValue = ObjectInspectorConverters.getConverter(oi, writableOI).convert(value);
switch (writableOI.getCategory()) {
case PRIMITIVE:
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
return PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
poi.getTypeInfo(), writableValue);
case LIST:
ListObjectInspector loi = (ListObjectInspector) oi;
return ObjectInspectorFactory.getStandardConstantListObjectInspector(
getStandardObjectInspector(
loi.getListElementObjectInspector(), ObjectInspectorCopyOption.WRITABLE),
(List<?>) writableValue);
case MAP:
MapObjectInspector moi = (MapObjectInspector) oi;
return ObjectInspectorFactory.getStandardConstantMapObjectInspector(
getStandardObjectInspector(
moi.getMapKeyObjectInspector(), ObjectInspectorCopyOption.WRITABLE),
getStandardObjectInspector(
moi.getMapValueObjectInspector(), ObjectInspectorCopyOption.WRITABLE),
(Map<?, ?>) writableValue);
default:
throw new IllegalArgumentException(
writableOI.getCategory() + " not yet supported for constant OI");
}
}
/**
* Returns an ObjectInspector for a primitive Class. The Class can be a Hive Writable class, or a
* Java Primitive Class.
*
* <p>A runtimeException will be thrown if the class is not recognized as a primitive type by
* Hive.
*/
public static PrimitiveObjectInspector getPrimitiveObjectInspectorFromClass(Class<?> c) {
if (Writable.class.isAssignableFrom(c)) {
// It is a writable class
PrimitiveTypeEntry te =
PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveWritableClass(c);
if (te == null) {
throw new RuntimeException("Internal error: Cannot recognize " + c);
}
return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
te.primitiveCategory);
} else {
// It is a Java class
PrimitiveTypeEntry te = PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(c);
if (te == null) {
throw new RuntimeException("Internal error: Cannot recognize " + c);
}
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(te.primitiveCategory);
}
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
String database;
if (context != null) {
database = context.getJobConf().get("hive.current.database");
} else {
database = SessionState.get().getCurrentDatabase();
}
return PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
TypeInfoFactory.stringTypeInfo, new Text(database));
}
/** Ensures that an ObjectInspector is Writable. */
public static ObjectInspector getWritableObjectInspector(ObjectInspector oi) {
// All non-primitive OIs are writable so we need only check this case.
if (oi.getCategory() == Category.PRIMITIVE) {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
if (!(poi instanceof AbstractPrimitiveWritableObjectInspector)) {
return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
poi.getTypeInfo());
}
}
return oi;
}
@Test
public void testDecimalPlusDecimal() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Decimal
HiveDecimalWritable left = new HiveDecimalWritable(HiveDecimal.create("14.5"));
HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(3, 1)),
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(5, 2))
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(6, 2), oi.getTypeInfo());
HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args);
Assert.assertEquals(HiveDecimal.create("249.47"), res.getHiveDecimal());
}
private static ObjectInspector getObjectInspector(TypeInfo type) throws IOException {
switch (type.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
primitiveType.getPrimitiveCategory());
case MAP:
MapTypeInfo mapType = (MapTypeInfo) type;
MapObjectInspector mapInspector =
ObjectInspectorFactory.getStandardMapObjectInspector(
getObjectInspector(mapType.getMapKeyTypeInfo()),
getObjectInspector(mapType.getMapValueTypeInfo()));
return mapInspector;
case LIST:
ListTypeInfo listType = (ListTypeInfo) type;
ListObjectInspector listInspector =
ObjectInspectorFactory.getStandardListObjectInspector(
getObjectInspector(listType.getListElementTypeInfo()));
return listInspector;
case STRUCT:
StructTypeInfo structType = (StructTypeInfo) type;
List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();
List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
for (TypeInfo fieldType : fieldTypes) {
fieldInspectors.add(getObjectInspector(fieldType));
}
StructObjectInspector structInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(
structType.getAllStructFieldNames(), fieldInspectors);
return structInspector;
default:
throw new IOException("Unknown field schema type");
}
}
@Test
public void testDoulePlusDecimal() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Double
DoubleWritable left = new DoubleWritable(74.52);
HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(5, 2))
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.doubleTypeInfo, oi.getTypeInfo());
DoubleWritable res = (DoubleWritable) udf.evaluate(args);
Assert.assertEquals(new Double(309.49), new Double(res.get()));
}
@Test
public void testLongPlusDecimal() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Long
LongWritable left = new LongWritable(104);
HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableLongObjectInspector,
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(9, 4))
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(24, 4), oi.getTypeInfo());
HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args);
Assert.assertEquals(HiveDecimal.create("338.97"), res.getHiveDecimal());
}
@Override
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
assert (parameters.length == 1);
super.init(m, parameters);
result = new HiveDecimalWritable(HiveDecimal.ZERO);
inputOI = (PrimitiveObjectInspector) parameters[0];
// The output precision is 10 greater than the input which should cover at least
// 10b rows. The scale is the same as the input.
DecimalTypeInfo outputTypeInfo = null;
if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
int precision = Math.min(HiveDecimal.MAX_PRECISION, inputOI.precision() + 10);
outputTypeInfo = TypeInfoFactory.getDecimalTypeInfo(precision, inputOI.scale());
} else {
outputTypeInfo = (DecimalTypeInfo) inputOI.getTypeInfo();
}
ObjectInspector oi =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(outputTypeInfo);
outputOI =
(PrimitiveObjectInspector)
ObjectInspectorUtils.getStandardObjectInspector(oi, ObjectInspectorCopyOption.JAVA);
return oi;
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
converter = getConverter(arguments[0]);
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.STRING);
}
public static ObjectInspector getStandardObjectInspector(
ObjectInspector oi, ObjectInspectorCopyOption objectInspectorOption) {
ObjectInspector result = null;
switch (oi.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
switch (objectInspectorOption) {
case DEFAULT:
{
if (poi.preferWritable()) {
result =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
poi.getTypeInfo());
} else {
result =
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getTypeInfo());
}
break;
}
case JAVA:
{
result =
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getTypeInfo());
break;
}
case WRITABLE:
result =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
poi.getTypeInfo());
break;
}
break;
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
result =
ObjectInspectorFactory.getStandardListObjectInspector(
getStandardObjectInspector(
loi.getListElementObjectInspector(), objectInspectorOption));
break;
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
result =
ObjectInspectorFactory.getStandardMapObjectInspector(
getStandardObjectInspector(moi.getMapKeyObjectInspector(), objectInspectorOption),
getStandardObjectInspector(
moi.getMapValueObjectInspector(), objectInspectorOption));
break;
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
List<String> fieldNames = new ArrayList<String>(fields.size());
List<ObjectInspector> fieldObjectInspectors =
new ArrayList<ObjectInspector>(fields.size());
for (StructField f : fields) {
fieldNames.add(f.getFieldName());
fieldObjectInspectors.add(
getStandardObjectInspector(f.getFieldObjectInspector(), objectInspectorOption));
}
result =
ObjectInspectorFactory.getStandardStructObjectInspector(
fieldNames, fieldObjectInspectors);
break;
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
for (ObjectInspector eoi : uoi.getObjectInspectors()) {
ois.add(getStandardObjectInspector(eoi, objectInspectorOption));
}
result = ObjectInspectorFactory.getStandardUnionObjectInspector(ois);
break;
}
default:
{
throw new RuntimeException("Unknown ObjectInspector category!");
}
}
return result;
}
private MapOpCtx initObjectInspector(
Configuration hconf, MapOpCtx opCtx, StructObjectInspector tableRowOI) throws Exception {
PartitionDesc pd = opCtx.partDesc;
TableDesc td = pd.getTableDesc();
// Use table properties in case of unpartitioned tables,
// and the union of table properties and partition properties, with partition
// taking precedence, in the case of partitioned tables
Properties overlayedProps =
SerDeUtils.createOverlayedProperties(td.getProperties(), pd.getProperties());
Map<String, String> partSpec = pd.getPartSpec();
opCtx.tableName = String.valueOf(overlayedProps.getProperty("name"));
opCtx.partName = String.valueOf(partSpec);
opCtx.deserializer = pd.getDeserializer(hconf);
StructObjectInspector partRawRowObjectInspector;
if (Utilities.isInputFileFormatSelfDescribing(pd)) {
partRawRowObjectInspector = tableRowOI;
} else {
partRawRowObjectInspector = (StructObjectInspector) opCtx.deserializer.getObjectInspector();
}
opCtx.partTblObjectInspectorConverter =
ObjectInspectorConverters.getConverter(partRawRowObjectInspector, tableRowOI);
// Next check if this table has partitions and if so
// get the list of partition names as well as allocate
// the serdes for the partition columns
String pcols = overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS);
if (pcols != null && pcols.length() > 0) {
String[] partKeys = pcols.trim().split("/");
String pcolTypes =
overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES);
String[] partKeyTypes = pcolTypes.trim().split(":");
if (partKeys.length > partKeyTypes.length) {
throw new HiveException(
"Internal error : partKeys length, "
+ partKeys.length
+ " greater than partKeyTypes length, "
+ partKeyTypes.length);
}
List<String> partNames = new ArrayList<String>(partKeys.length);
Object[] partValues = new Object[partKeys.length];
List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>(partKeys.length);
for (int i = 0; i < partKeys.length; i++) {
String key = partKeys[i];
partNames.add(key);
ObjectInspector oi =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getPrimitiveTypeInfo(partKeyTypes[i]));
// Partitions do not exist for this table
if (partSpec == null) {
// for partitionless table, initialize partValue to null
partValues[i] = null;
} else {
partValues[i] =
ObjectInspectorConverters.getConverter(
PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi)
.convert(partSpec.get(key));
}
partObjectInspectors.add(oi);
}
opCtx.rowWithPart = new Object[] {null, partValues};
opCtx.partObjectInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors);
}
// The op may not be a TableScan for mapjoins
// Consider the query: select /*+MAPJOIN(a)*/ count(*) FROM T1 a JOIN T2 b ON a.key = b.key;
// In that case, it will be a Select, but the rowOI need not be amended
if (opCtx.op instanceof TableScanOperator) {
TableScanOperator tsOp = (TableScanOperator) opCtx.op;
TableScanDesc tsDesc = tsOp.getConf();
if (tsDesc != null && tsDesc.hasVirtualCols()) {
opCtx.vcs = tsDesc.getVirtualCols();
opCtx.vcValues = new Object[opCtx.vcs.size()];
opCtx.vcsObjectInspector = VirtualColumn.getVCSObjectInspector(opCtx.vcs);
if (opCtx.isPartitioned()) {
opCtx.rowWithPartAndVC = Arrays.copyOfRange(opCtx.rowWithPart, 0, 3);
} else {
opCtx.rowWithPartAndVC = new Object[2];
}
}
}
if (!opCtx.hasVC() && !opCtx.isPartitioned()) {
opCtx.rowObjectInspector = tableRowOI;
return opCtx;
}
List<StructObjectInspector> inspectors = new ArrayList<StructObjectInspector>();
inspectors.add(tableRowOI);
if (opCtx.isPartitioned()) {
inspectors.add(opCtx.partObjectInspector);
}
if (opCtx.hasVC()) {
inspectors.add(opCtx.vcsObjectInspector);
}
opCtx.rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(inspectors);
return opCtx;
}
/**
* Returns the lazy binary object inspector that can be used to inspect an lazy binary object of
* that typeInfo
*
* <p>For primitive types, we use the standard writable object inspector.
*/
public static ObjectInspector getLazyBinaryObjectInspectorFromTypeInfo(TypeInfo typeInfo) {
ObjectInspector result = cachedLazyBinaryObjectInspector.get(typeInfo);
if (result == null) {
switch (typeInfo.getCategory()) {
case PRIMITIVE:
{
result =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
((PrimitiveTypeInfo) typeInfo));
break;
}
case LIST:
{
ObjectInspector elementObjectInspector =
getLazyBinaryObjectInspectorFromTypeInfo(
((ListTypeInfo) typeInfo).getListElementTypeInfo());
result =
LazyBinaryObjectInspectorFactory.getLazyBinaryListObjectInspector(
elementObjectInspector);
break;
}
case MAP:
{
MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
ObjectInspector keyObjectInspector =
getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo());
ObjectInspector valueObjectInspector =
getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo());
result =
LazyBinaryObjectInspectorFactory.getLazyBinaryMapObjectInspector(
keyObjectInspector, valueObjectInspector);
break;
}
case STRUCT:
{
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
List<String> fieldNames = structTypeInfo.getAllStructFieldNames();
List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
List<ObjectInspector> fieldObjectInspectors =
new ArrayList<ObjectInspector>(fieldTypeInfos.size());
for (int i = 0; i < fieldTypeInfos.size(); i++) {
fieldObjectInspectors.add(
getLazyBinaryObjectInspectorFromTypeInfo(fieldTypeInfos.get(i)));
}
result =
LazyBinaryObjectInspectorFactory.getLazyBinaryStructObjectInspector(
fieldNames, fieldObjectInspectors);
break;
}
case UNION:
{
UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
final List<TypeInfo> fieldTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
List<ObjectInspector> fieldObjectInspectors =
new ArrayList<ObjectInspector>(fieldTypeInfos.size());
for (int i = 0; i < fieldTypeInfos.size(); i++) {
fieldObjectInspectors.add(
getLazyBinaryObjectInspectorFromTypeInfo(fieldTypeInfos.get(i)));
}
result =
LazyBinaryObjectInspectorFactory.getLazyBinaryUnionObjectInspector(
fieldObjectInspectors);
break;
}
default:
{
result = null;
}
}
ObjectInspector prev = cachedLazyBinaryObjectInspector.putIfAbsent(typeInfo, result);
if (prev != null) {
result = prev;
}
}
return result;
}