public static PrimitiveObjectInspector asDoubleCompatibleOI(@Nonnull final ObjectInspector argOI)
throws UDFArgumentTypeException {
if (argOI.getCategory() != Category.PRIMITIVE) {
throw new UDFArgumentTypeException(
0,
"Only primitive type arguments are accepted but " + argOI.getTypeName() + " is passed.");
}
final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI;
switch (oi.getPrimitiveCategory()) {
case BYTE:
case SHORT:
case INT:
case LONG:
case FLOAT:
case DOUBLE:
case STRING:
case TIMESTAMP:
break;
default:
throw new UDFArgumentTypeException(
0,
"Only numeric or string type arguments are accepted but "
+ argOI.getTypeName()
+ " is passed.");
}
return oi;
}
@Override
public Writable serialize(Object obj, ObjectInspector oi) throws SerDeException {
if (oi.getCategory() != Category.STRUCT) {
throw new VoltSerdeException(
getClass().toString()
+ " can only serialize struct types, but we got: "
+ oi.getTypeName());
}
VoltRecord vr = new VoltRecord(m_voltConf.getTableName());
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> structFields = soi.getAllStructFieldRefs();
List<Object> fieldValues = soi.getStructFieldsDataAsList(obj);
final int size = m_oig.getColumnTypes().size();
for (int i = 0; i < size; ++i) {
ObjectInspector fieldOI = structFields.get(i).getFieldObjectInspector();
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) fieldOI;
Object fieldValue = poi.getPrimitiveJavaObject(fieldValues.get(i));
if (poi.getTypeInfo().equals(TypeInfoFactory.timestampTypeInfo)) {
fieldValue = fieldValue != null ? new Date(((Timestamp) fieldValue).getTime()) : null;
}
vr.add(fieldValue);
}
return vr;
}
private ObjectInspector solveOi(ObjectInspector arg) {
switch (arg.getCategory()) {
case PRIMITIVE:
// VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
// UNKNOWN
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getPrimitiveCategory());
case LIST:
return ObjectInspectorFactory.getStandardListObjectInspector(
solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
case MAP:
return ObjectInspectorFactory.getStandardMapObjectInspector(
solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) arg;
int size = soi.getAllStructFieldRefs().size();
ArrayList<String> fnl = new ArrayList<String>(size);
ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);
for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
fnl.add(sf.getFieldName());
foil.add(solveOi(sf.getFieldObjectInspector()));
}
return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
default:
return arg;
}
}
private void stringifyObject(StringBuilder buffer, Object obj, ObjectInspector inspector)
throws IOException {
if (inspector instanceof StructObjectInspector) {
buffer.append("{ ");
StructObjectInspector soi = (StructObjectInspector) inspector;
boolean isFirst = true;
for (StructField field : soi.getAllStructFieldRefs()) {
if (isFirst) {
isFirst = false;
} else {
buffer.append(", ");
}
buffer.append(field.getFieldName());
buffer.append(": ");
stringifyObject(
buffer, soi.getStructFieldData(obj, field), field.getFieldObjectInspector());
}
buffer.append(" }");
} else if (inspector instanceof PrimitiveObjectInspector) {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
buffer.append(poi.getPrimitiveJavaObject(obj).toString());
} else {
buffer.append("*unknown*");
}
}
public static LazySimpleSerDe getKeyValueLineSerde(
@Nonnull final PrimitiveObjectInspector keyOI,
@Nonnull final PrimitiveObjectInspector valueOI)
throws SerDeException {
LazySimpleSerDe serde = new LazySimpleSerDe();
Configuration conf = new Configuration();
Properties tbl = new Properties();
tbl.setProperty("columns", "key,value");
tbl.setProperty("columns.types", keyOI.getTypeName() + "," + valueOI.getTypeName());
serde.initialize(conf, tbl);
return serde;
}
/**
* Serializes a field. Since we have nested structures, it may be called recursively for instance
* when defining a list<struct<>>
*
* @param obj Object holding the fields' content
* @param oi The field's objec inspector
* @return the serialized object
*/
Object serializeField(Object obj, ObjectInspector oi) {
if (obj == null) {
return null;
}
Object result = null;
switch (oi.getCategory()) {
case PRIMITIVE:
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
switch (poi.getPrimitiveCategory()) {
case VOID:
result = null;
break;
case BOOLEAN:
result = (((BooleanObjectInspector) poi).get(obj) ? Boolean.TRUE : Boolean.FALSE);
break;
case BYTE:
result = (((ShortObjectInspector) poi).get(obj));
break;
case DOUBLE:
result = (((DoubleObjectInspector) poi).get(obj));
break;
case FLOAT:
result = (((FloatObjectInspector) poi).get(obj));
break;
case INT:
result = (((IntObjectInspector) poi).get(obj));
break;
case LONG:
result = (((LongObjectInspector) poi).get(obj));
break;
case SHORT:
result = (((ShortObjectInspector) poi).get(obj));
break;
case STRING:
result = (((StringObjectInspector) poi).getPrimitiveJavaObject(obj));
break;
case UNKNOWN:
throw new RuntimeException("Unknown primitive");
}
break;
case MAP:
result = serializeMap(obj, (MapObjectInspector) oi);
break;
case LIST:
result = serializeList(obj, (ListObjectInspector) oi);
break;
case STRUCT:
result = serializeStruct(obj, (StructObjectInspector) oi, null);
break;
}
return result;
}
public static void obtainBinaryConverter(
ObjectInspector[] arguments, int i, PrimitiveCategory[] inputTypes, Converter[] converters)
throws UDFArgumentTypeException {
PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
PrimitiveCategory inputType = inOi.getPrimitiveCategory();
Converter converter =
ObjectInspectorConverters.getConverter(
arguments[i], PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
converters[i] = converter;
inputTypes[i] = inputType;
}
@Test
public void testDecimalPlusDecimalSameParams() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(5, 2)),
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(5, 2))
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(6, 2), oi.getTypeInfo());
}
@Test
public void testFloatPlusFloat() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Float
FloatWritable f1 = new FloatWritable(4.5f);
FloatWritable f2 = new FloatWritable(0.0f);
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableFloatObjectInspector,
PrimitiveObjectInspectorFactory.writableFloatObjectInspector
};
DeferredObject[] args = {
new DeferredJavaObject(f1), new DeferredJavaObject(f2),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(oi.getTypeInfo(), TypeInfoFactory.floatTypeInfo);
FloatWritable res = (FloatWritable) udf.evaluate(args);
Assert.assertEquals(new Float(4.5), new Float(res.get()));
}
@Test
public void testDatePlusIntervalYearMonth() throws Exception {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
DateWritable left = new DateWritable(Date.valueOf("2001-06-15"));
HiveIntervalYearMonthWritable right =
new HiveIntervalYearMonthWritable(HiveIntervalYearMonth.valueOf("2-8"));
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableDateObjectInspector,
PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.dateTypeInfo, oi.getTypeInfo());
DateWritable res = (DateWritable) udf.evaluate(args);
Assert.assertEquals(Date.valueOf("2004-02-15"), res.get());
}
@Test
public void testBytePlusShort() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Byte
ByteWritable left = new ByteWritable((byte) 4);
ShortWritable right = new ShortWritable((short) 6);
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableByteObjectInspector,
PrimitiveObjectInspectorFactory.writableShortObjectInspector
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(oi.getTypeInfo(), TypeInfoFactory.shortTypeInfo);
ShortWritable res = (ShortWritable) udf.evaluate(args);
Assert.assertEquals(10, res.get());
}
@Test
public void testDoublePlusLong() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Int
DoubleWritable left = new DoubleWritable(4.5);
LongWritable right = new LongWritable(10);
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
PrimitiveObjectInspectorFactory.writableLongObjectInspector
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.doubleTypeInfo, oi.getTypeInfo());
DoubleWritable res = (DoubleWritable) udf.evaluate(args);
Assert.assertEquals(new Double(14.5), new Double(res.get()));
}
@Test
public void testTimestampPlusIntervalDayTime() throws Exception {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
TimestampWritable left = new TimestampWritable(Timestamp.valueOf("2001-01-01 00:00:00"));
HiveIntervalDayTimeWritable right =
new HiveIntervalDayTimeWritable(HiveIntervalDayTime.valueOf("1 2:3:4.567"));
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableTimestampObjectInspector,
PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo());
TimestampWritable res = (TimestampWritable) udf.evaluate(args);
Assert.assertEquals(Timestamp.valueOf("2001-01-02 2:3:4.567"), res.getTimestamp());
}
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info)
throws SemanticException {
final ObjectInspector[] inspectors = info.getParameterObjectInspectors();
if (inspectors.length != 1) {
throw new UDFArgumentException("One argument expected");
}
if (inspectors[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
throw new UDFArgumentTypeException(
0, "Primitive argument expected, but " + inspectors[0].getTypeName() + " was recieved");
}
final PrimitiveObjectInspector inspector = (PrimitiveObjectInspector) inspectors[0];
if (inspector.getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BINARY) {
throw new UDFArgumentTypeException(
0,
"Binary argument expected, but "
+ inspector.getPrimitiveCategory().name()
+ " was received");
}
return createEvaluator();
}
@Test
public void testLongPlusDecimal() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Long
LongWritable left = new LongWritable(104);
HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableLongObjectInspector,
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(9, 4))
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(24, 4), oi.getTypeInfo());
HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args);
Assert.assertEquals(HiveDecimal.create("338.97"), res.getHiveDecimal());
}
@Test
public void testDoulePlusDecimal() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Double
DoubleWritable left = new DoubleWritable(74.52);
HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(5, 2))
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.doubleTypeInfo, oi.getTypeInfo());
DoubleWritable res = (DoubleWritable) udf.evaluate(args);
Assert.assertEquals(new Double(309.49), new Double(res.get()));
}
@Test
public void testVarcharPlusInt() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Short
HiveVarcharWritable left = new HiveVarcharWritable();
left.set("123");
IntWritable right = new IntWritable(456);
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector,
PrimitiveObjectInspectorFactory.writableIntObjectInspector
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(oi.getTypeInfo(), TypeInfoFactory.doubleTypeInfo);
DoubleWritable res = (DoubleWritable) udf.evaluate(args);
Assert.assertEquals(new Double(579.0), new Double(res.get()));
}
@Test
public void testTimestampPlusIntervalYearMonth() throws Exception {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
TimestampWritable left =
new TimestampWritable(Timestamp.valueOf("2001-11-15 01:02:03.123456789"));
HiveIntervalYearMonthWritable right =
new HiveIntervalYearMonthWritable(HiveIntervalYearMonth.valueOf("2-2"));
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.writableTimestampObjectInspector,
PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo());
TimestampWritable res = (TimestampWritable) udf.evaluate(args);
Assert.assertEquals(Timestamp.valueOf("2004-01-15 01:02:03.123456789"), res.getTimestamp());
}
@Test
public void testDecimalPlusDecimal() throws HiveException {
GenericUDFOPPlus udf = new GenericUDFOPPlus();
// Decimal
HiveDecimalWritable left = new HiveDecimalWritable(HiveDecimal.create("14.5"));
HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
ObjectInspector[] inputOIs = {
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(3, 1)),
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getDecimalTypeInfo(5, 2))
};
DeferredObject[] args = {
new DeferredJavaObject(left), new DeferredJavaObject(right),
};
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(6, 2), oi.getTypeInfo());
HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args);
Assert.assertEquals(HiveDecimal.create("249.47"), res.getHiveDecimal());
}
private byte[] getConstantVal(Object writable, PrimitiveObjectInspector poi, boolean isKeyBinary)
throws IOException {
if (!isKeyBinary) {
// Key is stored in text format. Get bytes representation of constant also of
// text format.
byte[] startRow;
ByteStream.Output serializeStream = new ByteStream.Output();
LazyUtils.writePrimitiveUTF8(serializeStream, writable, poi, false, (byte) 0, null);
startRow = new byte[serializeStream.getLength()];
System.arraycopy(serializeStream.getData(), 0, startRow, 0, serializeStream.getLength());
return startRow;
}
PrimitiveCategory pc = poi.getPrimitiveCategory();
switch (poi.getPrimitiveCategory()) {
case INT:
return Bytes.toBytes(((IntWritable) writable).get());
case BOOLEAN:
return Bytes.toBytes(((BooleanWritable) writable).get());
case LONG:
return Bytes.toBytes(((LongWritable) writable).get());
case FLOAT:
return Bytes.toBytes(((FloatWritable) writable).get());
case DOUBLE:
return Bytes.toBytes(((DoubleWritable) writable).get());
case SHORT:
return Bytes.toBytes(((ShortWritable) writable).get());
case STRING:
return Bytes.toBytes(((Text) writable).toString());
case BYTE:
return Bytes.toBytes(((ByteWritable) writable).get());
default:
throw new IOException("Type not supported " + pc);
}
}
/** Create a lazy binary primitive class given the type name. */
public static LazyBinaryPrimitive<?, ?> createLazyBinaryPrimitiveClass(
PrimitiveObjectInspector oi) {
PrimitiveCategory p = oi.getPrimitiveCategory();
switch (p) {
case BOOLEAN:
return new LazyBinaryBoolean((WritableBooleanObjectInspector) oi);
case BYTE:
return new LazyBinaryByte((WritableByteObjectInspector) oi);
case SHORT:
return new LazyBinaryShort((WritableShortObjectInspector) oi);
case INT:
return new LazyBinaryInteger((WritableIntObjectInspector) oi);
case LONG:
return new LazyBinaryLong((WritableLongObjectInspector) oi);
case FLOAT:
return new LazyBinaryFloat((WritableFloatObjectInspector) oi);
case DOUBLE:
return new LazyBinaryDouble((WritableDoubleObjectInspector) oi);
case STRING:
return new LazyBinaryString((WritableStringObjectInspector) oi);
case CHAR:
return new LazyBinaryHiveChar((WritableHiveCharObjectInspector) oi);
case VARCHAR:
return new LazyBinaryHiveVarchar((WritableHiveVarcharObjectInspector) oi);
case VOID: // for NULL
return new LazyBinaryVoid((WritableVoidObjectInspector) oi);
case DATE:
return new LazyBinaryDate((WritableDateObjectInspector) oi);
case TIMESTAMP:
return new LazyBinaryTimestamp((WritableTimestampObjectInspector) oi);
case BINARY:
return new LazyBinaryBinary((WritableBinaryObjectInspector) oi);
case DECIMAL:
return new LazyBinaryHiveDecimal((WritableHiveDecimalObjectInspector) oi);
default:
throw new RuntimeException("Internal error: no LazyBinaryObject for " + p);
}
}
static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, boolean invert) {
// Is this field a null?
if (o == null) {
buffer.write((byte) 0, invert);
return;
}
// This field is not a null.
buffer.write((byte) 1, invert);
switch (oi.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
switch (poi.getPrimitiveCategory()) {
case VOID:
{
return;
}
case BOOLEAN:
{
boolean v = ((BooleanObjectInspector) poi).get(o);
buffer.write((byte) (v ? 2 : 1), invert);
return;
}
case BYTE:
{
ByteObjectInspector boi = (ByteObjectInspector) poi;
byte v = boi.get(o);
buffer.write((byte) (v ^ 0x80), invert);
return;
}
case SHORT:
{
ShortObjectInspector spoi = (ShortObjectInspector) poi;
short v = spoi.get(o);
buffer.write((byte) ((v >> 8) ^ 0x80), invert);
buffer.write((byte) v, invert);
return;
}
case INT:
{
IntObjectInspector ioi = (IntObjectInspector) poi;
int v = ioi.get(o);
buffer.write((byte) ((v >> 24) ^ 0x80), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case LONG:
{
LongObjectInspector loi = (LongObjectInspector) poi;
long v = loi.get(o);
buffer.write((byte) ((v >> 56) ^ 0x80), invert);
buffer.write((byte) (v >> 48), invert);
buffer.write((byte) (v >> 40), invert);
buffer.write((byte) (v >> 32), invert);
buffer.write((byte) (v >> 24), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case FLOAT:
{
FloatObjectInspector foi = (FloatObjectInspector) poi;
int v = Float.floatToIntBits(foi.get(o));
if ((v & (1 << 31)) != 0) {
// negative number, flip all bits
v = ~v;
} else {
// positive number, flip the first bit
v = v ^ (1 << 31);
}
buffer.write((byte) (v >> 24), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case DOUBLE:
{
DoubleObjectInspector doi = (DoubleObjectInspector) poi;
long v = Double.doubleToLongBits(doi.get(o));
if ((v & (1L << 63)) != 0) {
// negative number, flip all bits
v = ~v;
} else {
// positive number, flip the first bit
v = v ^ (1L << 63);
}
buffer.write((byte) (v >> 56), invert);
buffer.write((byte) (v >> 48), invert);
buffer.write((byte) (v >> 40), invert);
buffer.write((byte) (v >> 32), invert);
buffer.write((byte) (v >> 24), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case STRING:
{
StringObjectInspector soi = (StringObjectInspector) poi;
Text t = soi.getPrimitiveWritableObject(o);
serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
return;
}
case BINARY:
{
BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
BytesWritable ba = baoi.getPrimitiveWritableObject(o);
byte[] toSer = new byte[ba.getLength()];
System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
serializeBytes(buffer, toSer, ba.getLength(), invert);
return;
}
case DATE:
{
DateObjectInspector doi = (DateObjectInspector) poi;
long v = doi.getPrimitiveWritableObject(o).getTimeInSeconds();
buffer.write((byte) ((v >> 56) ^ 0x80), invert);
buffer.write((byte) (v >> 48), invert);
buffer.write((byte) (v >> 40), invert);
buffer.write((byte) (v >> 32), invert);
buffer.write((byte) (v >> 24), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case TIMESTAMP:
{
TimestampObjectInspector toi = (TimestampObjectInspector) poi;
TimestampWritable t = toi.getPrimitiveWritableObject(o);
byte[] data = t.getBinarySortable();
for (int i = 0; i < data.length; i++) {
buffer.write(data[i], invert);
}
return;
}
default:
{
throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
}
}
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
ObjectInspector eoi = loi.getListElementObjectInspector();
// \1 followed by each element
int size = loi.getListLength(o);
for (int eid = 0; eid < size; eid++) {
buffer.write((byte) 1, invert);
serialize(buffer, loi.getListElement(o, eid), eoi, invert);
}
// and \0 to terminate
buffer.write((byte) 0, invert);
return;
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
ObjectInspector koi = moi.getMapKeyObjectInspector();
ObjectInspector voi = moi.getMapValueObjectInspector();
// \1 followed by each key and then each value
Map<?, ?> map = moi.getMap(o);
for (Map.Entry<?, ?> entry : map.entrySet()) {
buffer.write((byte) 1, invert);
serialize(buffer, entry.getKey(), koi, invert);
serialize(buffer, entry.getValue(), voi, invert);
}
// and \0 to terminate
buffer.write((byte) 0, invert);
return;
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int i = 0; i < fields.size(); i++) {
serialize(
buffer,
soi.getStructFieldData(o, fields.get(i)),
fields.get(i).getFieldObjectInspector(),
invert);
}
return;
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
byte tag = uoi.getTag(o);
buffer.write(tag, invert);
serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert);
return;
}
default:
{
throw new RuntimeException("Unrecognized type: " + oi.getCategory());
}
}
}
private static void serializePrimitive(
Type type, BlockBuilder builder, Object object, PrimitiveObjectInspector inspector) {
requireNonNull(builder, "parent builder is null");
if (object == null) {
builder.appendNull();
return;
}
switch (inspector.getPrimitiveCategory()) {
case BOOLEAN:
BooleanType.BOOLEAN.writeBoolean(builder, ((BooleanObjectInspector) inspector).get(object));
return;
case BYTE:
TinyintType.TINYINT.writeLong(builder, ((ByteObjectInspector) inspector).get(object));
return;
case SHORT:
SmallintType.SMALLINT.writeLong(builder, ((ShortObjectInspector) inspector).get(object));
return;
case INT:
IntegerType.INTEGER.writeLong(builder, ((IntObjectInspector) inspector).get(object));
return;
case LONG:
BigintType.BIGINT.writeLong(builder, ((LongObjectInspector) inspector).get(object));
return;
case FLOAT:
DoubleType.DOUBLE.writeDouble(builder, ((FloatObjectInspector) inspector).get(object));
return;
case DOUBLE:
DoubleType.DOUBLE.writeDouble(builder, ((DoubleObjectInspector) inspector).get(object));
return;
case STRING:
type.writeSlice(
builder,
Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object)));
return;
case VARCHAR:
type.writeSlice(
builder,
Slices.utf8Slice(
((HiveVarcharObjectInspector) inspector)
.getPrimitiveJavaObject(object)
.getValue()));
return;
case CHAR:
CharType charType = checkType(type, CharType.class, "type");
HiveChar hiveChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(object);
type.writeSlice(
builder,
trimSpacesAndTruncateToLength(
Slices.utf8Slice(hiveChar.getValue()), charType.getLength()));
return;
case DATE:
DateType.DATE.writeLong(builder, formatDateAsLong(object, (DateObjectInspector) inspector));
return;
case TIMESTAMP:
TimestampType.TIMESTAMP.writeLong(
builder, formatTimestampAsLong(object, (TimestampObjectInspector) inspector));
return;
case BINARY:
VARBINARY.writeSlice(
builder,
Slices.wrappedBuffer(
((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object)));
return;
case DECIMAL:
DecimalType decimalType = checkType(type, DecimalType.class, "type");
HiveDecimalWritable hiveDecimal =
((HiveDecimalObjectInspector) inspector).getPrimitiveWritableObject(object);
if (decimalType.isShort()) {
decimalType.writeLong(
builder, DecimalUtils.getShortDecimalValue(hiveDecimal, decimalType.getScale()));
} else {
decimalType.writeSlice(
builder, DecimalUtils.getLongDecimalValue(hiveDecimal, decimalType.getScale()));
}
return;
}
throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory());
}
private Writable createPrimitive(final Object obj, final PrimitiveObjectInspector inspector)
throws SerDeException {
if (obj == null) {
return null;
}
switch (inspector.getPrimitiveCategory()) {
case VOID:
return null;
case BOOLEAN:
return new BooleanWritable(
((BooleanObjectInspector) inspector).get(obj) ? Boolean.TRUE : Boolean.FALSE);
case BYTE:
return new ByteWritable(((ByteObjectInspector) inspector).get(obj));
case DOUBLE:
return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj));
case FLOAT:
return new FloatWritable(((FloatObjectInspector) inspector).get(obj));
case INT:
return new IntWritable(((IntObjectInspector) inspector).get(obj));
case LONG:
return new LongWritable(((LongObjectInspector) inspector).get(obj));
case SHORT:
return new ShortWritable(((ShortObjectInspector) inspector).get(obj));
case STRING:
String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(obj);
try {
return new BytesWritable(v.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
throw new SerDeException("Failed to encode string in UTF-8", e);
}
case DECIMAL:
HiveDecimal hd = (HiveDecimal) inspector.getPrimitiveJavaObject(obj);
DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
int prec = decTypeInfo.precision();
int scale = decTypeInfo.scale();
byte[] src = hd.setScale(scale).unscaledValue().toByteArray();
// Estimated number of bytes needed.
int bytes = PRECISION_TO_BYTE_COUNT[prec - 1];
if (bytes == src.length) {
// No padding needed.
return new BytesWritable(src);
}
byte[] tgt = new byte[bytes];
if (hd.signum() == -1) {
// For negative number, initializing bits to 1
for (int i = 0; i < bytes; i++) {
tgt[i] |= 0xFF;
}
}
System.arraycopy(
src, 0, tgt, bytes - src.length, src.length); // Padding leading zeroes/ones.
return new BytesWritable(tgt);
case TIMESTAMP:
return new TimestampWritable(
((TimestampObjectInspector) inspector).getPrimitiveJavaObject(obj));
case CHAR:
String strippedValue =
((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(obj).getStrippedValue();
return new BytesWritable(Binary.fromString(strippedValue).getBytes());
case VARCHAR:
String value =
((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(obj).getValue();
return new BytesWritable(Binary.fromString(value).getBytes());
case BINARY:
return new BytesWritable(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(obj));
default:
throw new SerDeException("Unknown primitive : " + inspector.getPrimitiveCategory());
}
}
public static VectorColumnAssign buildObjectAssign(
VectorizedRowBatch outputBatch, int outColIndex, ObjectInspector objInspector)
throws HiveException {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector;
VectorColumnAssign outVCA = null;
ColumnVector destCol = outputBatch.cols[outColIndex];
if (destCol instanceof LongColumnVector) {
switch (poi.getPrimitiveCategory()) {
case BOOLEAN:
outVCA =
new VectorLongColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
BooleanWritable bw = (BooleanWritable) val;
assignLong(bw.get() ? 1 : 0, destIndex);
}
}
}.init(outputBatch, (LongColumnVector) destCol);
break;
case BYTE:
outVCA =
new VectorLongColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
ByteWritable bw = (ByteWritable) val;
assignLong(bw.get(), destIndex);
}
}
}.init(outputBatch, (LongColumnVector) destCol);
break;
case SHORT:
outVCA =
new VectorLongColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
ShortWritable bw = (ShortWritable) val;
assignLong(bw.get(), destIndex);
}
}
}.init(outputBatch, (LongColumnVector) destCol);
break;
case INT:
outVCA =
new VectorLongColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
IntWritable bw = (IntWritable) val;
assignLong(bw.get(), destIndex);
}
}
}.init(outputBatch, (LongColumnVector) destCol);
break;
case LONG:
outVCA =
new VectorLongColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
LongWritable bw = (LongWritable) val;
assignLong(bw.get(), destIndex);
}
}
}.init(outputBatch, (LongColumnVector) destCol);
break;
case TIMESTAMP:
outVCA =
new VectorLongColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
TimestampWritable bw = (TimestampWritable) val;
Timestamp t = bw.getTimestamp();
assignLong(TimestampUtils.getTimeNanoSec(t), destIndex);
}
}
}.init(outputBatch, (LongColumnVector) destCol);
break;
case DATE:
outVCA =
new VectorLongColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
DateWritable bw = (DateWritable) val;
assignLong(bw.getDays(), destIndex);
}
}
}.init(outputBatch, (LongColumnVector) destCol);
break;
default:
throw new HiveException(
"Incompatible Long vector column and primitive category "
+ poi.getPrimitiveCategory());
}
} else if (destCol instanceof DoubleColumnVector) {
switch (poi.getPrimitiveCategory()) {
case DOUBLE:
outVCA =
new VectorDoubleColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
DoubleWritable bw = (DoubleWritable) val;
assignDouble(bw.get(), destIndex);
}
}
}.init(outputBatch, (DoubleColumnVector) destCol);
break;
case FLOAT:
outVCA =
new VectorDoubleColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
FloatWritable bw = (FloatWritable) val;
assignDouble(bw.get(), destIndex);
}
}
}.init(outputBatch, (DoubleColumnVector) destCol);
break;
default:
throw new HiveException(
"Incompatible Double vector column and primitive category "
+ poi.getPrimitiveCategory());
}
} else if (destCol instanceof BytesColumnVector) {
switch (poi.getPrimitiveCategory()) {
case STRING:
outVCA =
new VectorBytesColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
Text bw = (Text) val;
byte[] bytes = bw.getBytes();
assignBytes(bytes, 0, bytes.length, destIndex);
}
}
}.init(outputBatch, (BytesColumnVector) destCol);
break;
default:
throw new HiveException(
"Incompatible Bytes vector column and primitive category "
+ poi.getPrimitiveCategory());
}
} else if (destCol instanceof DecimalColumnVector) {
switch (poi.getPrimitiveCategory()) {
case DECIMAL:
outVCA =
new VectorDecimalColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
} else {
HiveDecimalWritable hdw = (HiveDecimalWritable) val;
assignDecimal(hdw, destIndex);
}
}
}.init(outputBatch, (DecimalColumnVector) destCol);
break;
default:
throw new HiveException(
"Incompatible Decimal vector column and primitive category "
+ poi.getPrimitiveCategory());
}
} else {
throw new HiveException("Unknown vector column type " + destCol.getClass().getName());
}
return outVCA;
}
static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {
switch (oi.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
if (o == null) {
sb.append(nullStr);
} else {
switch (poi.getPrimitiveCategory()) {
case BOOLEAN:
{
boolean b = ((BooleanObjectInspector) poi).get(o);
sb.append(b ? "true" : "false");
break;
}
case BYTE:
{
sb.append(((ByteObjectInspector) poi).get(o));
break;
}
case SHORT:
{
sb.append(((ShortObjectInspector) poi).get(o));
break;
}
case INT:
{
sb.append(((IntObjectInspector) poi).get(o));
break;
}
case LONG:
{
sb.append(((LongObjectInspector) poi).get(o));
break;
}
case FLOAT:
{
sb.append(((FloatObjectInspector) poi).get(o));
break;
}
case DOUBLE:
{
sb.append(((DoubleObjectInspector) poi).get(o));
break;
}
case STRING:
{
sb.append('"');
sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
sb.append('"');
break;
}
case CHAR:
{
sb.append('"');
sb.append(
escapeString(
((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
sb.append('"');
break;
}
case VARCHAR:
{
sb.append('"');
sb.append(
escapeString(
((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
sb.append('"');
break;
}
case DATE:
{
sb.append('"');
sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
sb.append('"');
break;
}
case TIMESTAMP:
{
sb.append('"');
sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
sb.append('"');
break;
}
case BINARY:
{
BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
Text txt = new Text();
txt.set(bw.getBytes(), 0, bw.getLength());
sb.append(txt.toString());
break;
}
case DECIMAL:
{
sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
break;
}
default:
throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
}
}
break;
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
List<?> olist = loi.getList(o);
if (olist == null) {
sb.append(nullStr);
} else {
sb.append(LBRACKET);
for (int i = 0; i < olist.size(); i++) {
if (i > 0) {
sb.append(COMMA);
}
buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
}
sb.append(RBRACKET);
}
break;
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
Map<?, ?> omap = moi.getMap(o);
if (omap == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
boolean first = true;
for (Object entry : omap.entrySet()) {
if (first) {
first = false;
} else {
sb.append(COMMA);
}
Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
sb.append(COLON);
buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
}
sb.append(RBRACE);
}
break;
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> structFields = soi.getAllStructFieldRefs();
if (o == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
for (int i = 0; i < structFields.size(); i++) {
if (i > 0) {
sb.append(COMMA);
}
sb.append(QUOTE);
sb.append(structFields.get(i).getFieldName());
sb.append(QUOTE);
sb.append(COLON);
buildJSONString(
sb,
soi.getStructFieldData(o, structFields.get(i)),
structFields.get(i).getFieldObjectInspector(),
JSON_NULL);
}
sb.append(RBRACE);
}
break;
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
if (o == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
sb.append(uoi.getTag(o));
sb.append(COLON);
buildJSONString(
sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
sb.append(RBRACE);
}
break;
}
default:
throw new RuntimeException("Unknown type in ObjectInspector!");
}
}
