private Writable createMap(final Object obj, final MapObjectInspector inspector)
throws SerDeException {
final Map<?, ?> sourceMap = inspector.getMap(obj);
final ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();
final ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
final List<ArrayWritable> array = new ArrayList<ArrayWritable>();
if (sourceMap != null) {
for (final Entry<?, ?> keyValue : sourceMap.entrySet()) {
final Writable key = createObject(keyValue.getKey(), keyInspector);
final Writable value = createObject(keyValue.getValue(), valueInspector);
if (key != null) {
Writable[] arr = new Writable[2];
arr[0] = key;
arr[1] = value;
array.add(new ArrayWritable(Writable.class, arr));
}
}
}
if (array.size() > 0) {
final ArrayWritable subArray =
new ArrayWritable(ArrayWritable.class, array.toArray(new ArrayWritable[array.size()]));
return new ArrayWritable(Writable.class, new Writable[] {subArray});
} else {
return null;
}
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length != 1) {
throw new UDFArgumentLengthException("The function MAP_VALUES only accepts 1 argument.");
} else if (!(arguments[0] instanceof MapObjectInspector)) {
throw new UDFArgumentTypeException(
0,
"\""
+ Category.MAP.toString().toLowerCase()
+ "\" is expected at function MAP_VALUES, "
+ "but \""
+ arguments[0].getTypeName()
+ "\" is found");
}
mapOI = (MapObjectInspector) arguments[0];
ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();
return ObjectInspectorFactory.getStandardListObjectInspector(mapValueOI);
}
/**
* Serializes a Hive map<> using a JSONObject.
*
* @param obj the object to serialize
* @param moi the object's inspector
* @return
*/
private JSONObject serializeMap(Object obj, MapObjectInspector moi) {
if (obj == null) {
return null;
}
JSONObject jo = new JSONObject();
Map m = moi.getMap(obj);
for (Object k : m.keySet()) {
try {
jo.put(
serializeField(k, moi.getMapKeyObjectInspector()).toString(),
serializeField(m.get(k), moi.getMapValueObjectInspector()));
} catch (JSONException ex) {
LOG.warn("Problem serializing map");
}
}
return jo;
}
private static Block serializeMap(
Type type, BlockBuilder builder, Object object, MapObjectInspector inspector) {
Map<?, ?> map = inspector.getMap(object);
if (map == null) {
requireNonNull(builder, "parent builder is null").appendNull();
return null;
}
List<Type> typeParameters = type.getTypeParameters();
checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter");
Type keyType = typeParameters.get(0);
Type valueType = typeParameters.get(1);
ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();
ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
BlockBuilder currentBuilder;
if (builder != null) {
currentBuilder = builder.beginBlockEntry();
} else {
currentBuilder =
new InterleavedBlockBuilder(typeParameters, new BlockBuilderStatus(), map.size());
}
for (Map.Entry<?, ?> entry : map.entrySet()) {
// Hive skips map entries with null keys
if (entry.getKey() != null) {
serializeObject(keyType, currentBuilder, entry.getKey(), keyInspector);
serializeObject(valueType, currentBuilder, entry.getValue(), valueInspector);
}
}
if (builder != null) {
builder.closeEntry();
return null;
} else {
Block resultBlock = currentBuilder.build();
return resultBlock;
}
}
/**
* True if Object passed is representing null object.
*
* @param o The object
* @param oi The ObjectInspector
* @return true if the object passed is representing NULL object false otherwise
*/
public static boolean hasAnyNullObject(Object o, ObjectInspector oi) {
switch (oi.getCategory()) {
case PRIMITIVE:
{
if (o == null) {
return true;
}
return false;
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
List<?> olist = loi.getList(o);
if (olist == null) {
return true;
} else {
// there are no elements in the list
if (olist.size() == 0) {
return false;
}
// if all the elements are representing null, then return true
for (int i = 0; i < olist.size(); i++) {
if (hasAnyNullObject(olist.get(i), listElementObjectInspector)) {
return true;
}
}
return false;
}
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
Map<?, ?> omap = moi.getMap(o);
if (omap == null) {
return true;
} else {
// there are no elements in the map
if (omap.entrySet().size() == 0) {
return false;
}
// if all the entries of map are representing null, then return true
for (Map.Entry<?, ?> entry : omap.entrySet()) {
if (hasAnyNullObject(entry.getKey(), mapKeyObjectInspector)
|| hasAnyNullObject(entry.getValue(), mapValueObjectInspector)) {
return true;
}
}
return false;
}
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> structFields = soi.getAllStructFieldRefs();
if (o == null) {
return true;
} else {
// there are no fields in the struct
if (structFields.size() == 0) {
return false;
}
// if any the fields of struct are representing null, then return true
for (int i = 0; i < structFields.size(); i++) {
if (hasAnyNullObject(
soi.getStructFieldData(o, structFields.get(i)),
structFields.get(i).getFieldObjectInspector())) {
return true;
}
}
return false;
}
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
if (o == null) {
return true;
} else {
// there are no elements in the union
if (uoi.getObjectInspectors().size() == 0) {
return false;
}
return hasAnyNullObject(uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
}
}
default:
throw new RuntimeException("Unknown type in ObjectInspector!");
}
}
static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {
switch (oi.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
if (o == null) {
sb.append(nullStr);
} else {
switch (poi.getPrimitiveCategory()) {
case BOOLEAN:
{
boolean b = ((BooleanObjectInspector) poi).get(o);
sb.append(b ? "true" : "false");
break;
}
case BYTE:
{
sb.append(((ByteObjectInspector) poi).get(o));
break;
}
case SHORT:
{
sb.append(((ShortObjectInspector) poi).get(o));
break;
}
case INT:
{
sb.append(((IntObjectInspector) poi).get(o));
break;
}
case LONG:
{
sb.append(((LongObjectInspector) poi).get(o));
break;
}
case FLOAT:
{
sb.append(((FloatObjectInspector) poi).get(o));
break;
}
case DOUBLE:
{
sb.append(((DoubleObjectInspector) poi).get(o));
break;
}
case STRING:
{
sb.append('"');
sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
sb.append('"');
break;
}
case CHAR:
{
sb.append('"');
sb.append(
escapeString(
((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
sb.append('"');
break;
}
case VARCHAR:
{
sb.append('"');
sb.append(
escapeString(
((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
sb.append('"');
break;
}
case DATE:
{
sb.append('"');
sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
sb.append('"');
break;
}
case TIMESTAMP:
{
sb.append('"');
sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
sb.append('"');
break;
}
case BINARY:
{
BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
Text txt = new Text();
txt.set(bw.getBytes(), 0, bw.getLength());
sb.append(txt.toString());
break;
}
case DECIMAL:
{
sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
break;
}
default:
throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
}
}
break;
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
List<?> olist = loi.getList(o);
if (olist == null) {
sb.append(nullStr);
} else {
sb.append(LBRACKET);
for (int i = 0; i < olist.size(); i++) {
if (i > 0) {
sb.append(COMMA);
}
buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
}
sb.append(RBRACKET);
}
break;
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
Map<?, ?> omap = moi.getMap(o);
if (omap == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
boolean first = true;
for (Object entry : omap.entrySet()) {
if (first) {
first = false;
} else {
sb.append(COMMA);
}
Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
sb.append(COLON);
buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
}
sb.append(RBRACE);
}
break;
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> structFields = soi.getAllStructFieldRefs();
if (o == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
for (int i = 0; i < structFields.size(); i++) {
if (i > 0) {
sb.append(COMMA);
}
sb.append(QUOTE);
sb.append(structFields.get(i).getFieldName());
sb.append(QUOTE);
sb.append(COLON);
buildJSONString(
sb,
soi.getStructFieldData(o, structFields.get(i)),
structFields.get(i).getFieldObjectInspector(),
JSON_NULL);
}
sb.append(RBRACE);
}
break;
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
if (o == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
sb.append(uoi.getTag(o));
sb.append(COLON);
buildJSONString(
sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
sb.append(RBRACE);
}
break;
}
default:
throw new RuntimeException("Unknown type in ObjectInspector!");
}
}
static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, boolean invert) {
// Is this field a null?
if (o == null) {
buffer.write((byte) 0, invert);
return;
}
// This field is not a null.
buffer.write((byte) 1, invert);
switch (oi.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
switch (poi.getPrimitiveCategory()) {
case VOID:
{
return;
}
case BOOLEAN:
{
boolean v = ((BooleanObjectInspector) poi).get(o);
buffer.write((byte) (v ? 2 : 1), invert);
return;
}
case BYTE:
{
ByteObjectInspector boi = (ByteObjectInspector) poi;
byte v = boi.get(o);
buffer.write((byte) (v ^ 0x80), invert);
return;
}
case SHORT:
{
ShortObjectInspector spoi = (ShortObjectInspector) poi;
short v = spoi.get(o);
buffer.write((byte) ((v >> 8) ^ 0x80), invert);
buffer.write((byte) v, invert);
return;
}
case INT:
{
IntObjectInspector ioi = (IntObjectInspector) poi;
int v = ioi.get(o);
buffer.write((byte) ((v >> 24) ^ 0x80), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case LONG:
{
LongObjectInspector loi = (LongObjectInspector) poi;
long v = loi.get(o);
buffer.write((byte) ((v >> 56) ^ 0x80), invert);
buffer.write((byte) (v >> 48), invert);
buffer.write((byte) (v >> 40), invert);
buffer.write((byte) (v >> 32), invert);
buffer.write((byte) (v >> 24), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case FLOAT:
{
FloatObjectInspector foi = (FloatObjectInspector) poi;
int v = Float.floatToIntBits(foi.get(o));
if ((v & (1 << 31)) != 0) {
// negative number, flip all bits
v = ~v;
} else {
// positive number, flip the first bit
v = v ^ (1 << 31);
}
buffer.write((byte) (v >> 24), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case DOUBLE:
{
DoubleObjectInspector doi = (DoubleObjectInspector) poi;
long v = Double.doubleToLongBits(doi.get(o));
if ((v & (1L << 63)) != 0) {
// negative number, flip all bits
v = ~v;
} else {
// positive number, flip the first bit
v = v ^ (1L << 63);
}
buffer.write((byte) (v >> 56), invert);
buffer.write((byte) (v >> 48), invert);
buffer.write((byte) (v >> 40), invert);
buffer.write((byte) (v >> 32), invert);
buffer.write((byte) (v >> 24), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case STRING:
{
StringObjectInspector soi = (StringObjectInspector) poi;
Text t = soi.getPrimitiveWritableObject(o);
serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
return;
}
case BINARY:
{
BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
BytesWritable ba = baoi.getPrimitiveWritableObject(o);
byte[] toSer = new byte[ba.getLength()];
System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
serializeBytes(buffer, toSer, ba.getLength(), invert);
return;
}
case DATE:
{
DateObjectInspector doi = (DateObjectInspector) poi;
long v = doi.getPrimitiveWritableObject(o).getTimeInSeconds();
buffer.write((byte) ((v >> 56) ^ 0x80), invert);
buffer.write((byte) (v >> 48), invert);
buffer.write((byte) (v >> 40), invert);
buffer.write((byte) (v >> 32), invert);
buffer.write((byte) (v >> 24), invert);
buffer.write((byte) (v >> 16), invert);
buffer.write((byte) (v >> 8), invert);
buffer.write((byte) v, invert);
return;
}
case TIMESTAMP:
{
TimestampObjectInspector toi = (TimestampObjectInspector) poi;
TimestampWritable t = toi.getPrimitiveWritableObject(o);
byte[] data = t.getBinarySortable();
for (int i = 0; i < data.length; i++) {
buffer.write(data[i], invert);
}
return;
}
default:
{
throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
}
}
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
ObjectInspector eoi = loi.getListElementObjectInspector();
// \1 followed by each element
int size = loi.getListLength(o);
for (int eid = 0; eid < size; eid++) {
buffer.write((byte) 1, invert);
serialize(buffer, loi.getListElement(o, eid), eoi, invert);
}
// and \0 to terminate
buffer.write((byte) 0, invert);
return;
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
ObjectInspector koi = moi.getMapKeyObjectInspector();
ObjectInspector voi = moi.getMapValueObjectInspector();
// \1 followed by each key and then each value
Map<?, ?> map = moi.getMap(o);
for (Map.Entry<?, ?> entry : map.entrySet()) {
buffer.write((byte) 1, invert);
serialize(buffer, entry.getKey(), koi, invert);
serialize(buffer, entry.getValue(), voi, invert);
}
// and \0 to terminate
buffer.write((byte) 0, invert);
return;
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int i = 0; i < fields.size(); i++) {
serialize(
buffer,
soi.getStructFieldData(o, fields.get(i)),
fields.get(i).getFieldObjectInspector(),
invert);
}
return;
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
byte tag = uoi.getTag(o);
buffer.write(tag, invert);
serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert);
return;
}
default:
{
throw new RuntimeException("Unrecognized type: " + oi.getCategory());
}
}
}