private ObjectInspector solveOi(ObjectInspector arg) {
switch (arg.getCategory()) {
case PRIMITIVE:
// VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
// UNKNOWN
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getPrimitiveCategory());
case LIST:
return ObjectInspectorFactory.getStandardListObjectInspector(
solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
case MAP:
return ObjectInspectorFactory.getStandardMapObjectInspector(
solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) arg;
int size = soi.getAllStructFieldRefs().size();
ArrayList<String> fnl = new ArrayList<String>(size);
ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);
for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
fnl.add(sf.getFieldName());
foil.add(solveOi(sf.getFieldObjectInspector()));
}
return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
default:
return arg;
}
}
public static ConstantObjectInspector getConstantObjectInspector(
ObjectInspector oi, Object value) {
if (oi instanceof ConstantObjectInspector) {
return (ConstantObjectInspector) oi;
}
ObjectInspector writableOI = getStandardObjectInspector(oi, ObjectInspectorCopyOption.WRITABLE);
Object writableValue = ObjectInspectorConverters.getConverter(oi, writableOI).convert(value);
switch (writableOI.getCategory()) {
case PRIMITIVE:
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
return PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
poi.getTypeInfo(), writableValue);
case LIST:
ListObjectInspector loi = (ListObjectInspector) oi;
return ObjectInspectorFactory.getStandardConstantListObjectInspector(
getStandardObjectInspector(
loi.getListElementObjectInspector(), ObjectInspectorCopyOption.WRITABLE),
(List<?>) writableValue);
case MAP:
MapObjectInspector moi = (MapObjectInspector) oi;
return ObjectInspectorFactory.getStandardConstantMapObjectInspector(
getStandardObjectInspector(
moi.getMapKeyObjectInspector(), ObjectInspectorCopyOption.WRITABLE),
getStandardObjectInspector(
moi.getMapValueObjectInspector(), ObjectInspectorCopyOption.WRITABLE),
(Map<?, ?>) writableValue);
default:
throw new IllegalArgumentException(
writableOI.getCategory() + " not yet supported for constant OI");
}
}
/** Ensures that an ObjectInspector is Writable. */
public static ObjectInspector getWritableObjectInspector(ObjectInspector oi) {
// All non-primitive OIs are writable so we need only check this case.
if (oi.getCategory() == Category.PRIMITIVE) {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
if (!(poi instanceof AbstractPrimitiveWritableObjectInspector)) {
return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
poi.getTypeInfo());
}
}
return oi;
}
private static boolean isInstanceOfSettablePrimitiveOI(PrimitiveObjectInspector oi) {
switch (oi.getPrimitiveCategory()) {
case BOOLEAN:
return oi instanceof SettableBooleanObjectInspector;
case BYTE:
return oi instanceof SettableByteObjectInspector;
case SHORT:
return oi instanceof SettableShortObjectInspector;
case INT:
return oi instanceof SettableIntObjectInspector;
case LONG:
return oi instanceof SettableLongObjectInspector;
case FLOAT:
return oi instanceof SettableFloatObjectInspector;
case DOUBLE:
return oi instanceof SettableDoubleObjectInspector;
case STRING:
return oi instanceof WritableStringObjectInspector
|| oi instanceof JavaStringObjectInspector;
case CHAR:
return oi instanceof SettableHiveCharObjectInspector;
case VARCHAR:
return oi instanceof SettableHiveVarcharObjectInspector;
case DATE:
return oi instanceof SettableDateObjectInspector;
case TIMESTAMP:
return oi instanceof SettableTimestampObjectInspector;
case INTERVAL_YEAR_MONTH:
return oi instanceof SettableHiveIntervalYearMonthObjectInspector;
case INTERVAL_DAY_TIME:
return oi instanceof SettableHiveIntervalDayTimeObjectInspector;
case BINARY:
return oi instanceof SettableBinaryObjectInspector;
case DECIMAL:
return oi instanceof SettableHiveDecimalObjectInspector;
default:
throw new RuntimeException(
"Hive internal error inside isAssignableFromSettablePrimitiveOI "
+ oi.getTypeName()
+ " not supported yet.");
}
}
/** Compare two objects with their respective ObjectInspectors. */
public static int compare(
Object o1,
ObjectInspector oi1,
Object o2,
ObjectInspector oi2,
MapEqualComparer mapEqualComparer) {
if (oi1.getCategory() != oi2.getCategory()) {
return oi1.getCategory().compareTo(oi2.getCategory());
}
if (o1 == null) {
return o2 == null ? 0 : -1;
} else if (o2 == null) {
return 1;
}
switch (oi1.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi1 = ((PrimitiveObjectInspector) oi1);
PrimitiveObjectInspector poi2 = ((PrimitiveObjectInspector) oi2);
if (poi1.getPrimitiveCategory() != poi2.getPrimitiveCategory()) {
return poi1.getPrimitiveCategory().compareTo(poi2.getPrimitiveCategory());
}
switch (poi1.getPrimitiveCategory()) {
case VOID:
return 0;
case BOOLEAN:
{
int v1 = ((BooleanObjectInspector) poi1).get(o1) ? 1 : 0;
int v2 = ((BooleanObjectInspector) poi2).get(o2) ? 1 : 0;
return v1 - v2;
}
case BYTE:
{
int v1 = ((ByteObjectInspector) poi1).get(o1);
int v2 = ((ByteObjectInspector) poi2).get(o2);
return v1 - v2;
}
case SHORT:
{
int v1 = ((ShortObjectInspector) poi1).get(o1);
int v2 = ((ShortObjectInspector) poi2).get(o2);
return v1 - v2;
}
case INT:
{
int v1 = ((IntObjectInspector) poi1).get(o1);
int v2 = ((IntObjectInspector) poi2).get(o2);
return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
}
case LONG:
{
long v1 = ((LongObjectInspector) poi1).get(o1);
long v2 = ((LongObjectInspector) poi2).get(o2);
return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
}
case FLOAT:
{
float v1 = ((FloatObjectInspector) poi1).get(o1);
float v2 = ((FloatObjectInspector) poi2).get(o2);
return Float.compare(v1, v2);
}
case DOUBLE:
{
double v1 = ((DoubleObjectInspector) poi1).get(o1);
double v2 = ((DoubleObjectInspector) poi2).get(o2);
return Double.compare(v1, v2);
}
case STRING:
{
if (poi1.preferWritable() || poi2.preferWritable()) {
Text t1 = (Text) poi1.getPrimitiveWritableObject(o1);
Text t2 = (Text) poi2.getPrimitiveWritableObject(o2);
return t1 == null ? (t2 == null ? 0 : -1) : (t2 == null ? 1 : t1.compareTo(t2));
} else {
String s1 = (String) poi1.getPrimitiveJavaObject(o1);
String s2 = (String) poi2.getPrimitiveJavaObject(o2);
return s1 == null ? (s2 == null ? 0 : -1) : (s2 == null ? 1 : s1.compareTo(s2));
}
}
case CHAR:
{
HiveCharWritable t1 =
((HiveCharObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveCharWritable t2 =
((HiveCharObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
case VARCHAR:
{
HiveVarcharWritable t1 =
((HiveVarcharObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveVarcharWritable t2 =
((HiveVarcharObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
case BINARY:
{
BytesWritable bw1 = ((BinaryObjectInspector) poi1).getPrimitiveWritableObject(o1);
BytesWritable bw2 = ((BinaryObjectInspector) poi2).getPrimitiveWritableObject(o2);
return bw1.compareTo(bw2);
}
case DATE:
{
DateWritable d1 = ((DateObjectInspector) poi1).getPrimitiveWritableObject(o1);
DateWritable d2 = ((DateObjectInspector) poi2).getPrimitiveWritableObject(o2);
return d1.compareTo(d2);
}
case TIMESTAMP:
{
TimestampWritable t1 =
((TimestampObjectInspector) poi1).getPrimitiveWritableObject(o1);
TimestampWritable t2 =
((TimestampObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
case INTERVAL_YEAR_MONTH:
{
HiveIntervalYearMonthWritable i1 =
((HiveIntervalYearMonthObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveIntervalYearMonthWritable i2 =
((HiveIntervalYearMonthObjectInspector) poi2).getPrimitiveWritableObject(o2);
return i1.compareTo(i2);
}
case INTERVAL_DAY_TIME:
{
HiveIntervalDayTimeWritable i1 =
((HiveIntervalDayTimeObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveIntervalDayTimeWritable i2 =
((HiveIntervalDayTimeObjectInspector) poi2).getPrimitiveWritableObject(o2);
return i1.compareTo(i2);
}
case DECIMAL:
{
HiveDecimalWritable t1 =
((HiveDecimalObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveDecimalWritable t2 =
((HiveDecimalObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
default:
{
throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory());
}
}
}
case STRUCT:
{
StructObjectInspector soi1 = (StructObjectInspector) oi1;
StructObjectInspector soi2 = (StructObjectInspector) oi2;
List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
int minimum = Math.min(fields1.size(), fields2.size());
for (int i = 0; i < minimum; i++) {
int r =
compare(
soi1.getStructFieldData(o1, fields1.get(i)),
fields1.get(i).getFieldObjectInspector(),
soi2.getStructFieldData(o2, fields2.get(i)),
fields2.get(i).getFieldObjectInspector(),
mapEqualComparer);
if (r != 0) {
return r;
}
}
return fields1.size() - fields2.size();
}
case LIST:
{
ListObjectInspector loi1 = (ListObjectInspector) oi1;
ListObjectInspector loi2 = (ListObjectInspector) oi2;
int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2));
for (int i = 0; i < minimum; i++) {
int r =
compare(
loi1.getListElement(o1, i),
loi1.getListElementObjectInspector(),
loi2.getListElement(o2, i),
loi2.getListElementObjectInspector(),
mapEqualComparer);
if (r != 0) {
return r;
}
}
return loi1.getListLength(o1) - loi2.getListLength(o2);
}
case MAP:
{
if (mapEqualComparer == null) {
throw new RuntimeException("Compare on map type not supported!");
} else {
return mapEqualComparer.compare(
o1, (MapObjectInspector) oi1, o2, (MapObjectInspector) oi2);
}
}
case UNION:
{
UnionObjectInspector uoi1 = (UnionObjectInspector) oi1;
UnionObjectInspector uoi2 = (UnionObjectInspector) oi2;
byte tag1 = uoi1.getTag(o1);
byte tag2 = uoi2.getTag(o2);
if (tag1 != tag2) {
return tag1 - tag2;
}
return compare(
uoi1.getField(o1),
uoi1.getObjectInspectors().get(tag1),
uoi2.getField(o2),
uoi2.getObjectInspectors().get(tag2),
mapEqualComparer);
}
default:
throw new RuntimeException("Compare on unknown type: " + oi1.getCategory());
}
}
public static int hashCode(Object o, ObjectInspector objIns) {
if (o == null) {
return 0;
}
switch (objIns.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns);
switch (poi.getPrimitiveCategory()) {
case VOID:
return 0;
case BOOLEAN:
return ((BooleanObjectInspector) poi).get(o) ? 1 : 0;
case BYTE:
return ((ByteObjectInspector) poi).get(o);
case SHORT:
return ((ShortObjectInspector) poi).get(o);
case INT:
return ((IntObjectInspector) poi).get(o);
case LONG:
{
long a = ((LongObjectInspector) poi).get(o);
return (int) ((a >>> 32) ^ a);
}
case FLOAT:
return Float.floatToIntBits(((FloatObjectInspector) poi).get(o));
case DOUBLE:
{
// This hash function returns the same result as Double.hashCode()
// while DoubleWritable.hashCode returns a different result.
long a = Double.doubleToLongBits(((DoubleObjectInspector) poi).get(o));
return (int) ((a >>> 32) ^ a);
}
case STRING:
{
// This hash function returns the same result as String.hashCode() when
// all characters are ASCII, while Text.hashCode() always returns a
// different result.
Text t = ((StringObjectInspector) poi).getPrimitiveWritableObject(o);
int r = 0;
for (int i = 0; i < t.getLength(); i++) {
r = r * 31 + t.getBytes()[i];
}
return r;
}
case CHAR:
return ((HiveCharObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
case VARCHAR:
return ((HiveVarcharObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
case BINARY:
return ((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
case DATE:
return ((DateObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
case TIMESTAMP:
TimestampWritable t = ((TimestampObjectInspector) poi).getPrimitiveWritableObject(o);
return t.hashCode();
case INTERVAL_YEAR_MONTH:
HiveIntervalYearMonthWritable intervalYearMonth =
((HiveIntervalYearMonthObjectInspector) poi).getPrimitiveWritableObject(o);
return intervalYearMonth.hashCode();
case INTERVAL_DAY_TIME:
HiveIntervalDayTimeWritable intervalDayTime =
((HiveIntervalDayTimeObjectInspector) poi).getPrimitiveWritableObject(o);
return intervalDayTime.hashCode();
case DECIMAL:
return ((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
default:
{
throw new RuntimeException("Unknown type: " + poi.getPrimitiveCategory());
}
}
}
case LIST:
{
int r = 0;
ListObjectInspector listOI = (ListObjectInspector) objIns;
ObjectInspector elemOI = listOI.getListElementObjectInspector();
for (int ii = 0; ii < listOI.getListLength(o); ++ii) {
r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI);
}
return r;
}
case MAP:
{
int r = 0;
MapObjectInspector mapOI = (MapObjectInspector) objIns;
ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
ObjectInspector valueOI = mapOI.getMapValueObjectInspector();
Map<?, ?> map = mapOI.getMap(o);
for (Map.Entry<?, ?> entry : map.entrySet()) {
r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI);
}
return r;
}
case STRUCT:
int r = 0;
StructObjectInspector structOI = (StructObjectInspector) objIns;
List<? extends StructField> fields = structOI.getAllStructFieldRefs();
for (StructField field : fields) {
r =
31 * r
+ hashCode(
structOI.getStructFieldData(o, field), field.getFieldObjectInspector());
}
return r;
case UNION:
UnionObjectInspector uOI = (UnionObjectInspector) objIns;
byte tag = uOI.getTag(o);
return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag));
default:
throw new RuntimeException("Unknown type: " + objIns.getTypeName());
}
}
public static Object copyToStandardObject(
Object o, ObjectInspector oi, ObjectInspectorCopyOption objectInspectorOption) {
if (o == null) {
return null;
}
Object result = null;
switch (oi.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector loi = (PrimitiveObjectInspector) oi;
if (objectInspectorOption == ObjectInspectorCopyOption.DEFAULT) {
objectInspectorOption =
loi.preferWritable()
? ObjectInspectorCopyOption.WRITABLE
: ObjectInspectorCopyOption.JAVA;
}
switch (objectInspectorOption) {
case JAVA:
result = loi.getPrimitiveJavaObject(o);
if (loi.getPrimitiveCategory()
== PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP) {
result =
PrimitiveObjectInspectorFactory.javaTimestampObjectInspector.copyObject(result);
}
break;
case WRITABLE:
result = loi.getPrimitiveWritableObject(loi.copyObject(o));
break;
}
break;
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
int length = loi.getListLength(o);
ArrayList<Object> list = new ArrayList<Object>(length);
for (int i = 0; i < length; i++) {
list.add(
copyToStandardObject(
loi.getListElement(o, i),
loi.getListElementObjectInspector(),
objectInspectorOption));
}
result = list;
break;
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
HashMap<Object, Object> map = new HashMap<Object, Object>();
Map<? extends Object, ? extends Object> omap = moi.getMap(o);
for (Map.Entry<? extends Object, ? extends Object> entry : omap.entrySet()) {
map.put(
copyToStandardObject(
entry.getKey(), moi.getMapKeyObjectInspector(), objectInspectorOption),
copyToStandardObject(
entry.getValue(), moi.getMapValueObjectInspector(), objectInspectorOption));
}
result = map;
break;
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
ArrayList<Object> struct = new ArrayList<Object>(fields.size());
for (StructField f : fields) {
struct.add(
copyToStandardObject(
soi.getStructFieldData(o, f),
f.getFieldObjectInspector(),
objectInspectorOption));
}
result = struct;
break;
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
List<ObjectInspector> objectInspectors = uoi.getObjectInspectors();
Object object =
copyToStandardObject(
uoi.getField(o), objectInspectors.get(uoi.getTag(o)), objectInspectorOption);
result = object;
break;
}
default:
{
throw new RuntimeException("Unknown ObjectInspector category!");
}
}
return result;
}
public static ObjectInspector getStandardObjectInspector(
ObjectInspector oi, ObjectInspectorCopyOption objectInspectorOption) {
ObjectInspector result = null;
switch (oi.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
switch (objectInspectorOption) {
case DEFAULT:
{
if (poi.preferWritable()) {
result =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
poi.getTypeInfo());
} else {
result =
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getTypeInfo());
}
break;
}
case JAVA:
{
result =
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getTypeInfo());
break;
}
case WRITABLE:
result =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
poi.getTypeInfo());
break;
}
break;
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
result =
ObjectInspectorFactory.getStandardListObjectInspector(
getStandardObjectInspector(
loi.getListElementObjectInspector(), objectInspectorOption));
break;
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
result =
ObjectInspectorFactory.getStandardMapObjectInspector(
getStandardObjectInspector(moi.getMapKeyObjectInspector(), objectInspectorOption),
getStandardObjectInspector(
moi.getMapValueObjectInspector(), objectInspectorOption));
break;
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
List<String> fieldNames = new ArrayList<String>(fields.size());
List<ObjectInspector> fieldObjectInspectors =
new ArrayList<ObjectInspector>(fields.size());
for (StructField f : fields) {
fieldNames.add(f.getFieldName());
fieldObjectInspectors.add(
getStandardObjectInspector(f.getFieldObjectInspector(), objectInspectorOption));
}
result =
ObjectInspectorFactory.getStandardStructObjectInspector(
fieldNames, fieldObjectInspectors);
break;
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
for (ObjectInspector eoi : uoi.getObjectInspectors()) {
ois.add(getStandardObjectInspector(eoi, objectInspectorOption));
}
result = ObjectInspectorFactory.getStandardUnionObjectInspector(ois);
break;
}
default:
{
throw new RuntimeException("Unknown ObjectInspector category!");
}
}
return result;
}
