private ObjectInspector solveOi(ObjectInspector arg) { switch (arg.getCategory()) { case PRIMITIVE: // VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL, // UNKNOWN PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg; return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( poi.getPrimitiveCategory()); case LIST: return ObjectInspectorFactory.getStandardListObjectInspector( solveOi(((ListObjectInspector) arg).getListElementObjectInspector())); case MAP: return ObjectInspectorFactory.getStandardMapObjectInspector( solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()), solveOi(((MapObjectInspector) arg).getMapValueObjectInspector())); case STRUCT: StructObjectInspector soi = (StructObjectInspector) arg; int size = soi.getAllStructFieldRefs().size(); ArrayList<String> fnl = new ArrayList<String>(size); ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size); for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) { fnl.add(sf.getFieldName()); foil.add(solveOi(sf.getFieldObjectInspector())); } return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil); default: return arg; } }
private static boolean isInstanceOfSettablePrimitiveOI(PrimitiveObjectInspector oi) { switch (oi.getPrimitiveCategory()) { case BOOLEAN: return oi instanceof SettableBooleanObjectInspector; case BYTE: return oi instanceof SettableByteObjectInspector; case SHORT: return oi instanceof SettableShortObjectInspector; case INT: return oi instanceof SettableIntObjectInspector; case LONG: return oi instanceof SettableLongObjectInspector; case FLOAT: return oi instanceof SettableFloatObjectInspector; case DOUBLE: return oi instanceof SettableDoubleObjectInspector; case STRING: return oi instanceof WritableStringObjectInspector || oi instanceof JavaStringObjectInspector; case CHAR: return oi instanceof SettableHiveCharObjectInspector; case VARCHAR: return oi instanceof SettableHiveVarcharObjectInspector; case DATE: return oi instanceof SettableDateObjectInspector; case TIMESTAMP: return oi instanceof SettableTimestampObjectInspector; case INTERVAL_YEAR_MONTH: return oi instanceof SettableHiveIntervalYearMonthObjectInspector; case INTERVAL_DAY_TIME: return oi instanceof SettableHiveIntervalDayTimeObjectInspector; case BINARY: return oi instanceof SettableBinaryObjectInspector; case DECIMAL: return oi instanceof SettableHiveDecimalObjectInspector; default: throw new RuntimeException( "Hive internal error inside isAssignableFromSettablePrimitiveOI " + oi.getTypeName() + " not supported yet."); } }
/** Compare two objects with their respective ObjectInspectors. */ public static int compare( Object o1, ObjectInspector oi1, Object o2, ObjectInspector oi2, MapEqualComparer mapEqualComparer) { if (oi1.getCategory() != oi2.getCategory()) { return oi1.getCategory().compareTo(oi2.getCategory()); } if (o1 == null) { return o2 == null ? 0 : -1; } else if (o2 == null) { return 1; } switch (oi1.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi1 = ((PrimitiveObjectInspector) oi1); PrimitiveObjectInspector poi2 = ((PrimitiveObjectInspector) oi2); if (poi1.getPrimitiveCategory() != poi2.getPrimitiveCategory()) { return poi1.getPrimitiveCategory().compareTo(poi2.getPrimitiveCategory()); } switch (poi1.getPrimitiveCategory()) { case VOID: return 0; case BOOLEAN: { int v1 = ((BooleanObjectInspector) poi1).get(o1) ? 1 : 0; int v2 = ((BooleanObjectInspector) poi2).get(o2) ? 1 : 0; return v1 - v2; } case BYTE: { int v1 = ((ByteObjectInspector) poi1).get(o1); int v2 = ((ByteObjectInspector) poi2).get(o2); return v1 - v2; } case SHORT: { int v1 = ((ShortObjectInspector) poi1).get(o1); int v2 = ((ShortObjectInspector) poi2).get(o2); return v1 - v2; } case INT: { int v1 = ((IntObjectInspector) poi1).get(o1); int v2 = ((IntObjectInspector) poi2).get(o2); return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); } case LONG: { long v1 = ((LongObjectInspector) poi1).get(o1); long v2 = ((LongObjectInspector) poi2).get(o2); return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); } case FLOAT: { float v1 = ((FloatObjectInspector) poi1).get(o1); float v2 = ((FloatObjectInspector) poi2).get(o2); return Float.compare(v1, v2); } case DOUBLE: { double v1 = ((DoubleObjectInspector) poi1).get(o1); double v2 = ((DoubleObjectInspector) poi2).get(o2); return Double.compare(v1, v2); } case STRING: { if (poi1.preferWritable() || poi2.preferWritable()) { Text t1 = (Text) poi1.getPrimitiveWritableObject(o1); Text t2 = (Text) poi2.getPrimitiveWritableObject(o2); return t1 == null ? (t2 == null ? 0 : -1) : (t2 == null ? 1 : t1.compareTo(t2)); } else { String s1 = (String) poi1.getPrimitiveJavaObject(o1); String s2 = (String) poi2.getPrimitiveJavaObject(o2); return s1 == null ? (s2 == null ? 0 : -1) : (s2 == null ? 1 : s1.compareTo(s2)); } } case CHAR: { HiveCharWritable t1 = ((HiveCharObjectInspector) poi1).getPrimitiveWritableObject(o1); HiveCharWritable t2 = ((HiveCharObjectInspector) poi2).getPrimitiveWritableObject(o2); return t1.compareTo(t2); } case VARCHAR: { HiveVarcharWritable t1 = ((HiveVarcharObjectInspector) poi1).getPrimitiveWritableObject(o1); HiveVarcharWritable t2 = ((HiveVarcharObjectInspector) poi2).getPrimitiveWritableObject(o2); return t1.compareTo(t2); } case BINARY: { BytesWritable bw1 = ((BinaryObjectInspector) poi1).getPrimitiveWritableObject(o1); BytesWritable bw2 = ((BinaryObjectInspector) poi2).getPrimitiveWritableObject(o2); return bw1.compareTo(bw2); } case DATE: { DateWritable d1 = ((DateObjectInspector) poi1).getPrimitiveWritableObject(o1); DateWritable d2 = ((DateObjectInspector) poi2).getPrimitiveWritableObject(o2); return d1.compareTo(d2); } case TIMESTAMP: { TimestampWritable t1 = ((TimestampObjectInspector) poi1).getPrimitiveWritableObject(o1); TimestampWritable t2 = ((TimestampObjectInspector) poi2).getPrimitiveWritableObject(o2); return t1.compareTo(t2); } case INTERVAL_YEAR_MONTH: { HiveIntervalYearMonthWritable i1 = ((HiveIntervalYearMonthObjectInspector) poi1).getPrimitiveWritableObject(o1); HiveIntervalYearMonthWritable i2 = ((HiveIntervalYearMonthObjectInspector) poi2).getPrimitiveWritableObject(o2); return i1.compareTo(i2); } case INTERVAL_DAY_TIME: { HiveIntervalDayTimeWritable i1 = ((HiveIntervalDayTimeObjectInspector) poi1).getPrimitiveWritableObject(o1); HiveIntervalDayTimeWritable i2 = ((HiveIntervalDayTimeObjectInspector) poi2).getPrimitiveWritableObject(o2); return i1.compareTo(i2); } case DECIMAL: { HiveDecimalWritable t1 = ((HiveDecimalObjectInspector) poi1).getPrimitiveWritableObject(o1); HiveDecimalWritable t2 = ((HiveDecimalObjectInspector) poi2).getPrimitiveWritableObject(o2); return t1.compareTo(t2); } default: { throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory()); } } } case STRUCT: { StructObjectInspector soi1 = (StructObjectInspector) oi1; StructObjectInspector soi2 = (StructObjectInspector) oi2; List<? extends StructField> fields1 = soi1.getAllStructFieldRefs(); List<? extends StructField> fields2 = soi2.getAllStructFieldRefs(); int minimum = Math.min(fields1.size(), fields2.size()); for (int i = 0; i < minimum; i++) { int r = compare( soi1.getStructFieldData(o1, fields1.get(i)), fields1.get(i).getFieldObjectInspector(), soi2.getStructFieldData(o2, fields2.get(i)), fields2.get(i).getFieldObjectInspector(), mapEqualComparer); if (r != 0) { return r; } } return fields1.size() - fields2.size(); } case LIST: { ListObjectInspector loi1 = (ListObjectInspector) oi1; ListObjectInspector loi2 = (ListObjectInspector) oi2; int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2)); for (int i = 0; i < minimum; i++) { int r = compare( loi1.getListElement(o1, i), loi1.getListElementObjectInspector(), loi2.getListElement(o2, i), loi2.getListElementObjectInspector(), mapEqualComparer); if (r != 0) { return r; } } return loi1.getListLength(o1) - loi2.getListLength(o2); } case MAP: { if (mapEqualComparer == null) { throw new RuntimeException("Compare on map type not supported!"); } else { return mapEqualComparer.compare( o1, (MapObjectInspector) oi1, o2, (MapObjectInspector) oi2); } } case UNION: { UnionObjectInspector uoi1 = (UnionObjectInspector) oi1; UnionObjectInspector uoi2 = (UnionObjectInspector) oi2; byte tag1 = uoi1.getTag(o1); byte tag2 = uoi2.getTag(o2); if (tag1 != tag2) { return tag1 - tag2; } return compare( uoi1.getField(o1), uoi1.getObjectInspectors().get(tag1), uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2), mapEqualComparer); } default: throw new RuntimeException("Compare on unknown type: " + oi1.getCategory()); } }
public static int hashCode(Object o, ObjectInspector objIns) { if (o == null) { return 0; } switch (objIns.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns); switch (poi.getPrimitiveCategory()) { case VOID: return 0; case BOOLEAN: return ((BooleanObjectInspector) poi).get(o) ? 1 : 0; case BYTE: return ((ByteObjectInspector) poi).get(o); case SHORT: return ((ShortObjectInspector) poi).get(o); case INT: return ((IntObjectInspector) poi).get(o); case LONG: { long a = ((LongObjectInspector) poi).get(o); return (int) ((a >>> 32) ^ a); } case FLOAT: return Float.floatToIntBits(((FloatObjectInspector) poi).get(o)); case DOUBLE: { // This hash function returns the same result as Double.hashCode() // while DoubleWritable.hashCode returns a different result. long a = Double.doubleToLongBits(((DoubleObjectInspector) poi).get(o)); return (int) ((a >>> 32) ^ a); } case STRING: { // This hash function returns the same result as String.hashCode() when // all characters are ASCII, while Text.hashCode() always returns a // different result. Text t = ((StringObjectInspector) poi).getPrimitiveWritableObject(o); int r = 0; for (int i = 0; i < t.getLength(); i++) { r = r * 31 + t.getBytes()[i]; } return r; } case CHAR: return ((HiveCharObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); case VARCHAR: return ((HiveVarcharObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); case BINARY: return ((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); case DATE: return ((DateObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); case TIMESTAMP: TimestampWritable t = ((TimestampObjectInspector) poi).getPrimitiveWritableObject(o); return t.hashCode(); case INTERVAL_YEAR_MONTH: HiveIntervalYearMonthWritable intervalYearMonth = ((HiveIntervalYearMonthObjectInspector) poi).getPrimitiveWritableObject(o); return intervalYearMonth.hashCode(); case INTERVAL_DAY_TIME: HiveIntervalDayTimeWritable intervalDayTime = ((HiveIntervalDayTimeObjectInspector) poi).getPrimitiveWritableObject(o); return intervalDayTime.hashCode(); case DECIMAL: return ((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); default: { throw new RuntimeException("Unknown type: " + poi.getPrimitiveCategory()); } } } case LIST: { int r = 0; ListObjectInspector listOI = (ListObjectInspector) objIns; ObjectInspector elemOI = listOI.getListElementObjectInspector(); for (int ii = 0; ii < listOI.getListLength(o); ++ii) { r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI); } return r; } case MAP: { int r = 0; MapObjectInspector mapOI = (MapObjectInspector) objIns; ObjectInspector keyOI = mapOI.getMapKeyObjectInspector(); ObjectInspector valueOI = mapOI.getMapValueObjectInspector(); Map<?, ?> map = mapOI.getMap(o); for (Map.Entry<?, ?> entry : map.entrySet()) { r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI); } return r; } case STRUCT: int r = 0; StructObjectInspector structOI = (StructObjectInspector) objIns; List<? extends StructField> fields = structOI.getAllStructFieldRefs(); for (StructField field : fields) { r = 31 * r + hashCode( structOI.getStructFieldData(o, field), field.getFieldObjectInspector()); } return r; case UNION: UnionObjectInspector uOI = (UnionObjectInspector) objIns; byte tag = uOI.getTag(o); return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag)); default: throw new RuntimeException("Unknown type: " + objIns.getTypeName()); } }
public static Object copyToStandardObject( Object o, ObjectInspector oi, ObjectInspectorCopyOption objectInspectorOption) { if (o == null) { return null; } Object result = null; switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector loi = (PrimitiveObjectInspector) oi; if (objectInspectorOption == ObjectInspectorCopyOption.DEFAULT) { objectInspectorOption = loi.preferWritable() ? ObjectInspectorCopyOption.WRITABLE : ObjectInspectorCopyOption.JAVA; } switch (objectInspectorOption) { case JAVA: result = loi.getPrimitiveJavaObject(o); if (loi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP) { result = PrimitiveObjectInspectorFactory.javaTimestampObjectInspector.copyObject(result); } break; case WRITABLE: result = loi.getPrimitiveWritableObject(loi.copyObject(o)); break; } break; } case LIST: { ListObjectInspector loi = (ListObjectInspector) oi; int length = loi.getListLength(o); ArrayList<Object> list = new ArrayList<Object>(length); for (int i = 0; i < length; i++) { list.add( copyToStandardObject( loi.getListElement(o, i), loi.getListElementObjectInspector(), objectInspectorOption)); } result = list; break; } case MAP: { MapObjectInspector moi = (MapObjectInspector) oi; HashMap<Object, Object> map = new HashMap<Object, Object>(); Map<? extends Object, ? extends Object> omap = moi.getMap(o); for (Map.Entry<? extends Object, ? extends Object> entry : omap.entrySet()) { map.put( copyToStandardObject( entry.getKey(), moi.getMapKeyObjectInspector(), objectInspectorOption), copyToStandardObject( entry.getValue(), moi.getMapValueObjectInspector(), objectInspectorOption)); } result = map; break; } case STRUCT: { StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); ArrayList<Object> struct = new ArrayList<Object>(fields.size()); for (StructField f : fields) { struct.add( copyToStandardObject( soi.getStructFieldData(o, f), f.getFieldObjectInspector(), objectInspectorOption)); } result = struct; break; } case UNION: { UnionObjectInspector uoi = (UnionObjectInspector) oi; List<ObjectInspector> objectInspectors = uoi.getObjectInspectors(); Object object = copyToStandardObject( uoi.getField(o), objectInspectors.get(uoi.getTag(o)), objectInspectorOption); result = object; break; } default: { throw new RuntimeException("Unknown ObjectInspector category!"); } } return result; }