@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
if (arguments[0].get() == null) {
return null;
}
Object array = arguments[0].get();
ListObjectInspector arrayOI = (ListObjectInspector) argumentOIs[0];
List retArray = (List) arrayOI.getList(array);
final ObjectInspector valInspector = arrayOI.getListElementObjectInspector();
Collections.sort(
retArray,
new Comparator() {
@Override
public int compare(Object o1, Object o2) {
return ObjectInspectorUtils.compare(o1, valInspector, o2, valInspector);
}
});
ret.clear();
for (int i = 0; i < retArray.size(); i++) {
ret.add(converters[0].convert(retArray.get(i)));
}
return ret;
}
private static Block serializeList(
Type type, BlockBuilder builder, Object object, ListObjectInspector inspector) {
List<?> list = inspector.getList(object);
if (list == null) {
requireNonNull(builder, "parent builder is null").appendNull();
return null;
}
List<Type> typeParameters = type.getTypeParameters();
checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter");
Type elementType = typeParameters.get(0);
ObjectInspector elementInspector = inspector.getListElementObjectInspector();
BlockBuilder currentBuilder;
if (builder != null) {
currentBuilder = builder.beginBlockEntry();
} else {
currentBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), list.size());
}
for (Object element : list) {
serializeObject(elementType, currentBuilder, element, elementInspector);
}
if (builder != null) {
builder.closeEntry();
return null;
} else {
Block resultBlock = currentBuilder.build();
return resultBlock;
}
}
@Override
public void merge(AggregationBuffer agg, Object partial) throws HiveException {
DeleteBuffer myagg = (DeleteBuffer) agg;
List<Object> partialResult = (List<Object>) this.listKVOI.getList(partial);
ListObjectInspector subListOI =
(ListObjectInspector) listKVOI.getListElementObjectInspector();
List first = subListOI.getList(partialResult.get(0));
String tableName =
((StringObjectInspector) (subListOI.getListElementObjectInspector()))
.getPrimitiveJavaObject(first.get(0));
configMap.put(HTableFactory.TABLE_NAME_TAG, tableName);
String zookeeper =
((StringObjectInspector) (subListOI.getListElementObjectInspector()))
.getPrimitiveJavaObject(first.get(1));
configMap.put(HTableFactory.ZOOKEEPER_QUORUM_TAG, zookeeper);
//// Include arbitrary configurations, by adding strings of the form k=v
for (int j = 4; j < first.size(); ++j) {
String kvStr =
((StringObjectInspector) (subListOI.getListElementObjectInspector()))
.getPrimitiveJavaObject(first.get(j));
String[] kvArr = kvStr.split("=");
if (kvArr.length == 2) {
configMap.put(kvArr[0], kvArr[1]);
}
}
for (int i = 1; i < partialResult.size(); ++i) {
List kvList = subListOI.getList(partialResult.get(i));
String key =
((StringObjectInspector) (subListOI.getListElementObjectInspector()))
.getPrimitiveJavaObject(kvList.get(0));
myagg.addKey(key.getBytes());
}
if (myagg.deleteList.size() >= batchSize) {
batchUpdate(myagg, false);
}
}
@Override
public void process(Object[] o) throws HiveException {
List<?> list = listOI.getList(o[0]);
if (list == null) {
return;
}
for (Object r : list) {
forwardObj[0] = r;
forward(forwardObj);
}
}
private ArrayWritable createArray(final Object obj, final ListObjectInspector inspector)
throws SerDeException {
final List<?> sourceArray = inspector.getList(obj);
final ObjectInspector subInspector = inspector.getListElementObjectInspector();
final List<Writable> array = new ArrayList<Writable>();
if (sourceArray != null) {
for (final Object curObj : sourceArray) {
array.add(createObject(curObj, subInspector));
}
}
if (array.size() > 0) {
final ArrayWritable subArray =
new ArrayWritable(Writable.class, array.toArray(new Writable[array.size()]));
return new ArrayWritable(Writable.class, new Writable[] {subArray});
} else {
return null;
}
}
@Nullable
public static List<String> asStringList(
@Nonnull final DeferredObject arg, @Nonnull final ListObjectInspector listOI)
throws HiveException {
Object argObj = arg.get();
if (argObj == null) {
return null;
}
List<?> data = listOI.getList(argObj);
int size = data.size();
if (size == 0) {
return Collections.emptyList();
}
final String[] ary = new String[size];
for (int i = 0; i < size; i++) {
Object o = data.get(i);
if (o != null) {
ary[i] = o.toString();
}
}
return Arrays.asList(ary);
}
/**
* True if Object passed is representing null object.
*
* @param o The object
* @param oi The ObjectInspector
* @return true if the object passed is representing NULL object false otherwise
*/
public static boolean hasAnyNullObject(Object o, ObjectInspector oi) {
switch (oi.getCategory()) {
case PRIMITIVE:
{
if (o == null) {
return true;
}
return false;
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
List<?> olist = loi.getList(o);
if (olist == null) {
return true;
} else {
// there are no elements in the list
if (olist.size() == 0) {
return false;
}
// if all the elements are representing null, then return true
for (int i = 0; i < olist.size(); i++) {
if (hasAnyNullObject(olist.get(i), listElementObjectInspector)) {
return true;
}
}
return false;
}
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
Map<?, ?> omap = moi.getMap(o);
if (omap == null) {
return true;
} else {
// there are no elements in the map
if (omap.entrySet().size() == 0) {
return false;
}
// if all the entries of map are representing null, then return true
for (Map.Entry<?, ?> entry : omap.entrySet()) {
if (hasAnyNullObject(entry.getKey(), mapKeyObjectInspector)
|| hasAnyNullObject(entry.getValue(), mapValueObjectInspector)) {
return true;
}
}
return false;
}
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> structFields = soi.getAllStructFieldRefs();
if (o == null) {
return true;
} else {
// there are no fields in the struct
if (structFields.size() == 0) {
return false;
}
// if any the fields of struct are representing null, then return true
for (int i = 0; i < structFields.size(); i++) {
if (hasAnyNullObject(
soi.getStructFieldData(o, structFields.get(i)),
structFields.get(i).getFieldObjectInspector())) {
return true;
}
}
return false;
}
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
if (o == null) {
return true;
} else {
// there are no elements in the union
if (uoi.getObjectInspectors().size() == 0) {
return false;
}
return hasAnyNullObject(uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
}
}
default:
throw new RuntimeException("Unknown type in ObjectInspector!");
}
}
static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {
switch (oi.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
if (o == null) {
sb.append(nullStr);
} else {
switch (poi.getPrimitiveCategory()) {
case BOOLEAN:
{
boolean b = ((BooleanObjectInspector) poi).get(o);
sb.append(b ? "true" : "false");
break;
}
case BYTE:
{
sb.append(((ByteObjectInspector) poi).get(o));
break;
}
case SHORT:
{
sb.append(((ShortObjectInspector) poi).get(o));
break;
}
case INT:
{
sb.append(((IntObjectInspector) poi).get(o));
break;
}
case LONG:
{
sb.append(((LongObjectInspector) poi).get(o));
break;
}
case FLOAT:
{
sb.append(((FloatObjectInspector) poi).get(o));
break;
}
case DOUBLE:
{
sb.append(((DoubleObjectInspector) poi).get(o));
break;
}
case STRING:
{
sb.append('"');
sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
sb.append('"');
break;
}
case CHAR:
{
sb.append('"');
sb.append(
escapeString(
((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
sb.append('"');
break;
}
case VARCHAR:
{
sb.append('"');
sb.append(
escapeString(
((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
sb.append('"');
break;
}
case DATE:
{
sb.append('"');
sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
sb.append('"');
break;
}
case TIMESTAMP:
{
sb.append('"');
sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
sb.append('"');
break;
}
case BINARY:
{
BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
Text txt = new Text();
txt.set(bw.getBytes(), 0, bw.getLength());
sb.append(txt.toString());
break;
}
case DECIMAL:
{
sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
break;
}
default:
throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
}
}
break;
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
List<?> olist = loi.getList(o);
if (olist == null) {
sb.append(nullStr);
} else {
sb.append(LBRACKET);
for (int i = 0; i < olist.size(); i++) {
if (i > 0) {
sb.append(COMMA);
}
buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
}
sb.append(RBRACKET);
}
break;
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
Map<?, ?> omap = moi.getMap(o);
if (omap == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
boolean first = true;
for (Object entry : omap.entrySet()) {
if (first) {
first = false;
} else {
sb.append(COMMA);
}
Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
sb.append(COLON);
buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
}
sb.append(RBRACE);
}
break;
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> structFields = soi.getAllStructFieldRefs();
if (o == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
for (int i = 0; i < structFields.size(); i++) {
if (i > 0) {
sb.append(COMMA);
}
sb.append(QUOTE);
sb.append(structFields.get(i).getFieldName());
sb.append(QUOTE);
sb.append(COLON);
buildJSONString(
sb,
soi.getStructFieldData(o, structFields.get(i)),
structFields.get(i).getFieldObjectInspector(),
JSON_NULL);
}
sb.append(RBRACE);
}
break;
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
if (o == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
sb.append(uoi.getTag(o));
sb.append(COLON);
buildJSONString(
sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
sb.append(RBRACE);
}
break;
}
default:
throw new RuntimeException("Unknown type in ObjectInspector!");
}
}