示例#1
0
  @Override
  public Object evaluate(DeferredObject[] arguments) throws HiveException {
    if (arguments[0].get() == null) {
      return null;
    }

    Object array = arguments[0].get();
    ListObjectInspector arrayOI = (ListObjectInspector) argumentOIs[0];
    List retArray = (List) arrayOI.getList(array);
    final ObjectInspector valInspector = arrayOI.getListElementObjectInspector();
    Collections.sort(
        retArray,
        new Comparator() {

          @Override
          public int compare(Object o1, Object o2) {
            return ObjectInspectorUtils.compare(o1, valInspector, o2, valInspector);
          }
        });

    ret.clear();
    for (int i = 0; i < retArray.size(); i++) {
      ret.add(converters[0].convert(retArray.get(i)));
    }
    return ret;
  }
示例#2
0
  private static Block serializeList(
      Type type, BlockBuilder builder, Object object, ListObjectInspector inspector) {
    List<?> list = inspector.getList(object);
    if (list == null) {
      requireNonNull(builder, "parent builder is null").appendNull();
      return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter");
    Type elementType = typeParameters.get(0);
    ObjectInspector elementInspector = inspector.getListElementObjectInspector();
    BlockBuilder currentBuilder;
    if (builder != null) {
      currentBuilder = builder.beginBlockEntry();
    } else {
      currentBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), list.size());
    }

    for (Object element : list) {
      serializeObject(elementType, currentBuilder, element, elementInspector);
    }

    if (builder != null) {
      builder.closeEntry();
      return null;
    } else {
      Block resultBlock = currentBuilder.build();
      return resultBlock;
    }
  }
示例#3
0
    @Override
    public void merge(AggregationBuffer agg, Object partial) throws HiveException {
      DeleteBuffer myagg = (DeleteBuffer) agg;
      List<Object> partialResult = (List<Object>) this.listKVOI.getList(partial);
      ListObjectInspector subListOI =
          (ListObjectInspector) listKVOI.getListElementObjectInspector();

      List first = subListOI.getList(partialResult.get(0));
      String tableName =
          ((StringObjectInspector) (subListOI.getListElementObjectInspector()))
              .getPrimitiveJavaObject(first.get(0));
      configMap.put(HTableFactory.TABLE_NAME_TAG, tableName);
      String zookeeper =
          ((StringObjectInspector) (subListOI.getListElementObjectInspector()))
              .getPrimitiveJavaObject(first.get(1));
      configMap.put(HTableFactory.ZOOKEEPER_QUORUM_TAG, zookeeper);

      //// Include arbitrary configurations, by adding strings of the form k=v
      for (int j = 4; j < first.size(); ++j) {
        String kvStr =
            ((StringObjectInspector) (subListOI.getListElementObjectInspector()))
                .getPrimitiveJavaObject(first.get(j));
        String[] kvArr = kvStr.split("=");
        if (kvArr.length == 2) {
          configMap.put(kvArr[0], kvArr[1]);
        }
      }

      for (int i = 1; i < partialResult.size(); ++i) {

        List kvList = subListOI.getList(partialResult.get(i));
        String key =
            ((StringObjectInspector) (subListOI.getListElementObjectInspector()))
                .getPrimitiveJavaObject(kvList.get(0));

        myagg.addKey(key.getBytes());
      }

      if (myagg.deleteList.size() >= batchSize) {
        batchUpdate(myagg, false);
      }
    }
 @Override
 public void process(Object[] o) throws HiveException {
   List<?> list = listOI.getList(o[0]);
   if (list == null) {
     return;
   }
   for (Object r : list) {
     forwardObj[0] = r;
     forward(forwardObj);
   }
 }
 private ArrayWritable createArray(final Object obj, final ListObjectInspector inspector)
     throws SerDeException {
   final List<?> sourceArray = inspector.getList(obj);
   final ObjectInspector subInspector = inspector.getListElementObjectInspector();
   final List<Writable> array = new ArrayList<Writable>();
   if (sourceArray != null) {
     for (final Object curObj : sourceArray) {
       array.add(createObject(curObj, subInspector));
     }
   }
   if (array.size() > 0) {
     final ArrayWritable subArray =
         new ArrayWritable(Writable.class, array.toArray(new Writable[array.size()]));
     return new ArrayWritable(Writable.class, new Writable[] {subArray});
   } else {
     return null;
   }
 }
示例#6
0
 @Nullable
 public static List<String> asStringList(
     @Nonnull final DeferredObject arg, @Nonnull final ListObjectInspector listOI)
     throws HiveException {
   Object argObj = arg.get();
   if (argObj == null) {
     return null;
   }
   List<?> data = listOI.getList(argObj);
   int size = data.size();
   if (size == 0) {
     return Collections.emptyList();
   }
   final String[] ary = new String[size];
   for (int i = 0; i < size; i++) {
     Object o = data.get(i);
     if (o != null) {
       ary[i] = o.toString();
     }
   }
   return Arrays.asList(ary);
 }
示例#7
0
 /**
  * True if Object passed is representing null object.
  *
  * @param o The object
  * @param oi The ObjectInspector
  * @return true if the object passed is representing NULL object false otherwise
  */
 public static boolean hasAnyNullObject(Object o, ObjectInspector oi) {
   switch (oi.getCategory()) {
     case PRIMITIVE:
       {
         if (o == null) {
           return true;
         }
         return false;
       }
     case LIST:
       {
         ListObjectInspector loi = (ListObjectInspector) oi;
         ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
         List<?> olist = loi.getList(o);
         if (olist == null) {
           return true;
         } else {
           // there are no elements in the list
           if (olist.size() == 0) {
             return false;
           }
           // if all the elements are representing null, then return true
           for (int i = 0; i < olist.size(); i++) {
             if (hasAnyNullObject(olist.get(i), listElementObjectInspector)) {
               return true;
             }
           }
           return false;
         }
       }
     case MAP:
       {
         MapObjectInspector moi = (MapObjectInspector) oi;
         ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
         ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
         Map<?, ?> omap = moi.getMap(o);
         if (omap == null) {
           return true;
         } else {
           // there are no elements in the map
           if (omap.entrySet().size() == 0) {
             return false;
           }
           // if all the entries of map are representing null, then return true
           for (Map.Entry<?, ?> entry : omap.entrySet()) {
             if (hasAnyNullObject(entry.getKey(), mapKeyObjectInspector)
                 || hasAnyNullObject(entry.getValue(), mapValueObjectInspector)) {
               return true;
             }
           }
           return false;
         }
       }
     case STRUCT:
       {
         StructObjectInspector soi = (StructObjectInspector) oi;
         List<? extends StructField> structFields = soi.getAllStructFieldRefs();
         if (o == null) {
           return true;
         } else {
           // there are no fields in the struct
           if (structFields.size() == 0) {
             return false;
           }
           // if any the fields of struct are representing null, then return true
           for (int i = 0; i < structFields.size(); i++) {
             if (hasAnyNullObject(
                 soi.getStructFieldData(o, structFields.get(i)),
                 structFields.get(i).getFieldObjectInspector())) {
               return true;
             }
           }
           return false;
         }
       }
     case UNION:
       {
         UnionObjectInspector uoi = (UnionObjectInspector) oi;
         if (o == null) {
           return true;
         } else {
           // there are no elements in the union
           if (uoi.getObjectInspectors().size() == 0) {
             return false;
           }
           return hasAnyNullObject(uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
         }
       }
     default:
       throw new RuntimeException("Unknown type in ObjectInspector!");
   }
 }
示例#8
0
  static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {

    switch (oi.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            switch (poi.getPrimitiveCategory()) {
              case BOOLEAN:
                {
                  boolean b = ((BooleanObjectInspector) poi).get(o);
                  sb.append(b ? "true" : "false");
                  break;
                }
              case BYTE:
                {
                  sb.append(((ByteObjectInspector) poi).get(o));
                  break;
                }
              case SHORT:
                {
                  sb.append(((ShortObjectInspector) poi).get(o));
                  break;
                }
              case INT:
                {
                  sb.append(((IntObjectInspector) poi).get(o));
                  break;
                }
              case LONG:
                {
                  sb.append(((LongObjectInspector) poi).get(o));
                  break;
                }
              case FLOAT:
                {
                  sb.append(((FloatObjectInspector) poi).get(o));
                  break;
                }
              case DOUBLE:
                {
                  sb.append(((DoubleObjectInspector) poi).get(o));
                  break;
                }
              case STRING:
                {
                  sb.append('"');
                  sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
                  sb.append('"');
                  break;
                }
              case CHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case VARCHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case DATE:
                {
                  sb.append('"');
                  sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case TIMESTAMP:
                {
                  sb.append('"');
                  sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case BINARY:
                {
                  BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
                  Text txt = new Text();
                  txt.set(bw.getBytes(), 0, bw.getLength());
                  sb.append(txt.toString());
                  break;
                }
              case DECIMAL:
                {
                  sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
                  break;
                }
              default:
                throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
            }
          }
          break;
        }
      case LIST:
        {
          ListObjectInspector loi = (ListObjectInspector) oi;
          ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
          List<?> olist = loi.getList(o);
          if (olist == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACKET);
            for (int i = 0; i < olist.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
            }
            sb.append(RBRACKET);
          }
          break;
        }
      case MAP:
        {
          MapObjectInspector moi = (MapObjectInspector) oi;
          ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
          ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
          Map<?, ?> omap = moi.getMap(o);
          if (omap == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            boolean first = true;
            for (Object entry : omap.entrySet()) {
              if (first) {
                first = false;
              } else {
                sb.append(COMMA);
              }
              Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
              buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
              sb.append(COLON);
              buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case STRUCT:
        {
          StructObjectInspector soi = (StructObjectInspector) oi;
          List<? extends StructField> structFields = soi.getAllStructFieldRefs();
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            for (int i = 0; i < structFields.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              sb.append(QUOTE);
              sb.append(structFields.get(i).getFieldName());
              sb.append(QUOTE);
              sb.append(COLON);
              buildJSONString(
                  sb,
                  soi.getStructFieldData(o, structFields.get(i)),
                  structFields.get(i).getFieldObjectInspector(),
                  JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case UNION:
        {
          UnionObjectInspector uoi = (UnionObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            sb.append(uoi.getTag(o));
            sb.append(COLON);
            buildJSONString(
                sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
            sb.append(RBRACE);
          }
          break;
        }
      default:
        throw new RuntimeException("Unknown type in ObjectInspector!");
    }
  }