@Override
  public Object evaluate(DeferredObject[] arguments) throws HiveException {
    if (arguments[0].get() == null) {
      return null;
    }

    Object array = arguments[0].get();
    ListObjectInspector arrayOI = (ListObjectInspector) argumentOIs[0];
    List retArray = (List) arrayOI.getList(array);
    final ObjectInspector valInspector = arrayOI.getListElementObjectInspector();
    Collections.sort(
        retArray,
        new Comparator() {

          @Override
          public int compare(Object o1, Object o2) {
            return ObjectInspectorUtils.compare(o1, valInspector, o2, valInspector);
          }
        });

    ret.clear();
    for (int i = 0; i < retArray.size(); i++) {
      ret.add(converters[0].convert(retArray.get(i)));
    }
    return ret;
  }
Exemple #2
0
  private static Block serializeList(
      Type type, BlockBuilder builder, Object object, ListObjectInspector inspector) {
    List<?> list = inspector.getList(object);
    if (list == null) {
      requireNonNull(builder, "parent builder is null").appendNull();
      return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter");
    Type elementType = typeParameters.get(0);
    ObjectInspector elementInspector = inspector.getListElementObjectInspector();
    BlockBuilder currentBuilder;
    if (builder != null) {
      currentBuilder = builder.beginBlockEntry();
    } else {
      currentBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), list.size());
    }

    for (Object element : list) {
      serializeObject(elementType, currentBuilder, element, elementInspector);
    }

    if (builder != null) {
      builder.closeEntry();
      return null;
    } else {
      Block resultBlock = currentBuilder.build();
      return resultBlock;
    }
  }
 private ArrayWritable createArray(final Object obj, final ListObjectInspector inspector)
     throws SerDeException {
   final List<?> sourceArray = inspector.getList(obj);
   final ObjectInspector subInspector = inspector.getListElementObjectInspector();
   final List<Writable> array = new ArrayList<Writable>();
   if (sourceArray != null) {
     for (final Object curObj : sourceArray) {
       array.add(createObject(curObj, subInspector));
     }
   }
   if (array.size() > 0) {
     final ArrayWritable subArray =
         new ArrayWritable(Writable.class, array.toArray(new Writable[array.size()]));
     return new ArrayWritable(Writable.class, new Writable[] {subArray});
   } else {
     return null;
   }
 }
  /**
   * Serializes a Hive List using a JSONArray
   *
   * @param obj the object to serialize
   * @param loi the object's inspector
   * @return
   */
  private JSONArray serializeList(Object obj, ListObjectInspector loi) {
    // could be an array of whatever!
    // we do it in reverse order since the JSONArray is grown on demand,
    // as higher indexes are added.
    if (obj == null) {
      return null;
    }

    JSONArray ar = new JSONArray();
    for (int i = loi.getListLength(obj) - 1; i >= 0; i--) {
      Object element = loi.getListElement(obj, i);
      try {
        ar.put(i, serializeField(element, loi.getListElementObjectInspector()));
      } catch (JSONException ex) {
        LOG.warn("Problem serializing array", ex);
        throw new RuntimeException(ex);
      }
    }
    return ar;
  }
 @Override
 public void process(Object[] o) throws HiveException {
   List<?> list = listOI.getList(o[0]);
   if (list == null) {
     return;
   }
   for (Object r : list) {
     forwardObj[0] = r;
     forward(forwardObj);
   }
 }
  @Override
  public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException {
    if (args.length != 1) {
      throw new UDFArgumentException("explode() takes only one argument");
    }

    if (args[0].getCategory() != ObjectInspector.Category.LIST) {
      throw new UDFArgumentException("explode() takes an array as a parameter");
    }
    listOI = (ListObjectInspector) args[0];

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("col");
    fieldOIs.add(listOI.getListElementObjectInspector());
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
  }
    @Override
    public void merge(AggregationBuffer agg, Object partial) throws HiveException {
      DeleteBuffer myagg = (DeleteBuffer) agg;
      List<Object> partialResult = (List<Object>) this.listKVOI.getList(partial);
      ListObjectInspector subListOI =
          (ListObjectInspector) listKVOI.getListElementObjectInspector();

      List first = subListOI.getList(partialResult.get(0));
      String tableName =
          ((StringObjectInspector) (subListOI.getListElementObjectInspector()))
              .getPrimitiveJavaObject(first.get(0));
      configMap.put(HTableFactory.TABLE_NAME_TAG, tableName);
      String zookeeper =
          ((StringObjectInspector) (subListOI.getListElementObjectInspector()))
              .getPrimitiveJavaObject(first.get(1));
      configMap.put(HTableFactory.ZOOKEEPER_QUORUM_TAG, zookeeper);

      //// Include arbitrary configurations, by adding strings of the form k=v
      for (int j = 4; j < first.size(); ++j) {
        String kvStr =
            ((StringObjectInspector) (subListOI.getListElementObjectInspector()))
                .getPrimitiveJavaObject(first.get(j));
        String[] kvArr = kvStr.split("=");
        if (kvArr.length == 2) {
          configMap.put(kvArr[0], kvArr[1]);
        }
      }

      for (int i = 1; i < partialResult.size(); ++i) {

        List kvList = subListOI.getList(partialResult.get(i));
        String key =
            ((StringObjectInspector) (subListOI.getListElementObjectInspector()))
                .getPrimitiveJavaObject(kvList.get(0));

        myagg.addKey(key.getBytes());
      }

      if (myagg.deleteList.size() >= batchSize) {
        batchUpdate(myagg, false);
      }
    }
Exemple #8
0
 @Nullable
 public static List<String> asStringList(
     @Nonnull final DeferredObject arg, @Nonnull final ListObjectInspector listOI)
     throws HiveException {
   Object argObj = arg.get();
   if (argObj == null) {
     return null;
   }
   List<?> data = listOI.getList(argObj);
   int size = data.size();
   if (size == 0) {
     return Collections.emptyList();
   }
   final String[] ary = new String[size];
   for (int i = 0; i < size; i++) {
     Object o = data.get(i);
     if (o != null) {
       ary[i] = o.toString();
     }
   }
   return Arrays.asList(ary);
 }
  @Override
  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 2) {
      throw new UDFArgumentLengthException(
          "arrayContainsExample only takes 2 arguments: List<T>, T");
    }
    // 1. Check we received the right object types.
    ObjectInspector a = arguments[0];
    ObjectInspector b = arguments[1];
    if (!(a instanceof ListObjectInspector) || !(b instanceof StringObjectInspector)) {
      throw new UDFArgumentException(
          "first argument must be a list / array, second argument must be a string");
    }
    this.listOI = (ListObjectInspector) a;
    this.elementOI = (StringObjectInspector) b;

    // 2. Check that the list contains strings
    if (!(listOI.getListElementObjectInspector() instanceof StringObjectInspector)) {
      throw new UDFArgumentException("first argument must be a list of strings");
    }

    // the return type of our function is a boolean, so we provide the correct object inspector
    return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
  }
Exemple #10
0
 /**
  * True if Object passed is representing null object.
  *
  * @param o The object
  * @param oi The ObjectInspector
  * @return true if the object passed is representing NULL object false otherwise
  */
 public static boolean hasAnyNullObject(Object o, ObjectInspector oi) {
   switch (oi.getCategory()) {
     case PRIMITIVE:
       {
         if (o == null) {
           return true;
         }
         return false;
       }
     case LIST:
       {
         ListObjectInspector loi = (ListObjectInspector) oi;
         ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
         List<?> olist = loi.getList(o);
         if (olist == null) {
           return true;
         } else {
           // there are no elements in the list
           if (olist.size() == 0) {
             return false;
           }
           // if all the elements are representing null, then return true
           for (int i = 0; i < olist.size(); i++) {
             if (hasAnyNullObject(olist.get(i), listElementObjectInspector)) {
               return true;
             }
           }
           return false;
         }
       }
     case MAP:
       {
         MapObjectInspector moi = (MapObjectInspector) oi;
         ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
         ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
         Map<?, ?> omap = moi.getMap(o);
         if (omap == null) {
           return true;
         } else {
           // there are no elements in the map
           if (omap.entrySet().size() == 0) {
             return false;
           }
           // if all the entries of map are representing null, then return true
           for (Map.Entry<?, ?> entry : omap.entrySet()) {
             if (hasAnyNullObject(entry.getKey(), mapKeyObjectInspector)
                 || hasAnyNullObject(entry.getValue(), mapValueObjectInspector)) {
               return true;
             }
           }
           return false;
         }
       }
     case STRUCT:
       {
         StructObjectInspector soi = (StructObjectInspector) oi;
         List<? extends StructField> structFields = soi.getAllStructFieldRefs();
         if (o == null) {
           return true;
         } else {
           // there are no fields in the struct
           if (structFields.size() == 0) {
             return false;
           }
           // if any the fields of struct are representing null, then return true
           for (int i = 0; i < structFields.size(); i++) {
             if (hasAnyNullObject(
                 soi.getStructFieldData(o, structFields.get(i)),
                 structFields.get(i).getFieldObjectInspector())) {
               return true;
             }
           }
           return false;
         }
       }
     case UNION:
       {
         UnionObjectInspector uoi = (UnionObjectInspector) oi;
         if (o == null) {
           return true;
         } else {
           // there are no elements in the union
           if (uoi.getObjectInspectors().size() == 0) {
             return false;
           }
           return hasAnyNullObject(uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
         }
       }
     default:
       throw new RuntimeException("Unknown type in ObjectInspector!");
   }
 }
Exemple #11
0
  static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {

    switch (oi.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            switch (poi.getPrimitiveCategory()) {
              case BOOLEAN:
                {
                  boolean b = ((BooleanObjectInspector) poi).get(o);
                  sb.append(b ? "true" : "false");
                  break;
                }
              case BYTE:
                {
                  sb.append(((ByteObjectInspector) poi).get(o));
                  break;
                }
              case SHORT:
                {
                  sb.append(((ShortObjectInspector) poi).get(o));
                  break;
                }
              case INT:
                {
                  sb.append(((IntObjectInspector) poi).get(o));
                  break;
                }
              case LONG:
                {
                  sb.append(((LongObjectInspector) poi).get(o));
                  break;
                }
              case FLOAT:
                {
                  sb.append(((FloatObjectInspector) poi).get(o));
                  break;
                }
              case DOUBLE:
                {
                  sb.append(((DoubleObjectInspector) poi).get(o));
                  break;
                }
              case STRING:
                {
                  sb.append('"');
                  sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
                  sb.append('"');
                  break;
                }
              case CHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case VARCHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case DATE:
                {
                  sb.append('"');
                  sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case TIMESTAMP:
                {
                  sb.append('"');
                  sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case BINARY:
                {
                  BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
                  Text txt = new Text();
                  txt.set(bw.getBytes(), 0, bw.getLength());
                  sb.append(txt.toString());
                  break;
                }
              case DECIMAL:
                {
                  sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
                  break;
                }
              default:
                throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
            }
          }
          break;
        }
      case LIST:
        {
          ListObjectInspector loi = (ListObjectInspector) oi;
          ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
          List<?> olist = loi.getList(o);
          if (olist == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACKET);
            for (int i = 0; i < olist.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
            }
            sb.append(RBRACKET);
          }
          break;
        }
      case MAP:
        {
          MapObjectInspector moi = (MapObjectInspector) oi;
          ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
          ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
          Map<?, ?> omap = moi.getMap(o);
          if (omap == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            boolean first = true;
            for (Object entry : omap.entrySet()) {
              if (first) {
                first = false;
              } else {
                sb.append(COMMA);
              }
              Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
              buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
              sb.append(COLON);
              buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case STRUCT:
        {
          StructObjectInspector soi = (StructObjectInspector) oi;
          List<? extends StructField> structFields = soi.getAllStructFieldRefs();
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            for (int i = 0; i < structFields.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              sb.append(QUOTE);
              sb.append(structFields.get(i).getFieldName());
              sb.append(QUOTE);
              sb.append(COLON);
              buildJSONString(
                  sb,
                  soi.getStructFieldData(o, structFields.get(i)),
                  structFields.get(i).getFieldObjectInspector(),
                  JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case UNION:
        {
          UnionObjectInspector uoi = (UnionObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            sb.append(uoi.getTag(o));
            sb.append(COLON);
            buildJSONString(
                sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
            sb.append(RBRACE);
          }
          break;
        }
      default:
        throw new RuntimeException("Unknown type in ObjectInspector!");
    }
  }
  static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, boolean invert) {
    // Is this field a null?
    if (o == null) {
      buffer.write((byte) 0, invert);
      return;
    }
    // This field is not a null.
    buffer.write((byte) 1, invert);

    switch (oi.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
          switch (poi.getPrimitiveCategory()) {
            case VOID:
              {
                return;
              }
            case BOOLEAN:
              {
                boolean v = ((BooleanObjectInspector) poi).get(o);
                buffer.write((byte) (v ? 2 : 1), invert);
                return;
              }
            case BYTE:
              {
                ByteObjectInspector boi = (ByteObjectInspector) poi;
                byte v = boi.get(o);
                buffer.write((byte) (v ^ 0x80), invert);
                return;
              }
            case SHORT:
              {
                ShortObjectInspector spoi = (ShortObjectInspector) poi;
                short v = spoi.get(o);
                buffer.write((byte) ((v >> 8) ^ 0x80), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case INT:
              {
                IntObjectInspector ioi = (IntObjectInspector) poi;
                int v = ioi.get(o);
                buffer.write((byte) ((v >> 24) ^ 0x80), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case LONG:
              {
                LongObjectInspector loi = (LongObjectInspector) poi;
                long v = loi.get(o);
                buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case FLOAT:
              {
                FloatObjectInspector foi = (FloatObjectInspector) poi;
                int v = Float.floatToIntBits(foi.get(o));
                if ((v & (1 << 31)) != 0) {
                  // negative number, flip all bits
                  v = ~v;
                } else {
                  // positive number, flip the first bit
                  v = v ^ (1 << 31);
                }
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case DOUBLE:
              {
                DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                long v = Double.doubleToLongBits(doi.get(o));
                if ((v & (1L << 63)) != 0) {
                  // negative number, flip all bits
                  v = ~v;
                } else {
                  // positive number, flip the first bit
                  v = v ^ (1L << 63);
                }
                buffer.write((byte) (v >> 56), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case STRING:
              {
                StringObjectInspector soi = (StringObjectInspector) poi;
                Text t = soi.getPrimitiveWritableObject(o);
                serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                return;
              }

            case BINARY:
              {
                BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
                BytesWritable ba = baoi.getPrimitiveWritableObject(o);
                byte[] toSer = new byte[ba.getLength()];
                System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
                serializeBytes(buffer, toSer, ba.getLength(), invert);
                return;
              }
            case DATE:
              {
                DateObjectInspector doi = (DateObjectInspector) poi;
                long v = doi.getPrimitiveWritableObject(o).getTimeInSeconds();
                buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case TIMESTAMP:
              {
                TimestampObjectInspector toi = (TimestampObjectInspector) poi;
                TimestampWritable t = toi.getPrimitiveWritableObject(o);
                byte[] data = t.getBinarySortable();
                for (int i = 0; i < data.length; i++) {
                  buffer.write(data[i], invert);
                }
                return;
              }
            default:
              {
                throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
              }
          }
        }
      case LIST:
        {
          ListObjectInspector loi = (ListObjectInspector) oi;
          ObjectInspector eoi = loi.getListElementObjectInspector();

          // \1 followed by each element
          int size = loi.getListLength(o);
          for (int eid = 0; eid < size; eid++) {
            buffer.write((byte) 1, invert);
            serialize(buffer, loi.getListElement(o, eid), eoi, invert);
          }
          // and \0 to terminate
          buffer.write((byte) 0, invert);
          return;
        }
      case MAP:
        {
          MapObjectInspector moi = (MapObjectInspector) oi;
          ObjectInspector koi = moi.getMapKeyObjectInspector();
          ObjectInspector voi = moi.getMapValueObjectInspector();

          // \1 followed by each key and then each value
          Map<?, ?> map = moi.getMap(o);
          for (Map.Entry<?, ?> entry : map.entrySet()) {
            buffer.write((byte) 1, invert);
            serialize(buffer, entry.getKey(), koi, invert);
            serialize(buffer, entry.getValue(), voi, invert);
          }
          // and \0 to terminate
          buffer.write((byte) 0, invert);
          return;
        }
      case STRUCT:
        {
          StructObjectInspector soi = (StructObjectInspector) oi;
          List<? extends StructField> fields = soi.getAllStructFieldRefs();

          for (int i = 0; i < fields.size(); i++) {
            serialize(
                buffer,
                soi.getStructFieldData(o, fields.get(i)),
                fields.get(i).getFieldObjectInspector(),
                invert);
          }
          return;
        }
      case UNION:
        {
          UnionObjectInspector uoi = (UnionObjectInspector) oi;
          byte tag = uoi.getTag(o);
          buffer.write(tag, invert);
          serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert);
          return;
        }
      default:
        {
          throw new RuntimeException("Unrecognized type: " + oi.getCategory());
        }
    }
  }