private Writable createMap(final Object obj, final MapObjectInspector inspector)
      throws SerDeException {
    final Map<?, ?> sourceMap = inspector.getMap(obj);
    final ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();
    final ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
    final List<ArrayWritable> array = new ArrayList<ArrayWritable>();

    if (sourceMap != null) {
      for (final Entry<?, ?> keyValue : sourceMap.entrySet()) {
        final Writable key = createObject(keyValue.getKey(), keyInspector);
        final Writable value = createObject(keyValue.getValue(), valueInspector);
        if (key != null) {
          Writable[] arr = new Writable[2];
          arr[0] = key;
          arr[1] = value;
          array.add(new ArrayWritable(Writable.class, arr));
        }
      }
    }
    if (array.size() > 0) {
      final ArrayWritable subArray =
          new ArrayWritable(ArrayWritable.class, array.toArray(new ArrayWritable[array.size()]));
      return new ArrayWritable(Writable.class, new Writable[] {subArray});
    } else {
      return null;
    }
  }
Пример #2
0
 @Override
 public Object evaluate(DeferredObject[] arguments) throws HiveException {
   retArray.clear();
   Object mapObj = arguments[0].get();
   retArray.addAll(mapOI.getMap(mapObj).values());
   return retArray;
 }
Пример #3
0
  /**
   * Serializes a Hive map<> using a JSONObject.
   *
   * @param obj the object to serialize
   * @param moi the object's inspector
   * @return
   */
  private JSONObject serializeMap(Object obj, MapObjectInspector moi) {
    if (obj == null) {
      return null;
    }

    JSONObject jo = new JSONObject();
    Map m = moi.getMap(obj);

    for (Object k : m.keySet()) {
      try {
        jo.put(
            serializeField(k, moi.getMapKeyObjectInspector()).toString(),
            serializeField(m.get(k), moi.getMapValueObjectInspector()));
      } catch (JSONException ex) {
        LOG.warn("Problem serializing map");
      }
    }
    return jo;
  }
Пример #4
0
  private static Block serializeMap(
      Type type, BlockBuilder builder, Object object, MapObjectInspector inspector) {
    Map<?, ?> map = inspector.getMap(object);
    if (map == null) {
      requireNonNull(builder, "parent builder is null").appendNull();
      return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter");
    Type keyType = typeParameters.get(0);
    Type valueType = typeParameters.get(1);
    ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();
    ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
    BlockBuilder currentBuilder;
    if (builder != null) {
      currentBuilder = builder.beginBlockEntry();
    } else {
      currentBuilder =
          new InterleavedBlockBuilder(typeParameters, new BlockBuilderStatus(), map.size());
    }

    for (Map.Entry<?, ?> entry : map.entrySet()) {
      // Hive skips map entries with null keys
      if (entry.getKey() != null) {
        serializeObject(keyType, currentBuilder, entry.getKey(), keyInspector);
        serializeObject(valueType, currentBuilder, entry.getValue(), valueInspector);
      }
    }

    if (builder != null) {
      builder.closeEntry();
      return null;
    } else {
      Block resultBlock = currentBuilder.build();
      return resultBlock;
    }
  }
Пример #5
0
  @Override
  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 1) {
      throw new UDFArgumentLengthException("The function MAP_VALUES only accepts 1 argument.");
    } else if (!(arguments[0] instanceof MapObjectInspector)) {
      throw new UDFArgumentTypeException(
          0,
          "\""
              + Category.MAP.toString().toLowerCase()
              + "\" is expected at function MAP_VALUES, "
              + "but \""
              + arguments[0].getTypeName()
              + "\" is found");
    }

    mapOI = (MapObjectInspector) arguments[0];
    ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();
    return ObjectInspectorFactory.getStandardListObjectInspector(mapValueOI);
  }
Пример #6
0
 /**
  * True if Object passed is representing null object.
  *
  * @param o The object
  * @param oi The ObjectInspector
  * @return true if the object passed is representing NULL object false otherwise
  */
 public static boolean hasAnyNullObject(Object o, ObjectInspector oi) {
   switch (oi.getCategory()) {
     case PRIMITIVE:
       {
         if (o == null) {
           return true;
         }
         return false;
       }
     case LIST:
       {
         ListObjectInspector loi = (ListObjectInspector) oi;
         ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
         List<?> olist = loi.getList(o);
         if (olist == null) {
           return true;
         } else {
           // there are no elements in the list
           if (olist.size() == 0) {
             return false;
           }
           // if all the elements are representing null, then return true
           for (int i = 0; i < olist.size(); i++) {
             if (hasAnyNullObject(olist.get(i), listElementObjectInspector)) {
               return true;
             }
           }
           return false;
         }
       }
     case MAP:
       {
         MapObjectInspector moi = (MapObjectInspector) oi;
         ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
         ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
         Map<?, ?> omap = moi.getMap(o);
         if (omap == null) {
           return true;
         } else {
           // there are no elements in the map
           if (omap.entrySet().size() == 0) {
             return false;
           }
           // if all the entries of map are representing null, then return true
           for (Map.Entry<?, ?> entry : omap.entrySet()) {
             if (hasAnyNullObject(entry.getKey(), mapKeyObjectInspector)
                 || hasAnyNullObject(entry.getValue(), mapValueObjectInspector)) {
               return true;
             }
           }
           return false;
         }
       }
     case STRUCT:
       {
         StructObjectInspector soi = (StructObjectInspector) oi;
         List<? extends StructField> structFields = soi.getAllStructFieldRefs();
         if (o == null) {
           return true;
         } else {
           // there are no fields in the struct
           if (structFields.size() == 0) {
             return false;
           }
           // if any the fields of struct are representing null, then return true
           for (int i = 0; i < structFields.size(); i++) {
             if (hasAnyNullObject(
                 soi.getStructFieldData(o, structFields.get(i)),
                 structFields.get(i).getFieldObjectInspector())) {
               return true;
             }
           }
           return false;
         }
       }
     case UNION:
       {
         UnionObjectInspector uoi = (UnionObjectInspector) oi;
         if (o == null) {
           return true;
         } else {
           // there are no elements in the union
           if (uoi.getObjectInspectors().size() == 0) {
             return false;
           }
           return hasAnyNullObject(uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
         }
       }
     default:
       throw new RuntimeException("Unknown type in ObjectInspector!");
   }
 }
Пример #7
0
  static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {

    switch (oi.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            switch (poi.getPrimitiveCategory()) {
              case BOOLEAN:
                {
                  boolean b = ((BooleanObjectInspector) poi).get(o);
                  sb.append(b ? "true" : "false");
                  break;
                }
              case BYTE:
                {
                  sb.append(((ByteObjectInspector) poi).get(o));
                  break;
                }
              case SHORT:
                {
                  sb.append(((ShortObjectInspector) poi).get(o));
                  break;
                }
              case INT:
                {
                  sb.append(((IntObjectInspector) poi).get(o));
                  break;
                }
              case LONG:
                {
                  sb.append(((LongObjectInspector) poi).get(o));
                  break;
                }
              case FLOAT:
                {
                  sb.append(((FloatObjectInspector) poi).get(o));
                  break;
                }
              case DOUBLE:
                {
                  sb.append(((DoubleObjectInspector) poi).get(o));
                  break;
                }
              case STRING:
                {
                  sb.append('"');
                  sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
                  sb.append('"');
                  break;
                }
              case CHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case VARCHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case DATE:
                {
                  sb.append('"');
                  sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case TIMESTAMP:
                {
                  sb.append('"');
                  sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case BINARY:
                {
                  BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
                  Text txt = new Text();
                  txt.set(bw.getBytes(), 0, bw.getLength());
                  sb.append(txt.toString());
                  break;
                }
              case DECIMAL:
                {
                  sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
                  break;
                }
              default:
                throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
            }
          }
          break;
        }
      case LIST:
        {
          ListObjectInspector loi = (ListObjectInspector) oi;
          ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
          List<?> olist = loi.getList(o);
          if (olist == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACKET);
            for (int i = 0; i < olist.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
            }
            sb.append(RBRACKET);
          }
          break;
        }
      case MAP:
        {
          MapObjectInspector moi = (MapObjectInspector) oi;
          ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
          ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
          Map<?, ?> omap = moi.getMap(o);
          if (omap == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            boolean first = true;
            for (Object entry : omap.entrySet()) {
              if (first) {
                first = false;
              } else {
                sb.append(COMMA);
              }
              Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
              buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
              sb.append(COLON);
              buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case STRUCT:
        {
          StructObjectInspector soi = (StructObjectInspector) oi;
          List<? extends StructField> structFields = soi.getAllStructFieldRefs();
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            for (int i = 0; i < structFields.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              sb.append(QUOTE);
              sb.append(structFields.get(i).getFieldName());
              sb.append(QUOTE);
              sb.append(COLON);
              buildJSONString(
                  sb,
                  soi.getStructFieldData(o, structFields.get(i)),
                  structFields.get(i).getFieldObjectInspector(),
                  JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case UNION:
        {
          UnionObjectInspector uoi = (UnionObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            sb.append(uoi.getTag(o));
            sb.append(COLON);
            buildJSONString(
                sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
            sb.append(RBRACE);
          }
          break;
        }
      default:
        throw new RuntimeException("Unknown type in ObjectInspector!");
    }
  }
  static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, boolean invert) {
    // Is this field a null?
    if (o == null) {
      buffer.write((byte) 0, invert);
      return;
    }
    // This field is not a null.
    buffer.write((byte) 1, invert);

    switch (oi.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
          switch (poi.getPrimitiveCategory()) {
            case VOID:
              {
                return;
              }
            case BOOLEAN:
              {
                boolean v = ((BooleanObjectInspector) poi).get(o);
                buffer.write((byte) (v ? 2 : 1), invert);
                return;
              }
            case BYTE:
              {
                ByteObjectInspector boi = (ByteObjectInspector) poi;
                byte v = boi.get(o);
                buffer.write((byte) (v ^ 0x80), invert);
                return;
              }
            case SHORT:
              {
                ShortObjectInspector spoi = (ShortObjectInspector) poi;
                short v = spoi.get(o);
                buffer.write((byte) ((v >> 8) ^ 0x80), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case INT:
              {
                IntObjectInspector ioi = (IntObjectInspector) poi;
                int v = ioi.get(o);
                buffer.write((byte) ((v >> 24) ^ 0x80), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case LONG:
              {
                LongObjectInspector loi = (LongObjectInspector) poi;
                long v = loi.get(o);
                buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case FLOAT:
              {
                FloatObjectInspector foi = (FloatObjectInspector) poi;
                int v = Float.floatToIntBits(foi.get(o));
                if ((v & (1 << 31)) != 0) {
                  // negative number, flip all bits
                  v = ~v;
                } else {
                  // positive number, flip the first bit
                  v = v ^ (1 << 31);
                }
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case DOUBLE:
              {
                DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                long v = Double.doubleToLongBits(doi.get(o));
                if ((v & (1L << 63)) != 0) {
                  // negative number, flip all bits
                  v = ~v;
                } else {
                  // positive number, flip the first bit
                  v = v ^ (1L << 63);
                }
                buffer.write((byte) (v >> 56), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case STRING:
              {
                StringObjectInspector soi = (StringObjectInspector) poi;
                Text t = soi.getPrimitiveWritableObject(o);
                serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                return;
              }

            case BINARY:
              {
                BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
                BytesWritable ba = baoi.getPrimitiveWritableObject(o);
                byte[] toSer = new byte[ba.getLength()];
                System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
                serializeBytes(buffer, toSer, ba.getLength(), invert);
                return;
              }
            case DATE:
              {
                DateObjectInspector doi = (DateObjectInspector) poi;
                long v = doi.getPrimitiveWritableObject(o).getTimeInSeconds();
                buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case TIMESTAMP:
              {
                TimestampObjectInspector toi = (TimestampObjectInspector) poi;
                TimestampWritable t = toi.getPrimitiveWritableObject(o);
                byte[] data = t.getBinarySortable();
                for (int i = 0; i < data.length; i++) {
                  buffer.write(data[i], invert);
                }
                return;
              }
            default:
              {
                throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
              }
          }
        }
      case LIST:
        {
          ListObjectInspector loi = (ListObjectInspector) oi;
          ObjectInspector eoi = loi.getListElementObjectInspector();

          // \1 followed by each element
          int size = loi.getListLength(o);
          for (int eid = 0; eid < size; eid++) {
            buffer.write((byte) 1, invert);
            serialize(buffer, loi.getListElement(o, eid), eoi, invert);
          }
          // and \0 to terminate
          buffer.write((byte) 0, invert);
          return;
        }
      case MAP:
        {
          MapObjectInspector moi = (MapObjectInspector) oi;
          ObjectInspector koi = moi.getMapKeyObjectInspector();
          ObjectInspector voi = moi.getMapValueObjectInspector();

          // \1 followed by each key and then each value
          Map<?, ?> map = moi.getMap(o);
          for (Map.Entry<?, ?> entry : map.entrySet()) {
            buffer.write((byte) 1, invert);
            serialize(buffer, entry.getKey(), koi, invert);
            serialize(buffer, entry.getValue(), voi, invert);
          }
          // and \0 to terminate
          buffer.write((byte) 0, invert);
          return;
        }
      case STRUCT:
        {
          StructObjectInspector soi = (StructObjectInspector) oi;
          List<? extends StructField> fields = soi.getAllStructFieldRefs();

          for (int i = 0; i < fields.size(); i++) {
            serialize(
                buffer,
                soi.getStructFieldData(o, fields.get(i)),
                fields.get(i).getFieldObjectInspector(),
                invert);
          }
          return;
        }
      case UNION:
        {
          UnionObjectInspector uoi = (UnionObjectInspector) oi;
          byte tag = uoi.getTag(o);
          buffer.write(tag, invert);
          serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert);
          return;
        }
      default:
        {
          throw new RuntimeException("Unrecognized type: " + oi.getCategory());
        }
    }
  }