コード例 #1
0
  private ObjectInspector solveOi(ObjectInspector arg) {

    switch (arg.getCategory()) {
      case PRIMITIVE:

        // VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
        // UNKNOWN
        PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
        return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
            poi.getPrimitiveCategory());
      case LIST:
        return ObjectInspectorFactory.getStandardListObjectInspector(
            solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
      case MAP:
        return ObjectInspectorFactory.getStandardMapObjectInspector(
            solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
            solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
      case STRUCT:
        StructObjectInspector soi = (StructObjectInspector) arg;
        int size = soi.getAllStructFieldRefs().size();
        ArrayList<String> fnl = new ArrayList<String>(size);
        ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);

        for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
          fnl.add(sf.getFieldName());
          foil.add(solveOi(sf.getFieldObjectInspector()));
        }

        return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
      default:
        return arg;
    }
  }
コード例 #2
0
ファイル: HiveUtils.java プロジェクト: naritta/hivemall
 public static PrimitiveObjectInspector asDoubleCompatibleOI(@Nonnull final ObjectInspector argOI)
     throws UDFArgumentTypeException {
   if (argOI.getCategory() != Category.PRIMITIVE) {
     throw new UDFArgumentTypeException(
         0,
         "Only primitive type arguments are accepted but " + argOI.getTypeName() + " is passed.");
   }
   final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI;
   switch (oi.getPrimitiveCategory()) {
     case BYTE:
     case SHORT:
     case INT:
     case LONG:
     case FLOAT:
     case DOUBLE:
     case STRING:
     case TIMESTAMP:
       break;
     default:
       throw new UDFArgumentTypeException(
           0,
           "Only numeric or string type arguments are accepted but "
               + argOI.getTypeName()
               + " is passed.");
   }
   return oi;
 }
コード例 #3
0
 @Override
 public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info)
     throws SemanticException {
   final ObjectInspector[] inspectors = info.getParameterObjectInspectors();
   if (inspectors.length != 1) {
     throw new UDFArgumentException("One argument expected");
   }
   if (inspectors[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
     throw new UDFArgumentTypeException(
         0, "Primitive argument expected, but " + inspectors[0].getTypeName() + " was recieved");
   }
   final PrimitiveObjectInspector inspector = (PrimitiveObjectInspector) inspectors[0];
   if (inspector.getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BINARY) {
     throw new UDFArgumentTypeException(
         0,
         "Binary argument expected, but "
             + inspector.getPrimitiveCategory().name()
             + " was received");
   }
   return createEvaluator();
 }
コード例 #4
0
ファイル: GenericUDFParamUtils.java プロジェクト: Leolh/hive
  public static void obtainBinaryConverter(
      ObjectInspector[] arguments, int i, PrimitiveCategory[] inputTypes, Converter[] converters)
      throws UDFArgumentTypeException {
    PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
    PrimitiveCategory inputType = inOi.getPrimitiveCategory();

    Converter converter =
        ObjectInspectorConverters.getConverter(
            arguments[i], PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
    converters[i] = converter;
    inputTypes[i] = inputType;
  }
コード例 #5
0
  /**
   * Serializes a field. Since we have nested structures, it may be called recursively for instance
   * when defining a list<struct<>>
   *
   * @param obj Object holding the fields' content
   * @param oi The field's objec inspector
   * @return the serialized object
   */
  Object serializeField(Object obj, ObjectInspector oi) {
    if (obj == null) {
      return null;
    }

    Object result = null;
    switch (oi.getCategory()) {
      case PRIMITIVE:
        PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
        switch (poi.getPrimitiveCategory()) {
          case VOID:
            result = null;
            break;
          case BOOLEAN:
            result = (((BooleanObjectInspector) poi).get(obj) ? Boolean.TRUE : Boolean.FALSE);
            break;
          case BYTE:
            result = (((ShortObjectInspector) poi).get(obj));
            break;
          case DOUBLE:
            result = (((DoubleObjectInspector) poi).get(obj));
            break;
          case FLOAT:
            result = (((FloatObjectInspector) poi).get(obj));
            break;
          case INT:
            result = (((IntObjectInspector) poi).get(obj));
            break;
          case LONG:
            result = (((LongObjectInspector) poi).get(obj));
            break;
          case SHORT:
            result = (((ShortObjectInspector) poi).get(obj));
            break;
          case STRING:
            result = (((StringObjectInspector) poi).getPrimitiveJavaObject(obj));
            break;
          case UNKNOWN:
            throw new RuntimeException("Unknown primitive");
        }
        break;
      case MAP:
        result = serializeMap(obj, (MapObjectInspector) oi);
        break;
      case LIST:
        result = serializeList(obj, (ListObjectInspector) oi);
        break;
      case STRUCT:
        result = serializeStruct(obj, (StructObjectInspector) oi, null);
        break;
    }
    return result;
  }
  private byte[] getConstantVal(Object writable, PrimitiveObjectInspector poi, boolean isKeyBinary)
      throws IOException {

    if (!isKeyBinary) {
      // Key is stored in text format. Get bytes representation of constant also of
      // text format.
      byte[] startRow;
      ByteStream.Output serializeStream = new ByteStream.Output();
      LazyUtils.writePrimitiveUTF8(serializeStream, writable, poi, false, (byte) 0, null);
      startRow = new byte[serializeStream.getLength()];
      System.arraycopy(serializeStream.getData(), 0, startRow, 0, serializeStream.getLength());
      return startRow;
    }

    PrimitiveCategory pc = poi.getPrimitiveCategory();
    switch (poi.getPrimitiveCategory()) {
      case INT:
        return Bytes.toBytes(((IntWritable) writable).get());
      case BOOLEAN:
        return Bytes.toBytes(((BooleanWritable) writable).get());
      case LONG:
        return Bytes.toBytes(((LongWritable) writable).get());
      case FLOAT:
        return Bytes.toBytes(((FloatWritable) writable).get());
      case DOUBLE:
        return Bytes.toBytes(((DoubleWritable) writable).get());
      case SHORT:
        return Bytes.toBytes(((ShortWritable) writable).get());
      case STRING:
        return Bytes.toBytes(((Text) writable).toString());
      case BYTE:
        return Bytes.toBytes(((ByteWritable) writable).get());

      default:
        throw new IOException("Type not supported " + pc);
    }
  }
コード例 #7
0
ファイル: LazyBinaryFactory.java プロジェクト: rohegde/hive
 /** Create a lazy binary primitive class given the type name. */
 public static LazyBinaryPrimitive<?, ?> createLazyBinaryPrimitiveClass(
     PrimitiveObjectInspector oi) {
   PrimitiveCategory p = oi.getPrimitiveCategory();
   switch (p) {
     case BOOLEAN:
       return new LazyBinaryBoolean((WritableBooleanObjectInspector) oi);
     case BYTE:
       return new LazyBinaryByte((WritableByteObjectInspector) oi);
     case SHORT:
       return new LazyBinaryShort((WritableShortObjectInspector) oi);
     case INT:
       return new LazyBinaryInteger((WritableIntObjectInspector) oi);
     case LONG:
       return new LazyBinaryLong((WritableLongObjectInspector) oi);
     case FLOAT:
       return new LazyBinaryFloat((WritableFloatObjectInspector) oi);
     case DOUBLE:
       return new LazyBinaryDouble((WritableDoubleObjectInspector) oi);
     case STRING:
       return new LazyBinaryString((WritableStringObjectInspector) oi);
     case CHAR:
       return new LazyBinaryHiveChar((WritableHiveCharObjectInspector) oi);
     case VARCHAR:
       return new LazyBinaryHiveVarchar((WritableHiveVarcharObjectInspector) oi);
     case VOID: // for NULL
       return new LazyBinaryVoid((WritableVoidObjectInspector) oi);
     case DATE:
       return new LazyBinaryDate((WritableDateObjectInspector) oi);
     case TIMESTAMP:
       return new LazyBinaryTimestamp((WritableTimestampObjectInspector) oi);
     case BINARY:
       return new LazyBinaryBinary((WritableBinaryObjectInspector) oi);
     case DECIMAL:
       return new LazyBinaryHiveDecimal((WritableHiveDecimalObjectInspector) oi);
     default:
       throw new RuntimeException("Internal error: no LazyBinaryObject for " + p);
   }
 }
コード例 #8
0
ファイル: SerDeUtils.java プロジェクト: nileema/presto
  private static void serializePrimitive(
      Type type, BlockBuilder builder, Object object, PrimitiveObjectInspector inspector) {
    requireNonNull(builder, "parent builder is null");

    if (object == null) {
      builder.appendNull();
      return;
    }

    switch (inspector.getPrimitiveCategory()) {
      case BOOLEAN:
        BooleanType.BOOLEAN.writeBoolean(builder, ((BooleanObjectInspector) inspector).get(object));
        return;
      case BYTE:
        TinyintType.TINYINT.writeLong(builder, ((ByteObjectInspector) inspector).get(object));
        return;
      case SHORT:
        SmallintType.SMALLINT.writeLong(builder, ((ShortObjectInspector) inspector).get(object));
        return;
      case INT:
        IntegerType.INTEGER.writeLong(builder, ((IntObjectInspector) inspector).get(object));
        return;
      case LONG:
        BigintType.BIGINT.writeLong(builder, ((LongObjectInspector) inspector).get(object));
        return;
      case FLOAT:
        DoubleType.DOUBLE.writeDouble(builder, ((FloatObjectInspector) inspector).get(object));
        return;
      case DOUBLE:
        DoubleType.DOUBLE.writeDouble(builder, ((DoubleObjectInspector) inspector).get(object));
        return;
      case STRING:
        type.writeSlice(
            builder,
            Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object)));
        return;
      case VARCHAR:
        type.writeSlice(
            builder,
            Slices.utf8Slice(
                ((HiveVarcharObjectInspector) inspector)
                    .getPrimitiveJavaObject(object)
                    .getValue()));
        return;
      case CHAR:
        CharType charType = checkType(type, CharType.class, "type");
        HiveChar hiveChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(object);
        type.writeSlice(
            builder,
            trimSpacesAndTruncateToLength(
                Slices.utf8Slice(hiveChar.getValue()), charType.getLength()));
        return;
      case DATE:
        DateType.DATE.writeLong(builder, formatDateAsLong(object, (DateObjectInspector) inspector));
        return;
      case TIMESTAMP:
        TimestampType.TIMESTAMP.writeLong(
            builder, formatTimestampAsLong(object, (TimestampObjectInspector) inspector));
        return;
      case BINARY:
        VARBINARY.writeSlice(
            builder,
            Slices.wrappedBuffer(
                ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object)));
        return;
      case DECIMAL:
        DecimalType decimalType = checkType(type, DecimalType.class, "type");
        HiveDecimalWritable hiveDecimal =
            ((HiveDecimalObjectInspector) inspector).getPrimitiveWritableObject(object);
        if (decimalType.isShort()) {
          decimalType.writeLong(
              builder, DecimalUtils.getShortDecimalValue(hiveDecimal, decimalType.getScale()));
        } else {
          decimalType.writeSlice(
              builder, DecimalUtils.getLongDecimalValue(hiveDecimal, decimalType.getScale()));
        }
        return;
    }
    throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory());
  }
コード例 #9
0
 private Writable createPrimitive(final Object obj, final PrimitiveObjectInspector inspector)
     throws SerDeException {
   if (obj == null) {
     return null;
   }
   switch (inspector.getPrimitiveCategory()) {
     case VOID:
       return null;
     case BOOLEAN:
       return new BooleanWritable(
           ((BooleanObjectInspector) inspector).get(obj) ? Boolean.TRUE : Boolean.FALSE);
     case BYTE:
       return new ByteWritable(((ByteObjectInspector) inspector).get(obj));
     case DOUBLE:
       return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj));
     case FLOAT:
       return new FloatWritable(((FloatObjectInspector) inspector).get(obj));
     case INT:
       return new IntWritable(((IntObjectInspector) inspector).get(obj));
     case LONG:
       return new LongWritable(((LongObjectInspector) inspector).get(obj));
     case SHORT:
       return new ShortWritable(((ShortObjectInspector) inspector).get(obj));
     case STRING:
       String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(obj);
       try {
         return new BytesWritable(v.getBytes("UTF-8"));
       } catch (UnsupportedEncodingException e) {
         throw new SerDeException("Failed to encode string in UTF-8", e);
       }
     case DECIMAL:
       HiveDecimal hd = (HiveDecimal) inspector.getPrimitiveJavaObject(obj);
       DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
       int prec = decTypeInfo.precision();
       int scale = decTypeInfo.scale();
       byte[] src = hd.setScale(scale).unscaledValue().toByteArray();
       // Estimated number of bytes needed.
       int bytes = PRECISION_TO_BYTE_COUNT[prec - 1];
       if (bytes == src.length) {
         // No padding needed.
         return new BytesWritable(src);
       }
       byte[] tgt = new byte[bytes];
       if (hd.signum() == -1) {
         // For negative number, initializing bits to 1
         for (int i = 0; i < bytes; i++) {
           tgt[i] |= 0xFF;
         }
       }
       System.arraycopy(
           src, 0, tgt, bytes - src.length, src.length); // Padding leading zeroes/ones.
       return new BytesWritable(tgt);
     case TIMESTAMP:
       return new TimestampWritable(
           ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(obj));
     case CHAR:
       String strippedValue =
           ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(obj).getStrippedValue();
       return new BytesWritable(Binary.fromString(strippedValue).getBytes());
     case VARCHAR:
       String value =
           ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(obj).getValue();
       return new BytesWritable(Binary.fromString(value).getBytes());
     case BINARY:
       return new BytesWritable(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(obj));
     default:
       throw new SerDeException("Unknown primitive : " + inspector.getPrimitiveCategory());
   }
 }
コード例 #10
0
 public static VectorColumnAssign buildObjectAssign(
     VectorizedRowBatch outputBatch, int outColIndex, ObjectInspector objInspector)
     throws HiveException {
   PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector;
   VectorColumnAssign outVCA = null;
   ColumnVector destCol = outputBatch.cols[outColIndex];
   if (destCol instanceof LongColumnVector) {
     switch (poi.getPrimitiveCategory()) {
       case BOOLEAN:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   BooleanWritable bw = (BooleanWritable) val;
                   assignLong(bw.get() ? 1 : 0, destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case BYTE:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   ByteWritable bw = (ByteWritable) val;
                   assignLong(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case SHORT:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   ShortWritable bw = (ShortWritable) val;
                   assignLong(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case INT:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   IntWritable bw = (IntWritable) val;
                   assignLong(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case LONG:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   LongWritable bw = (LongWritable) val;
                   assignLong(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case TIMESTAMP:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   TimestampWritable bw = (TimestampWritable) val;
                   Timestamp t = bw.getTimestamp();
                   assignLong(TimestampUtils.getTimeNanoSec(t), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case DATE:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   DateWritable bw = (DateWritable) val;
                   assignLong(bw.getDays(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       default:
         throw new HiveException(
             "Incompatible Long vector column and primitive category "
                 + poi.getPrimitiveCategory());
     }
   } else if (destCol instanceof DoubleColumnVector) {
     switch (poi.getPrimitiveCategory()) {
       case DOUBLE:
         outVCA =
             new VectorDoubleColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   DoubleWritable bw = (DoubleWritable) val;
                   assignDouble(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (DoubleColumnVector) destCol);
         break;
       case FLOAT:
         outVCA =
             new VectorDoubleColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   FloatWritable bw = (FloatWritable) val;
                   assignDouble(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (DoubleColumnVector) destCol);
         break;
       default:
         throw new HiveException(
             "Incompatible Double vector column and primitive category "
                 + poi.getPrimitiveCategory());
     }
   } else if (destCol instanceof BytesColumnVector) {
     switch (poi.getPrimitiveCategory()) {
       case STRING:
         outVCA =
             new VectorBytesColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   Text bw = (Text) val;
                   byte[] bytes = bw.getBytes();
                   assignBytes(bytes, 0, bytes.length, destIndex);
                 }
               }
             }.init(outputBatch, (BytesColumnVector) destCol);
         break;
       default:
         throw new HiveException(
             "Incompatible Bytes vector column and primitive category "
                 + poi.getPrimitiveCategory());
     }
   } else if (destCol instanceof DecimalColumnVector) {
     switch (poi.getPrimitiveCategory()) {
       case DECIMAL:
         outVCA =
             new VectorDecimalColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   HiveDecimalWritable hdw = (HiveDecimalWritable) val;
                   assignDecimal(hdw, destIndex);
                 }
               }
             }.init(outputBatch, (DecimalColumnVector) destCol);
         break;
       default:
         throw new HiveException(
             "Incompatible Decimal vector column and primitive category "
                 + poi.getPrimitiveCategory());
     }
   } else {
     throw new HiveException("Unknown vector column type " + destCol.getClass().getName());
   }
   return outVCA;
 }
コード例 #11
0
ファイル: SerDeUtils.java プロジェクト: jnthm/hive
  static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {

    switch (oi.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            switch (poi.getPrimitiveCategory()) {
              case BOOLEAN:
                {
                  boolean b = ((BooleanObjectInspector) poi).get(o);
                  sb.append(b ? "true" : "false");
                  break;
                }
              case BYTE:
                {
                  sb.append(((ByteObjectInspector) poi).get(o));
                  break;
                }
              case SHORT:
                {
                  sb.append(((ShortObjectInspector) poi).get(o));
                  break;
                }
              case INT:
                {
                  sb.append(((IntObjectInspector) poi).get(o));
                  break;
                }
              case LONG:
                {
                  sb.append(((LongObjectInspector) poi).get(o));
                  break;
                }
              case FLOAT:
                {
                  sb.append(((FloatObjectInspector) poi).get(o));
                  break;
                }
              case DOUBLE:
                {
                  sb.append(((DoubleObjectInspector) poi).get(o));
                  break;
                }
              case STRING:
                {
                  sb.append('"');
                  sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
                  sb.append('"');
                  break;
                }
              case CHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case VARCHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case DATE:
                {
                  sb.append('"');
                  sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case TIMESTAMP:
                {
                  sb.append('"');
                  sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case BINARY:
                {
                  BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
                  Text txt = new Text();
                  txt.set(bw.getBytes(), 0, bw.getLength());
                  sb.append(txt.toString());
                  break;
                }
              case DECIMAL:
                {
                  sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
                  break;
                }
              default:
                throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
            }
          }
          break;
        }
      case LIST:
        {
          ListObjectInspector loi = (ListObjectInspector) oi;
          ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
          List<?> olist = loi.getList(o);
          if (olist == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACKET);
            for (int i = 0; i < olist.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
            }
            sb.append(RBRACKET);
          }
          break;
        }
      case MAP:
        {
          MapObjectInspector moi = (MapObjectInspector) oi;
          ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
          ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
          Map<?, ?> omap = moi.getMap(o);
          if (omap == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            boolean first = true;
            for (Object entry : omap.entrySet()) {
              if (first) {
                first = false;
              } else {
                sb.append(COMMA);
              }
              Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
              buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
              sb.append(COLON);
              buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case STRUCT:
        {
          StructObjectInspector soi = (StructObjectInspector) oi;
          List<? extends StructField> structFields = soi.getAllStructFieldRefs();
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            for (int i = 0; i < structFields.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              sb.append(QUOTE);
              sb.append(structFields.get(i).getFieldName());
              sb.append(QUOTE);
              sb.append(COLON);
              buildJSONString(
                  sb,
                  soi.getStructFieldData(o, structFields.get(i)),
                  structFields.get(i).getFieldObjectInspector(),
                  JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case UNION:
        {
          UnionObjectInspector uoi = (UnionObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            sb.append(uoi.getTag(o));
            sb.append(COLON);
            buildJSONString(
                sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
            sb.append(RBRACE);
          }
          break;
        }
      default:
        throw new RuntimeException("Unknown type in ObjectInspector!");
    }
  }
コード例 #12
0
  static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, boolean invert) {
    // Is this field a null?
    if (o == null) {
      buffer.write((byte) 0, invert);
      return;
    }
    // This field is not a null.
    buffer.write((byte) 1, invert);

    switch (oi.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
          switch (poi.getPrimitiveCategory()) {
            case VOID:
              {
                return;
              }
            case BOOLEAN:
              {
                boolean v = ((BooleanObjectInspector) poi).get(o);
                buffer.write((byte) (v ? 2 : 1), invert);
                return;
              }
            case BYTE:
              {
                ByteObjectInspector boi = (ByteObjectInspector) poi;
                byte v = boi.get(o);
                buffer.write((byte) (v ^ 0x80), invert);
                return;
              }
            case SHORT:
              {
                ShortObjectInspector spoi = (ShortObjectInspector) poi;
                short v = spoi.get(o);
                buffer.write((byte) ((v >> 8) ^ 0x80), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case INT:
              {
                IntObjectInspector ioi = (IntObjectInspector) poi;
                int v = ioi.get(o);
                buffer.write((byte) ((v >> 24) ^ 0x80), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case LONG:
              {
                LongObjectInspector loi = (LongObjectInspector) poi;
                long v = loi.get(o);
                buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case FLOAT:
              {
                FloatObjectInspector foi = (FloatObjectInspector) poi;
                int v = Float.floatToIntBits(foi.get(o));
                if ((v & (1 << 31)) != 0) {
                  // negative number, flip all bits
                  v = ~v;
                } else {
                  // positive number, flip the first bit
                  v = v ^ (1 << 31);
                }
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case DOUBLE:
              {
                DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                long v = Double.doubleToLongBits(doi.get(o));
                if ((v & (1L << 63)) != 0) {
                  // negative number, flip all bits
                  v = ~v;
                } else {
                  // positive number, flip the first bit
                  v = v ^ (1L << 63);
                }
                buffer.write((byte) (v >> 56), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case STRING:
              {
                StringObjectInspector soi = (StringObjectInspector) poi;
                Text t = soi.getPrimitiveWritableObject(o);
                serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                return;
              }

            case BINARY:
              {
                BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
                BytesWritable ba = baoi.getPrimitiveWritableObject(o);
                byte[] toSer = new byte[ba.getLength()];
                System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
                serializeBytes(buffer, toSer, ba.getLength(), invert);
                return;
              }
            case DATE:
              {
                DateObjectInspector doi = (DateObjectInspector) poi;
                long v = doi.getPrimitiveWritableObject(o).getTimeInSeconds();
                buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case TIMESTAMP:
              {
                TimestampObjectInspector toi = (TimestampObjectInspector) poi;
                TimestampWritable t = toi.getPrimitiveWritableObject(o);
                byte[] data = t.getBinarySortable();
                for (int i = 0; i < data.length; i++) {
                  buffer.write(data[i], invert);
                }
                return;
              }
            default:
              {
                throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
              }
          }
        }
      case LIST:
        {
          ListObjectInspector loi = (ListObjectInspector) oi;
          ObjectInspector eoi = loi.getListElementObjectInspector();

          // \1 followed by each element
          int size = loi.getListLength(o);
          for (int eid = 0; eid < size; eid++) {
            buffer.write((byte) 1, invert);
            serialize(buffer, loi.getListElement(o, eid), eoi, invert);
          }
          // and \0 to terminate
          buffer.write((byte) 0, invert);
          return;
        }
      case MAP:
        {
          MapObjectInspector moi = (MapObjectInspector) oi;
          ObjectInspector koi = moi.getMapKeyObjectInspector();
          ObjectInspector voi = moi.getMapValueObjectInspector();

          // \1 followed by each key and then each value
          Map<?, ?> map = moi.getMap(o);
          for (Map.Entry<?, ?> entry : map.entrySet()) {
            buffer.write((byte) 1, invert);
            serialize(buffer, entry.getKey(), koi, invert);
            serialize(buffer, entry.getValue(), voi, invert);
          }
          // and \0 to terminate
          buffer.write((byte) 0, invert);
          return;
        }
      case STRUCT:
        {
          StructObjectInspector soi = (StructObjectInspector) oi;
          List<? extends StructField> fields = soi.getAllStructFieldRefs();

          for (int i = 0; i < fields.size(); i++) {
            serialize(
                buffer,
                soi.getStructFieldData(o, fields.get(i)),
                fields.get(i).getFieldObjectInspector(),
                invert);
          }
          return;
        }
      case UNION:
        {
          UnionObjectInspector uoi = (UnionObjectInspector) oi;
          byte tag = uoi.getTag(o);
          buffer.write(tag, invert);
          serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert);
          return;
        }
      default:
        {
          throw new RuntimeException("Unrecognized type: " + oi.getCategory());
        }
    }
  }