Ejemplo n.º 1
0
 public static PrimitiveObjectInspector asDoubleCompatibleOI(@Nonnull final ObjectInspector argOI)
     throws UDFArgumentTypeException {
   if (argOI.getCategory() != Category.PRIMITIVE) {
     throw new UDFArgumentTypeException(
         0,
         "Only primitive type arguments are accepted but " + argOI.getTypeName() + " is passed.");
   }
   final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI;
   switch (oi.getPrimitiveCategory()) {
     case BYTE:
     case SHORT:
     case INT:
     case LONG:
     case FLOAT:
     case DOUBLE:
     case STRING:
     case TIMESTAMP:
       break;
     default:
       throw new UDFArgumentTypeException(
           0,
           "Only numeric or string type arguments are accepted but "
               + argOI.getTypeName()
               + " is passed.");
   }
   return oi;
 }
Ejemplo n.º 2
0
  @Override
  public Writable serialize(Object obj, ObjectInspector oi) throws SerDeException {
    if (oi.getCategory() != Category.STRUCT) {
      throw new VoltSerdeException(
          getClass().toString()
              + " can only serialize struct types, but we got: "
              + oi.getTypeName());
    }
    VoltRecord vr = new VoltRecord(m_voltConf.getTableName());
    StructObjectInspector soi = (StructObjectInspector) oi;
    List<? extends StructField> structFields = soi.getAllStructFieldRefs();
    List<Object> fieldValues = soi.getStructFieldsDataAsList(obj);

    final int size = m_oig.getColumnTypes().size();

    for (int i = 0; i < size; ++i) {
      ObjectInspector fieldOI = structFields.get(i).getFieldObjectInspector();
      PrimitiveObjectInspector poi = (PrimitiveObjectInspector) fieldOI;

      Object fieldValue = poi.getPrimitiveJavaObject(fieldValues.get(i));
      if (poi.getTypeInfo().equals(TypeInfoFactory.timestampTypeInfo)) {
        fieldValue = fieldValue != null ? new Date(((Timestamp) fieldValue).getTime()) : null;
      }
      vr.add(fieldValue);
    }

    return vr;
  }
  private ObjectInspector solveOi(ObjectInspector arg) {

    switch (arg.getCategory()) {
      case PRIMITIVE:

        // VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
        // UNKNOWN
        PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
        return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
            poi.getPrimitiveCategory());
      case LIST:
        return ObjectInspectorFactory.getStandardListObjectInspector(
            solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
      case MAP:
        return ObjectInspectorFactory.getStandardMapObjectInspector(
            solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
            solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
      case STRUCT:
        StructObjectInspector soi = (StructObjectInspector) arg;
        int size = soi.getAllStructFieldRefs().size();
        ArrayList<String> fnl = new ArrayList<String>(size);
        ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);

        for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
          fnl.add(sf.getFieldName());
          foil.add(solveOi(sf.getFieldObjectInspector()));
        }

        return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
      default:
        return arg;
    }
  }
Ejemplo n.º 4
0
 private void stringifyObject(StringBuilder buffer, Object obj, ObjectInspector inspector)
     throws IOException {
   if (inspector instanceof StructObjectInspector) {
     buffer.append("{ ");
     StructObjectInspector soi = (StructObjectInspector) inspector;
     boolean isFirst = true;
     for (StructField field : soi.getAllStructFieldRefs()) {
       if (isFirst) {
         isFirst = false;
       } else {
         buffer.append(", ");
       }
       buffer.append(field.getFieldName());
       buffer.append(": ");
       stringifyObject(
           buffer, soi.getStructFieldData(obj, field), field.getFieldObjectInspector());
     }
     buffer.append(" }");
   } else if (inspector instanceof PrimitiveObjectInspector) {
     PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
     buffer.append(poi.getPrimitiveJavaObject(obj).toString());
   } else {
     buffer.append("*unknown*");
   }
 }
Ejemplo n.º 5
0
 public static LazySimpleSerDe getKeyValueLineSerde(
     @Nonnull final PrimitiveObjectInspector keyOI,
     @Nonnull final PrimitiveObjectInspector valueOI)
     throws SerDeException {
   LazySimpleSerDe serde = new LazySimpleSerDe();
   Configuration conf = new Configuration();
   Properties tbl = new Properties();
   tbl.setProperty("columns", "key,value");
   tbl.setProperty("columns.types", keyOI.getTypeName() + "," + valueOI.getTypeName());
   serde.initialize(conf, tbl);
   return serde;
 }
Ejemplo n.º 6
0
  /**
   * Serializes a field. Since we have nested structures, it may be called recursively for instance
   * when defining a list<struct<>>
   *
   * @param obj Object holding the fields' content
   * @param oi The field's objec inspector
   * @return the serialized object
   */
  Object serializeField(Object obj, ObjectInspector oi) {
    if (obj == null) {
      return null;
    }

    Object result = null;
    switch (oi.getCategory()) {
      case PRIMITIVE:
        PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
        switch (poi.getPrimitiveCategory()) {
          case VOID:
            result = null;
            break;
          case BOOLEAN:
            result = (((BooleanObjectInspector) poi).get(obj) ? Boolean.TRUE : Boolean.FALSE);
            break;
          case BYTE:
            result = (((ShortObjectInspector) poi).get(obj));
            break;
          case DOUBLE:
            result = (((DoubleObjectInspector) poi).get(obj));
            break;
          case FLOAT:
            result = (((FloatObjectInspector) poi).get(obj));
            break;
          case INT:
            result = (((IntObjectInspector) poi).get(obj));
            break;
          case LONG:
            result = (((LongObjectInspector) poi).get(obj));
            break;
          case SHORT:
            result = (((ShortObjectInspector) poi).get(obj));
            break;
          case STRING:
            result = (((StringObjectInspector) poi).getPrimitiveJavaObject(obj));
            break;
          case UNKNOWN:
            throw new RuntimeException("Unknown primitive");
        }
        break;
      case MAP:
        result = serializeMap(obj, (MapObjectInspector) oi);
        break;
      case LIST:
        result = serializeList(obj, (ListObjectInspector) oi);
        break;
      case STRUCT:
        result = serializeStruct(obj, (StructObjectInspector) oi, null);
        break;
    }
    return result;
  }
Ejemplo n.º 7
0
  public static void obtainBinaryConverter(
      ObjectInspector[] arguments, int i, PrimitiveCategory[] inputTypes, Converter[] converters)
      throws UDFArgumentTypeException {
    PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
    PrimitiveCategory inputType = inOi.getPrimitiveCategory();

    Converter converter =
        ObjectInspectorConverters.getConverter(
            arguments[i], PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
    converters[i] = converter;
    inputTypes[i] = inputType;
  }
Ejemplo n.º 8
0
  @Test
  public void testDecimalPlusDecimalSameParams() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(5, 2)),
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(5, 2))
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(6, 2), oi.getTypeInfo());
  }
Ejemplo n.º 9
0
  @Test
  public void testFloatPlusFloat() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Float
    FloatWritable f1 = new FloatWritable(4.5f);
    FloatWritable f2 = new FloatWritable(0.0f);
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableFloatObjectInspector,
      PrimitiveObjectInspectorFactory.writableFloatObjectInspector
    };
    DeferredObject[] args = {
      new DeferredJavaObject(f1), new DeferredJavaObject(f2),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(oi.getTypeInfo(), TypeInfoFactory.floatTypeInfo);
    FloatWritable res = (FloatWritable) udf.evaluate(args);
    Assert.assertEquals(new Float(4.5), new Float(res.get()));
  }
Ejemplo n.º 10
0
  @Test
  public void testDatePlusIntervalYearMonth() throws Exception {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    DateWritable left = new DateWritable(Date.valueOf("2001-06-15"));
    HiveIntervalYearMonthWritable right =
        new HiveIntervalYearMonthWritable(HiveIntervalYearMonth.valueOf("2-8"));
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableDateObjectInspector,
      PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.dateTypeInfo, oi.getTypeInfo());
    DateWritable res = (DateWritable) udf.evaluate(args);
    Assert.assertEquals(Date.valueOf("2004-02-15"), res.get());
  }
Ejemplo n.º 11
0
  @Test
  public void testBytePlusShort() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Byte
    ByteWritable left = new ByteWritable((byte) 4);
    ShortWritable right = new ShortWritable((short) 6);
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableByteObjectInspector,
      PrimitiveObjectInspectorFactory.writableShortObjectInspector
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(oi.getTypeInfo(), TypeInfoFactory.shortTypeInfo);
    ShortWritable res = (ShortWritable) udf.evaluate(args);
    Assert.assertEquals(10, res.get());
  }
Ejemplo n.º 12
0
  @Test
  public void testDoublePlusLong() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Int
    DoubleWritable left = new DoubleWritable(4.5);
    LongWritable right = new LongWritable(10);
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
      PrimitiveObjectInspectorFactory.writableLongObjectInspector
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.doubleTypeInfo, oi.getTypeInfo());
    DoubleWritable res = (DoubleWritable) udf.evaluate(args);
    Assert.assertEquals(new Double(14.5), new Double(res.get()));
  }
Ejemplo n.º 13
0
  @Test
  public void testTimestampPlusIntervalDayTime() throws Exception {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    TimestampWritable left = new TimestampWritable(Timestamp.valueOf("2001-01-01 00:00:00"));
    HiveIntervalDayTimeWritable right =
        new HiveIntervalDayTimeWritable(HiveIntervalDayTime.valueOf("1 2:3:4.567"));
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableTimestampObjectInspector,
      PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo());
    TimestampWritable res = (TimestampWritable) udf.evaluate(args);
    Assert.assertEquals(Timestamp.valueOf("2001-01-02 2:3:4.567"), res.getTimestamp());
  }
 @Override
 public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info)
     throws SemanticException {
   final ObjectInspector[] inspectors = info.getParameterObjectInspectors();
   if (inspectors.length != 1) {
     throw new UDFArgumentException("One argument expected");
   }
   if (inspectors[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
     throw new UDFArgumentTypeException(
         0, "Primitive argument expected, but " + inspectors[0].getTypeName() + " was recieved");
   }
   final PrimitiveObjectInspector inspector = (PrimitiveObjectInspector) inspectors[0];
   if (inspector.getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BINARY) {
     throw new UDFArgumentTypeException(
         0,
         "Binary argument expected, but "
             + inspector.getPrimitiveCategory().name()
             + " was received");
   }
   return createEvaluator();
 }
Ejemplo n.º 15
0
  @Test
  public void testLongPlusDecimal() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Long
    LongWritable left = new LongWritable(104);
    HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableLongObjectInspector,
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(9, 4))
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(24, 4), oi.getTypeInfo());
    HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args);
    Assert.assertEquals(HiveDecimal.create("338.97"), res.getHiveDecimal());
  }
Ejemplo n.º 16
0
  @Test
  public void testDoulePlusDecimal() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Double
    DoubleWritable left = new DoubleWritable(74.52);
    HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(5, 2))
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.doubleTypeInfo, oi.getTypeInfo());
    DoubleWritable res = (DoubleWritable) udf.evaluate(args);
    Assert.assertEquals(new Double(309.49), new Double(res.get()));
  }
Ejemplo n.º 17
0
  @Test
  public void testVarcharPlusInt() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Short
    HiveVarcharWritable left = new HiveVarcharWritable();
    left.set("123");
    IntWritable right = new IntWritable(456);
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector,
      PrimitiveObjectInspectorFactory.writableIntObjectInspector
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(oi.getTypeInfo(), TypeInfoFactory.doubleTypeInfo);
    DoubleWritable res = (DoubleWritable) udf.evaluate(args);
    Assert.assertEquals(new Double(579.0), new Double(res.get()));
  }
Ejemplo n.º 18
0
  @Test
  public void testTimestampPlusIntervalYearMonth() throws Exception {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    TimestampWritable left =
        new TimestampWritable(Timestamp.valueOf("2001-11-15 01:02:03.123456789"));
    HiveIntervalYearMonthWritable right =
        new HiveIntervalYearMonthWritable(HiveIntervalYearMonth.valueOf("2-2"));
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableTimestampObjectInspector,
      PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo());
    TimestampWritable res = (TimestampWritable) udf.evaluate(args);
    Assert.assertEquals(Timestamp.valueOf("2004-01-15 01:02:03.123456789"), res.getTimestamp());
  }
Ejemplo n.º 19
0
  @Test
  public void testDecimalPlusDecimal() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Decimal
    HiveDecimalWritable left = new HiveDecimalWritable(HiveDecimal.create("14.5"));
    HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(3, 1)),
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(5, 2))
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(6, 2), oi.getTypeInfo());
    HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args);
    Assert.assertEquals(HiveDecimal.create("249.47"), res.getHiveDecimal());
  }
  private byte[] getConstantVal(Object writable, PrimitiveObjectInspector poi, boolean isKeyBinary)
      throws IOException {

    if (!isKeyBinary) {
      // Key is stored in text format. Get bytes representation of constant also of
      // text format.
      byte[] startRow;
      ByteStream.Output serializeStream = new ByteStream.Output();
      LazyUtils.writePrimitiveUTF8(serializeStream, writable, poi, false, (byte) 0, null);
      startRow = new byte[serializeStream.getLength()];
      System.arraycopy(serializeStream.getData(), 0, startRow, 0, serializeStream.getLength());
      return startRow;
    }

    PrimitiveCategory pc = poi.getPrimitiveCategory();
    switch (poi.getPrimitiveCategory()) {
      case INT:
        return Bytes.toBytes(((IntWritable) writable).get());
      case BOOLEAN:
        return Bytes.toBytes(((BooleanWritable) writable).get());
      case LONG:
        return Bytes.toBytes(((LongWritable) writable).get());
      case FLOAT:
        return Bytes.toBytes(((FloatWritable) writable).get());
      case DOUBLE:
        return Bytes.toBytes(((DoubleWritable) writable).get());
      case SHORT:
        return Bytes.toBytes(((ShortWritable) writable).get());
      case STRING:
        return Bytes.toBytes(((Text) writable).toString());
      case BYTE:
        return Bytes.toBytes(((ByteWritable) writable).get());

      default:
        throw new IOException("Type not supported " + pc);
    }
  }
Ejemplo n.º 21
0
 /** Create a lazy binary primitive class given the type name. */
 public static LazyBinaryPrimitive<?, ?> createLazyBinaryPrimitiveClass(
     PrimitiveObjectInspector oi) {
   PrimitiveCategory p = oi.getPrimitiveCategory();
   switch (p) {
     case BOOLEAN:
       return new LazyBinaryBoolean((WritableBooleanObjectInspector) oi);
     case BYTE:
       return new LazyBinaryByte((WritableByteObjectInspector) oi);
     case SHORT:
       return new LazyBinaryShort((WritableShortObjectInspector) oi);
     case INT:
       return new LazyBinaryInteger((WritableIntObjectInspector) oi);
     case LONG:
       return new LazyBinaryLong((WritableLongObjectInspector) oi);
     case FLOAT:
       return new LazyBinaryFloat((WritableFloatObjectInspector) oi);
     case DOUBLE:
       return new LazyBinaryDouble((WritableDoubleObjectInspector) oi);
     case STRING:
       return new LazyBinaryString((WritableStringObjectInspector) oi);
     case CHAR:
       return new LazyBinaryHiveChar((WritableHiveCharObjectInspector) oi);
     case VARCHAR:
       return new LazyBinaryHiveVarchar((WritableHiveVarcharObjectInspector) oi);
     case VOID: // for NULL
       return new LazyBinaryVoid((WritableVoidObjectInspector) oi);
     case DATE:
       return new LazyBinaryDate((WritableDateObjectInspector) oi);
     case TIMESTAMP:
       return new LazyBinaryTimestamp((WritableTimestampObjectInspector) oi);
     case BINARY:
       return new LazyBinaryBinary((WritableBinaryObjectInspector) oi);
     case DECIMAL:
       return new LazyBinaryHiveDecimal((WritableHiveDecimalObjectInspector) oi);
     default:
       throw new RuntimeException("Internal error: no LazyBinaryObject for " + p);
   }
 }
  static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, boolean invert) {
    // Is this field a null?
    if (o == null) {
      buffer.write((byte) 0, invert);
      return;
    }
    // This field is not a null.
    buffer.write((byte) 1, invert);

    switch (oi.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
          switch (poi.getPrimitiveCategory()) {
            case VOID:
              {
                return;
              }
            case BOOLEAN:
              {
                boolean v = ((BooleanObjectInspector) poi).get(o);
                buffer.write((byte) (v ? 2 : 1), invert);
                return;
              }
            case BYTE:
              {
                ByteObjectInspector boi = (ByteObjectInspector) poi;
                byte v = boi.get(o);
                buffer.write((byte) (v ^ 0x80), invert);
                return;
              }
            case SHORT:
              {
                ShortObjectInspector spoi = (ShortObjectInspector) poi;
                short v = spoi.get(o);
                buffer.write((byte) ((v >> 8) ^ 0x80), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case INT:
              {
                IntObjectInspector ioi = (IntObjectInspector) poi;
                int v = ioi.get(o);
                buffer.write((byte) ((v >> 24) ^ 0x80), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case LONG:
              {
                LongObjectInspector loi = (LongObjectInspector) poi;
                long v = loi.get(o);
                buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case FLOAT:
              {
                FloatObjectInspector foi = (FloatObjectInspector) poi;
                int v = Float.floatToIntBits(foi.get(o));
                if ((v & (1 << 31)) != 0) {
                  // negative number, flip all bits
                  v = ~v;
                } else {
                  // positive number, flip the first bit
                  v = v ^ (1 << 31);
                }
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case DOUBLE:
              {
                DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                long v = Double.doubleToLongBits(doi.get(o));
                if ((v & (1L << 63)) != 0) {
                  // negative number, flip all bits
                  v = ~v;
                } else {
                  // positive number, flip the first bit
                  v = v ^ (1L << 63);
                }
                buffer.write((byte) (v >> 56), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case STRING:
              {
                StringObjectInspector soi = (StringObjectInspector) poi;
                Text t = soi.getPrimitiveWritableObject(o);
                serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                return;
              }

            case BINARY:
              {
                BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
                BytesWritable ba = baoi.getPrimitiveWritableObject(o);
                byte[] toSer = new byte[ba.getLength()];
                System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
                serializeBytes(buffer, toSer, ba.getLength(), invert);
                return;
              }
            case DATE:
              {
                DateObjectInspector doi = (DateObjectInspector) poi;
                long v = doi.getPrimitiveWritableObject(o).getTimeInSeconds();
                buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                buffer.write((byte) (v >> 48), invert);
                buffer.write((byte) (v >> 40), invert);
                buffer.write((byte) (v >> 32), invert);
                buffer.write((byte) (v >> 24), invert);
                buffer.write((byte) (v >> 16), invert);
                buffer.write((byte) (v >> 8), invert);
                buffer.write((byte) v, invert);
                return;
              }
            case TIMESTAMP:
              {
                TimestampObjectInspector toi = (TimestampObjectInspector) poi;
                TimestampWritable t = toi.getPrimitiveWritableObject(o);
                byte[] data = t.getBinarySortable();
                for (int i = 0; i < data.length; i++) {
                  buffer.write(data[i], invert);
                }
                return;
              }
            default:
              {
                throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
              }
          }
        }
      case LIST:
        {
          ListObjectInspector loi = (ListObjectInspector) oi;
          ObjectInspector eoi = loi.getListElementObjectInspector();

          // \1 followed by each element
          int size = loi.getListLength(o);
          for (int eid = 0; eid < size; eid++) {
            buffer.write((byte) 1, invert);
            serialize(buffer, loi.getListElement(o, eid), eoi, invert);
          }
          // and \0 to terminate
          buffer.write((byte) 0, invert);
          return;
        }
      case MAP:
        {
          MapObjectInspector moi = (MapObjectInspector) oi;
          ObjectInspector koi = moi.getMapKeyObjectInspector();
          ObjectInspector voi = moi.getMapValueObjectInspector();

          // \1 followed by each key and then each value
          Map<?, ?> map = moi.getMap(o);
          for (Map.Entry<?, ?> entry : map.entrySet()) {
            buffer.write((byte) 1, invert);
            serialize(buffer, entry.getKey(), koi, invert);
            serialize(buffer, entry.getValue(), voi, invert);
          }
          // and \0 to terminate
          buffer.write((byte) 0, invert);
          return;
        }
      case STRUCT:
        {
          StructObjectInspector soi = (StructObjectInspector) oi;
          List<? extends StructField> fields = soi.getAllStructFieldRefs();

          for (int i = 0; i < fields.size(); i++) {
            serialize(
                buffer,
                soi.getStructFieldData(o, fields.get(i)),
                fields.get(i).getFieldObjectInspector(),
                invert);
          }
          return;
        }
      case UNION:
        {
          UnionObjectInspector uoi = (UnionObjectInspector) oi;
          byte tag = uoi.getTag(o);
          buffer.write(tag, invert);
          serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert);
          return;
        }
      default:
        {
          throw new RuntimeException("Unrecognized type: " + oi.getCategory());
        }
    }
  }
Ejemplo n.º 23
0
  private static void serializePrimitive(
      Type type, BlockBuilder builder, Object object, PrimitiveObjectInspector inspector) {
    requireNonNull(builder, "parent builder is null");

    if (object == null) {
      builder.appendNull();
      return;
    }

    switch (inspector.getPrimitiveCategory()) {
      case BOOLEAN:
        BooleanType.BOOLEAN.writeBoolean(builder, ((BooleanObjectInspector) inspector).get(object));
        return;
      case BYTE:
        TinyintType.TINYINT.writeLong(builder, ((ByteObjectInspector) inspector).get(object));
        return;
      case SHORT:
        SmallintType.SMALLINT.writeLong(builder, ((ShortObjectInspector) inspector).get(object));
        return;
      case INT:
        IntegerType.INTEGER.writeLong(builder, ((IntObjectInspector) inspector).get(object));
        return;
      case LONG:
        BigintType.BIGINT.writeLong(builder, ((LongObjectInspector) inspector).get(object));
        return;
      case FLOAT:
        DoubleType.DOUBLE.writeDouble(builder, ((FloatObjectInspector) inspector).get(object));
        return;
      case DOUBLE:
        DoubleType.DOUBLE.writeDouble(builder, ((DoubleObjectInspector) inspector).get(object));
        return;
      case STRING:
        type.writeSlice(
            builder,
            Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object)));
        return;
      case VARCHAR:
        type.writeSlice(
            builder,
            Slices.utf8Slice(
                ((HiveVarcharObjectInspector) inspector)
                    .getPrimitiveJavaObject(object)
                    .getValue()));
        return;
      case CHAR:
        CharType charType = checkType(type, CharType.class, "type");
        HiveChar hiveChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(object);
        type.writeSlice(
            builder,
            trimSpacesAndTruncateToLength(
                Slices.utf8Slice(hiveChar.getValue()), charType.getLength()));
        return;
      case DATE:
        DateType.DATE.writeLong(builder, formatDateAsLong(object, (DateObjectInspector) inspector));
        return;
      case TIMESTAMP:
        TimestampType.TIMESTAMP.writeLong(
            builder, formatTimestampAsLong(object, (TimestampObjectInspector) inspector));
        return;
      case BINARY:
        VARBINARY.writeSlice(
            builder,
            Slices.wrappedBuffer(
                ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object)));
        return;
      case DECIMAL:
        DecimalType decimalType = checkType(type, DecimalType.class, "type");
        HiveDecimalWritable hiveDecimal =
            ((HiveDecimalObjectInspector) inspector).getPrimitiveWritableObject(object);
        if (decimalType.isShort()) {
          decimalType.writeLong(
              builder, DecimalUtils.getShortDecimalValue(hiveDecimal, decimalType.getScale()));
        } else {
          decimalType.writeSlice(
              builder, DecimalUtils.getLongDecimalValue(hiveDecimal, decimalType.getScale()));
        }
        return;
    }
    throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory());
  }
Ejemplo n.º 24
0
 private Writable createPrimitive(final Object obj, final PrimitiveObjectInspector inspector)
     throws SerDeException {
   if (obj == null) {
     return null;
   }
   switch (inspector.getPrimitiveCategory()) {
     case VOID:
       return null;
     case BOOLEAN:
       return new BooleanWritable(
           ((BooleanObjectInspector) inspector).get(obj) ? Boolean.TRUE : Boolean.FALSE);
     case BYTE:
       return new ByteWritable(((ByteObjectInspector) inspector).get(obj));
     case DOUBLE:
       return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj));
     case FLOAT:
       return new FloatWritable(((FloatObjectInspector) inspector).get(obj));
     case INT:
       return new IntWritable(((IntObjectInspector) inspector).get(obj));
     case LONG:
       return new LongWritable(((LongObjectInspector) inspector).get(obj));
     case SHORT:
       return new ShortWritable(((ShortObjectInspector) inspector).get(obj));
     case STRING:
       String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(obj);
       try {
         return new BytesWritable(v.getBytes("UTF-8"));
       } catch (UnsupportedEncodingException e) {
         throw new SerDeException("Failed to encode string in UTF-8", e);
       }
     case DECIMAL:
       HiveDecimal hd = (HiveDecimal) inspector.getPrimitiveJavaObject(obj);
       DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
       int prec = decTypeInfo.precision();
       int scale = decTypeInfo.scale();
       byte[] src = hd.setScale(scale).unscaledValue().toByteArray();
       // Estimated number of bytes needed.
       int bytes = PRECISION_TO_BYTE_COUNT[prec - 1];
       if (bytes == src.length) {
         // No padding needed.
         return new BytesWritable(src);
       }
       byte[] tgt = new byte[bytes];
       if (hd.signum() == -1) {
         // For negative number, initializing bits to 1
         for (int i = 0; i < bytes; i++) {
           tgt[i] |= 0xFF;
         }
       }
       System.arraycopy(
           src, 0, tgt, bytes - src.length, src.length); // Padding leading zeroes/ones.
       return new BytesWritable(tgt);
     case TIMESTAMP:
       return new TimestampWritable(
           ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(obj));
     case CHAR:
       String strippedValue =
           ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(obj).getStrippedValue();
       return new BytesWritable(Binary.fromString(strippedValue).getBytes());
     case VARCHAR:
       String value =
           ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(obj).getValue();
       return new BytesWritable(Binary.fromString(value).getBytes());
     case BINARY:
       return new BytesWritable(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(obj));
     default:
       throw new SerDeException("Unknown primitive : " + inspector.getPrimitiveCategory());
   }
 }
 public static VectorColumnAssign buildObjectAssign(
     VectorizedRowBatch outputBatch, int outColIndex, ObjectInspector objInspector)
     throws HiveException {
   PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector;
   VectorColumnAssign outVCA = null;
   ColumnVector destCol = outputBatch.cols[outColIndex];
   if (destCol instanceof LongColumnVector) {
     switch (poi.getPrimitiveCategory()) {
       case BOOLEAN:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   BooleanWritable bw = (BooleanWritable) val;
                   assignLong(bw.get() ? 1 : 0, destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case BYTE:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   ByteWritable bw = (ByteWritable) val;
                   assignLong(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case SHORT:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   ShortWritable bw = (ShortWritable) val;
                   assignLong(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case INT:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   IntWritable bw = (IntWritable) val;
                   assignLong(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case LONG:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   LongWritable bw = (LongWritable) val;
                   assignLong(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case TIMESTAMP:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   TimestampWritable bw = (TimestampWritable) val;
                   Timestamp t = bw.getTimestamp();
                   assignLong(TimestampUtils.getTimeNanoSec(t), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       case DATE:
         outVCA =
             new VectorLongColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   DateWritable bw = (DateWritable) val;
                   assignLong(bw.getDays(), destIndex);
                 }
               }
             }.init(outputBatch, (LongColumnVector) destCol);
         break;
       default:
         throw new HiveException(
             "Incompatible Long vector column and primitive category "
                 + poi.getPrimitiveCategory());
     }
   } else if (destCol instanceof DoubleColumnVector) {
     switch (poi.getPrimitiveCategory()) {
       case DOUBLE:
         outVCA =
             new VectorDoubleColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   DoubleWritable bw = (DoubleWritable) val;
                   assignDouble(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (DoubleColumnVector) destCol);
         break;
       case FLOAT:
         outVCA =
             new VectorDoubleColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   FloatWritable bw = (FloatWritable) val;
                   assignDouble(bw.get(), destIndex);
                 }
               }
             }.init(outputBatch, (DoubleColumnVector) destCol);
         break;
       default:
         throw new HiveException(
             "Incompatible Double vector column and primitive category "
                 + poi.getPrimitiveCategory());
     }
   } else if (destCol instanceof BytesColumnVector) {
     switch (poi.getPrimitiveCategory()) {
       case STRING:
         outVCA =
             new VectorBytesColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   Text bw = (Text) val;
                   byte[] bytes = bw.getBytes();
                   assignBytes(bytes, 0, bytes.length, destIndex);
                 }
               }
             }.init(outputBatch, (BytesColumnVector) destCol);
         break;
       default:
         throw new HiveException(
             "Incompatible Bytes vector column and primitive category "
                 + poi.getPrimitiveCategory());
     }
   } else if (destCol instanceof DecimalColumnVector) {
     switch (poi.getPrimitiveCategory()) {
       case DECIMAL:
         outVCA =
             new VectorDecimalColumnAssign() {
               @Override
               public void assignObjectValue(Object val, int destIndex) throws HiveException {
                 if (val == null) {
                   assignNull(destIndex);
                 } else {
                   HiveDecimalWritable hdw = (HiveDecimalWritable) val;
                   assignDecimal(hdw, destIndex);
                 }
               }
             }.init(outputBatch, (DecimalColumnVector) destCol);
         break;
       default:
         throw new HiveException(
             "Incompatible Decimal vector column and primitive category "
                 + poi.getPrimitiveCategory());
     }
   } else {
     throw new HiveException("Unknown vector column type " + destCol.getClass().getName());
   }
   return outVCA;
 }
Ejemplo n.º 26
0
  static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {

    switch (oi.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            switch (poi.getPrimitiveCategory()) {
              case BOOLEAN:
                {
                  boolean b = ((BooleanObjectInspector) poi).get(o);
                  sb.append(b ? "true" : "false");
                  break;
                }
              case BYTE:
                {
                  sb.append(((ByteObjectInspector) poi).get(o));
                  break;
                }
              case SHORT:
                {
                  sb.append(((ShortObjectInspector) poi).get(o));
                  break;
                }
              case INT:
                {
                  sb.append(((IntObjectInspector) poi).get(o));
                  break;
                }
              case LONG:
                {
                  sb.append(((LongObjectInspector) poi).get(o));
                  break;
                }
              case FLOAT:
                {
                  sb.append(((FloatObjectInspector) poi).get(o));
                  break;
                }
              case DOUBLE:
                {
                  sb.append(((DoubleObjectInspector) poi).get(o));
                  break;
                }
              case STRING:
                {
                  sb.append('"');
                  sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
                  sb.append('"');
                  break;
                }
              case CHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case VARCHAR:
                {
                  sb.append('"');
                  sb.append(
                      escapeString(
                          ((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                  sb.append('"');
                  break;
                }
              case DATE:
                {
                  sb.append('"');
                  sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case TIMESTAMP:
                {
                  sb.append('"');
                  sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
                  sb.append('"');
                  break;
                }
              case BINARY:
                {
                  BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
                  Text txt = new Text();
                  txt.set(bw.getBytes(), 0, bw.getLength());
                  sb.append(txt.toString());
                  break;
                }
              case DECIMAL:
                {
                  sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
                  break;
                }
              default:
                throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
            }
          }
          break;
        }
      case LIST:
        {
          ListObjectInspector loi = (ListObjectInspector) oi;
          ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
          List<?> olist = loi.getList(o);
          if (olist == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACKET);
            for (int i = 0; i < olist.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
            }
            sb.append(RBRACKET);
          }
          break;
        }
      case MAP:
        {
          MapObjectInspector moi = (MapObjectInspector) oi;
          ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
          ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
          Map<?, ?> omap = moi.getMap(o);
          if (omap == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            boolean first = true;
            for (Object entry : omap.entrySet()) {
              if (first) {
                first = false;
              } else {
                sb.append(COMMA);
              }
              Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
              buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
              sb.append(COLON);
              buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case STRUCT:
        {
          StructObjectInspector soi = (StructObjectInspector) oi;
          List<? extends StructField> structFields = soi.getAllStructFieldRefs();
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            for (int i = 0; i < structFields.size(); i++) {
              if (i > 0) {
                sb.append(COMMA);
              }
              sb.append(QUOTE);
              sb.append(structFields.get(i).getFieldName());
              sb.append(QUOTE);
              sb.append(COLON);
              buildJSONString(
                  sb,
                  soi.getStructFieldData(o, structFields.get(i)),
                  structFields.get(i).getFieldObjectInspector(),
                  JSON_NULL);
            }
            sb.append(RBRACE);
          }
          break;
        }
      case UNION:
        {
          UnionObjectInspector uoi = (UnionObjectInspector) oi;
          if (o == null) {
            sb.append(nullStr);
          } else {
            sb.append(LBRACE);
            sb.append(uoi.getTag(o));
            sb.append(COLON);
            buildJSONString(
                sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
            sb.append(RBRACE);
          }
          break;
        }
      default:
        throw new RuntimeException("Unknown type in ObjectInspector!");
    }
  }