Esempio n. 1
0
  @Test
  public void testDecimalPlusDecimalSameParams() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(5, 2)),
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(5, 2))
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(6, 2), oi.getTypeInfo());
  }
  private ObjectInspector solveOi(ObjectInspector arg) {

    switch (arg.getCategory()) {
      case PRIMITIVE:

        // VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
        // UNKNOWN
        PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
        return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
            poi.getPrimitiveCategory());
      case LIST:
        return ObjectInspectorFactory.getStandardListObjectInspector(
            solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
      case MAP:
        return ObjectInspectorFactory.getStandardMapObjectInspector(
            solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
            solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
      case STRUCT:
        StructObjectInspector soi = (StructObjectInspector) arg;
        int size = soi.getAllStructFieldRefs().size();
        ArrayList<String> fnl = new ArrayList<String>(size);
        ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);

        for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
          fnl.add(sf.getFieldName());
          foil.add(solveOi(sf.getFieldObjectInspector()));
        }

        return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
      default:
        return arg;
    }
  }
 public static ConstantObjectInspector getConstantObjectInspector(
     ObjectInspector oi, Object value) {
   if (oi instanceof ConstantObjectInspector) {
     return (ConstantObjectInspector) oi;
   }
   ObjectInspector writableOI = getStandardObjectInspector(oi, ObjectInspectorCopyOption.WRITABLE);
   Object writableValue = ObjectInspectorConverters.getConverter(oi, writableOI).convert(value);
   switch (writableOI.getCategory()) {
     case PRIMITIVE:
       PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
       return PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
           poi.getTypeInfo(), writableValue);
     case LIST:
       ListObjectInspector loi = (ListObjectInspector) oi;
       return ObjectInspectorFactory.getStandardConstantListObjectInspector(
           getStandardObjectInspector(
               loi.getListElementObjectInspector(), ObjectInspectorCopyOption.WRITABLE),
           (List<?>) writableValue);
     case MAP:
       MapObjectInspector moi = (MapObjectInspector) oi;
       return ObjectInspectorFactory.getStandardConstantMapObjectInspector(
           getStandardObjectInspector(
               moi.getMapKeyObjectInspector(), ObjectInspectorCopyOption.WRITABLE),
           getStandardObjectInspector(
               moi.getMapValueObjectInspector(), ObjectInspectorCopyOption.WRITABLE),
           (Map<?, ?>) writableValue);
     default:
       throw new IllegalArgumentException(
           writableOI.getCategory() + " not yet supported for constant OI");
   }
 }
 /**
  * Returns an ObjectInspector for a primitive Class. The Class can be a Hive Writable class, or a
  * Java Primitive Class.
  *
  * <p>A runtimeException will be thrown if the class is not recognized as a primitive type by
  * Hive.
  */
 public static PrimitiveObjectInspector getPrimitiveObjectInspectorFromClass(Class<?> c) {
   if (Writable.class.isAssignableFrom(c)) {
     // It is a writable class
     PrimitiveTypeEntry te =
         PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveWritableClass(c);
     if (te == null) {
       throw new RuntimeException("Internal error: Cannot recognize " + c);
     }
     return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
         te.primitiveCategory);
   } else {
     // It is a Java class
     PrimitiveTypeEntry te = PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(c);
     if (te == null) {
       throw new RuntimeException("Internal error: Cannot recognize " + c);
     }
     return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(te.primitiveCategory);
   }
 }
Esempio n. 5
0
 @Override
 public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
   String database;
   if (context != null) {
     database = context.getJobConf().get("hive.current.database");
   } else {
     database = SessionState.get().getCurrentDatabase();
   }
   return PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
       TypeInfoFactory.stringTypeInfo, new Text(database));
 }
 /** Ensures that an ObjectInspector is Writable. */
 public static ObjectInspector getWritableObjectInspector(ObjectInspector oi) {
   // All non-primitive OIs are writable so we need only check this case.
   if (oi.getCategory() == Category.PRIMITIVE) {
     PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
     if (!(poi instanceof AbstractPrimitiveWritableObjectInspector)) {
       return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
           poi.getTypeInfo());
     }
   }
   return oi;
 }
Esempio n. 7
0
  @Test
  public void testDecimalPlusDecimal() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Decimal
    HiveDecimalWritable left = new HiveDecimalWritable(HiveDecimal.create("14.5"));
    HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(3, 1)),
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(5, 2))
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(6, 2), oi.getTypeInfo());
    HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args);
    Assert.assertEquals(HiveDecimal.create("249.47"), res.getHiveDecimal());
  }
  private static ObjectInspector getObjectInspector(TypeInfo type) throws IOException {

    switch (type.getCategory()) {
      case PRIMITIVE:
        PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
        return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
            primitiveType.getPrimitiveCategory());

      case MAP:
        MapTypeInfo mapType = (MapTypeInfo) type;
        MapObjectInspector mapInspector =
            ObjectInspectorFactory.getStandardMapObjectInspector(
                getObjectInspector(mapType.getMapKeyTypeInfo()),
                getObjectInspector(mapType.getMapValueTypeInfo()));
        return mapInspector;

      case LIST:
        ListTypeInfo listType = (ListTypeInfo) type;
        ListObjectInspector listInspector =
            ObjectInspectorFactory.getStandardListObjectInspector(
                getObjectInspector(listType.getListElementTypeInfo()));
        return listInspector;

      case STRUCT:
        StructTypeInfo structType = (StructTypeInfo) type;
        List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();

        List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
        for (TypeInfo fieldType : fieldTypes) {
          fieldInspectors.add(getObjectInspector(fieldType));
        }

        StructObjectInspector structInspector =
            ObjectInspectorFactory.getStandardStructObjectInspector(
                structType.getAllStructFieldNames(), fieldInspectors);
        return structInspector;

      default:
        throw new IOException("Unknown field schema type");
    }
  }
Esempio n. 9
0
  @Test
  public void testDoulePlusDecimal() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Double
    DoubleWritable left = new DoubleWritable(74.52);
    HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(5, 2))
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.doubleTypeInfo, oi.getTypeInfo());
    DoubleWritable res = (DoubleWritable) udf.evaluate(args);
    Assert.assertEquals(new Double(309.49), new Double(res.get()));
  }
Esempio n. 10
0
  @Test
  public void testLongPlusDecimal() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();

    // Long
    LongWritable left = new LongWritable(104);
    HiveDecimalWritable right = new HiveDecimalWritable(HiveDecimal.create("234.97"));
    ObjectInspector[] inputOIs = {
      PrimitiveObjectInspectorFactory.writableLongObjectInspector,
      PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
          TypeInfoFactory.getDecimalTypeInfo(9, 4))
    };
    DeferredObject[] args = {
      new DeferredJavaObject(left), new DeferredJavaObject(right),
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(24, 4), oi.getTypeInfo());
    HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args);
    Assert.assertEquals(HiveDecimal.create("338.97"), res.getHiveDecimal());
  }
Esempio n. 11
0
    @Override
    public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
      assert (parameters.length == 1);
      super.init(m, parameters);
      result = new HiveDecimalWritable(HiveDecimal.ZERO);
      inputOI = (PrimitiveObjectInspector) parameters[0];
      // The output precision is 10 greater than the input which should cover at least
      // 10b rows. The scale is the same as the input.
      DecimalTypeInfo outputTypeInfo = null;
      if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
        int precision = Math.min(HiveDecimal.MAX_PRECISION, inputOI.precision() + 10);
        outputTypeInfo = TypeInfoFactory.getDecimalTypeInfo(precision, inputOI.scale());
      } else {
        outputTypeInfo = (DecimalTypeInfo) inputOI.getTypeInfo();
      }
      ObjectInspector oi =
          PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(outputTypeInfo);
      outputOI =
          (PrimitiveObjectInspector)
              ObjectInspectorUtils.getStandardObjectInspector(oi, ObjectInspectorCopyOption.JAVA);

      return oi;
    }
 @Override
 public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
   converter = getConverter(arguments[0]);
   return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
       PrimitiveObjectInspector.PrimitiveCategory.STRING);
 }
 public static ObjectInspector getStandardObjectInspector(
     ObjectInspector oi, ObjectInspectorCopyOption objectInspectorOption) {
   ObjectInspector result = null;
   switch (oi.getCategory()) {
     case PRIMITIVE:
       {
         PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
         switch (objectInspectorOption) {
           case DEFAULT:
             {
               if (poi.preferWritable()) {
                 result =
                     PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
                         poi.getTypeInfo());
               } else {
                 result =
                     PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                         poi.getTypeInfo());
               }
               break;
             }
           case JAVA:
             {
               result =
                   PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                       poi.getTypeInfo());
               break;
             }
           case WRITABLE:
             result =
                 PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
                     poi.getTypeInfo());
             break;
         }
         break;
       }
     case LIST:
       {
         ListObjectInspector loi = (ListObjectInspector) oi;
         result =
             ObjectInspectorFactory.getStandardListObjectInspector(
                 getStandardObjectInspector(
                     loi.getListElementObjectInspector(), objectInspectorOption));
         break;
       }
     case MAP:
       {
         MapObjectInspector moi = (MapObjectInspector) oi;
         result =
             ObjectInspectorFactory.getStandardMapObjectInspector(
                 getStandardObjectInspector(moi.getMapKeyObjectInspector(), objectInspectorOption),
                 getStandardObjectInspector(
                     moi.getMapValueObjectInspector(), objectInspectorOption));
         break;
       }
     case STRUCT:
       {
         StructObjectInspector soi = (StructObjectInspector) oi;
         List<? extends StructField> fields = soi.getAllStructFieldRefs();
         List<String> fieldNames = new ArrayList<String>(fields.size());
         List<ObjectInspector> fieldObjectInspectors =
             new ArrayList<ObjectInspector>(fields.size());
         for (StructField f : fields) {
           fieldNames.add(f.getFieldName());
           fieldObjectInspectors.add(
               getStandardObjectInspector(f.getFieldObjectInspector(), objectInspectorOption));
         }
         result =
             ObjectInspectorFactory.getStandardStructObjectInspector(
                 fieldNames, fieldObjectInspectors);
         break;
       }
     case UNION:
       {
         UnionObjectInspector uoi = (UnionObjectInspector) oi;
         List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
         for (ObjectInspector eoi : uoi.getObjectInspectors()) {
           ois.add(getStandardObjectInspector(eoi, objectInspectorOption));
         }
         result = ObjectInspectorFactory.getStandardUnionObjectInspector(ois);
         break;
       }
     default:
       {
         throw new RuntimeException("Unknown ObjectInspector category!");
       }
   }
   return result;
 }
Esempio n. 14
0
  private MapOpCtx initObjectInspector(
      Configuration hconf, MapOpCtx opCtx, StructObjectInspector tableRowOI) throws Exception {
    PartitionDesc pd = opCtx.partDesc;
    TableDesc td = pd.getTableDesc();

    // Use table properties in case of unpartitioned tables,
    // and the union of table properties and partition properties, with partition
    // taking precedence, in the case of partitioned tables
    Properties overlayedProps =
        SerDeUtils.createOverlayedProperties(td.getProperties(), pd.getProperties());

    Map<String, String> partSpec = pd.getPartSpec();

    opCtx.tableName = String.valueOf(overlayedProps.getProperty("name"));
    opCtx.partName = String.valueOf(partSpec);
    opCtx.deserializer = pd.getDeserializer(hconf);

    StructObjectInspector partRawRowObjectInspector;
    if (Utilities.isInputFileFormatSelfDescribing(pd)) {
      partRawRowObjectInspector = tableRowOI;
    } else {
      partRawRowObjectInspector = (StructObjectInspector) opCtx.deserializer.getObjectInspector();
    }

    opCtx.partTblObjectInspectorConverter =
        ObjectInspectorConverters.getConverter(partRawRowObjectInspector, tableRowOI);

    // Next check if this table has partitions and if so
    // get the list of partition names as well as allocate
    // the serdes for the partition columns
    String pcols = overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS);

    if (pcols != null && pcols.length() > 0) {
      String[] partKeys = pcols.trim().split("/");
      String pcolTypes =
          overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES);
      String[] partKeyTypes = pcolTypes.trim().split(":");

      if (partKeys.length > partKeyTypes.length) {
        throw new HiveException(
            "Internal error : partKeys length, "
                + partKeys.length
                + " greater than partKeyTypes length, "
                + partKeyTypes.length);
      }

      List<String> partNames = new ArrayList<String>(partKeys.length);
      Object[] partValues = new Object[partKeys.length];
      List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>(partKeys.length);

      for (int i = 0; i < partKeys.length; i++) {
        String key = partKeys[i];
        partNames.add(key);
        ObjectInspector oi =
            PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
                TypeInfoFactory.getPrimitiveTypeInfo(partKeyTypes[i]));

        // Partitions do not exist for this table
        if (partSpec == null) {
          // for partitionless table, initialize partValue to null
          partValues[i] = null;
        } else {
          partValues[i] =
              ObjectInspectorConverters.getConverter(
                      PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi)
                  .convert(partSpec.get(key));
        }
        partObjectInspectors.add(oi);
      }
      opCtx.rowWithPart = new Object[] {null, partValues};
      opCtx.partObjectInspector =
          ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors);
    }

    // The op may not be a TableScan for mapjoins
    // Consider the query: select /*+MAPJOIN(a)*/ count(*) FROM T1 a JOIN T2 b ON a.key = b.key;
    // In that case, it will be a Select, but the rowOI need not be amended
    if (opCtx.op instanceof TableScanOperator) {
      TableScanOperator tsOp = (TableScanOperator) opCtx.op;
      TableScanDesc tsDesc = tsOp.getConf();
      if (tsDesc != null && tsDesc.hasVirtualCols()) {
        opCtx.vcs = tsDesc.getVirtualCols();
        opCtx.vcValues = new Object[opCtx.vcs.size()];
        opCtx.vcsObjectInspector = VirtualColumn.getVCSObjectInspector(opCtx.vcs);
        if (opCtx.isPartitioned()) {
          opCtx.rowWithPartAndVC = Arrays.copyOfRange(opCtx.rowWithPart, 0, 3);
        } else {
          opCtx.rowWithPartAndVC = new Object[2];
        }
      }
    }
    if (!opCtx.hasVC() && !opCtx.isPartitioned()) {
      opCtx.rowObjectInspector = tableRowOI;
      return opCtx;
    }
    List<StructObjectInspector> inspectors = new ArrayList<StructObjectInspector>();
    inspectors.add(tableRowOI);
    if (opCtx.isPartitioned()) {
      inspectors.add(opCtx.partObjectInspector);
    }
    if (opCtx.hasVC()) {
      inspectors.add(opCtx.vcsObjectInspector);
    }
    opCtx.rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(inspectors);
    return opCtx;
  }
 /**
  * Returns the lazy binary object inspector that can be used to inspect an lazy binary object of
  * that typeInfo
  *
  * <p>For primitive types, we use the standard writable object inspector.
  */
 public static ObjectInspector getLazyBinaryObjectInspectorFromTypeInfo(TypeInfo typeInfo) {
   ObjectInspector result = cachedLazyBinaryObjectInspector.get(typeInfo);
   if (result == null) {
     switch (typeInfo.getCategory()) {
       case PRIMITIVE:
         {
           result =
               PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
                   ((PrimitiveTypeInfo) typeInfo));
           break;
         }
       case LIST:
         {
           ObjectInspector elementObjectInspector =
               getLazyBinaryObjectInspectorFromTypeInfo(
                   ((ListTypeInfo) typeInfo).getListElementTypeInfo());
           result =
               LazyBinaryObjectInspectorFactory.getLazyBinaryListObjectInspector(
                   elementObjectInspector);
           break;
         }
       case MAP:
         {
           MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
           ObjectInspector keyObjectInspector =
               getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo());
           ObjectInspector valueObjectInspector =
               getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo());
           result =
               LazyBinaryObjectInspectorFactory.getLazyBinaryMapObjectInspector(
                   keyObjectInspector, valueObjectInspector);
           break;
         }
       case STRUCT:
         {
           StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
           List<String> fieldNames = structTypeInfo.getAllStructFieldNames();
           List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
           List<ObjectInspector> fieldObjectInspectors =
               new ArrayList<ObjectInspector>(fieldTypeInfos.size());
           for (int i = 0; i < fieldTypeInfos.size(); i++) {
             fieldObjectInspectors.add(
                 getLazyBinaryObjectInspectorFromTypeInfo(fieldTypeInfos.get(i)));
           }
           result =
               LazyBinaryObjectInspectorFactory.getLazyBinaryStructObjectInspector(
                   fieldNames, fieldObjectInspectors);
           break;
         }
       case UNION:
         {
           UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
           final List<TypeInfo> fieldTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
           List<ObjectInspector> fieldObjectInspectors =
               new ArrayList<ObjectInspector>(fieldTypeInfos.size());
           for (int i = 0; i < fieldTypeInfos.size(); i++) {
             fieldObjectInspectors.add(
                 getLazyBinaryObjectInspectorFromTypeInfo(fieldTypeInfos.get(i)));
           }
           result =
               LazyBinaryObjectInspectorFactory.getLazyBinaryUnionObjectInspector(
                   fieldObjectInspectors);
           break;
         }
       default:
         {
           result = null;
         }
     }
     ObjectInspector prev = cachedLazyBinaryObjectInspector.putIfAbsent(typeInfo, result);
     if (prev != null) {
       result = prev;
     }
   }
   return result;
 }