private ObjectInspectorConverters.Converter getConverter(ObjectInspector arg) {

    switch (arg.getCategory()) {
      case PRIMITIVE:
        return ObjectInspectorConverters.getConverter(arg, arg);
      case LIST:
      case MAP:
      case STRUCT:
        return ObjectInspectorConverters.getConverter(arg, solveOi(arg));
      default:
        return null;
    }
  }
Exemplo n.º 2
0
  @Override
  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {

    if (arguments.length % 2 != 0) {
      throw new UDFArgumentLengthException("Arguments must be in key/value pairs");
    }

    GenericUDFUtils.ReturnObjectInspectorResolver keyOIResolver =
        new GenericUDFUtils.ReturnObjectInspectorResolver(true);
    GenericUDFUtils.ReturnObjectInspectorResolver valueOIResolver =
        new GenericUDFUtils.ReturnObjectInspectorResolver(true);

    for (int i = 0; i < arguments.length; i++) {
      if (i % 2 == 0) {
        // Keys
        if (!(arguments[i] instanceof PrimitiveObjectInspector)) {
          throw new UDFArgumentTypeException(
              1, "Primitive Type is expected but " + arguments[i].getTypeName() + "\" is found");
        }
        if (!keyOIResolver.update(arguments[i])) {
          throw new UDFArgumentTypeException(
              i,
              "Key type \""
                  + arguments[i].getTypeName()
                  + "\" is different from preceding key types. "
                  + "Previous key type was \""
                  + arguments[i - 2].getTypeName()
                  + "\"");
        }
      } else {
        // Values
        if (!valueOIResolver.update(arguments[i])
            && !compatibleTypes(arguments[i], arguments[i - 2])) {
          throw new UDFArgumentTypeException(
              i,
              "Value type \""
                  + arguments[i].getTypeName()
                  + "\" is different from preceding value types. "
                  + "Previous value type was \""
                  + arguments[i - 2].getTypeName()
                  + "\"");
        }
      }
    }

    ObjectInspector keyOI =
        keyOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector valueOI =
        valueOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

    converters = new Converter[arguments.length];

    for (int i = 0; i < arguments.length; i++) {
      converters[i] =
          ObjectInspectorConverters.getConverter(arguments[i], i % 2 == 0 ? keyOI : valueOI);
    }

    return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI);
  }
Exemplo n.º 3
0
 /**
  * Converts the skewedValue available as a string in the metadata to the appropriate object by
  * using the type of the column from the join key.
  *
  * @param skewedValue
  * @param keyCol
  * @return an expression node descriptor of the appropriate constant
  */
 private ExprNodeConstantDesc createConstDesc(String skewedValue, ExprNodeColumnDesc keyCol) {
   ObjectInspector inputOI =
       TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(TypeInfoFactory.stringTypeInfo);
   ObjectInspector outputOI =
       TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(keyCol.getTypeInfo());
   Converter converter = ObjectInspectorConverters.getConverter(inputOI, outputOI);
   Object skewedValueObject = converter.convert(skewedValue);
   return new ExprNodeConstantDesc(keyCol.getTypeInfo(), skewedValueObject);
 }
Exemplo n.º 4
0
  public static void obtainBinaryConverter(
      ObjectInspector[] arguments, int i, PrimitiveCategory[] inputTypes, Converter[] converters)
      throws UDFArgumentTypeException {
    PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
    PrimitiveCategory inputType = inOi.getPrimitiveCategory();

    Converter converter =
        ObjectInspectorConverters.getConverter(
            arguments[i], PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
    converters[i] = converter;
    inputTypes[i] = inputType;
  }
Exemplo n.º 5
0
 private Converter checkArguments(ObjectInspector[] arguments, int i) throws UDFArgumentException {
   if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) {
     throw new UDFArgumentTypeException(
         0,
         "Only primitive type arguments are accepted but "
             + arguments[i].getTypeName()
             + " is passed. as first arguments");
   }
   PrimitiveCategory inputType = ((PrimitiveObjectInspector) arguments[i]).getPrimitiveCategory();
   Converter converter;
   switch (inputType) {
     case STRING:
     case VARCHAR:
     case CHAR:
       converter =
           ObjectInspectorConverters.getConverter(
               (PrimitiveObjectInspector) arguments[i],
               PrimitiveObjectInspectorFactory.writableStringObjectInspector);
       break;
     case TIMESTAMP:
       converter =
           new TimestampConverter(
               (PrimitiveObjectInspector) arguments[i],
               PrimitiveObjectInspectorFactory.writableTimestampObjectInspector);
       break;
     case DATE:
       converter =
           ObjectInspectorConverters.getConverter(
               (PrimitiveObjectInspector) arguments[i],
               PrimitiveObjectInspectorFactory.writableDateObjectInspector);
       break;
     default:
       throw new UDFArgumentException(
           " DATEDIFF() only takes STRING/TIMESTAMP/DATEWRITABLE types as "
               + (i + 1)
               + "-th argument, got "
               + inputType);
   }
   return converter;
 }
Exemplo n.º 6
0
  @Override
  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length < 2) {
      throw new UDFArgumentLengthException(
          "The function ELT(N,str1,str2,str3,...) needs at least two arguments.");
    }

    for (int i = 0; i < arguments.length; i++) {
      Category category = arguments[i].getCategory();
      if (category != Category.PRIMITIVE) {
        throw new UDFArgumentTypeException(
            i,
            "The "
                + GenericUDFUtils.getOrdinal(i + 1)
                + " argument of function ELT is expected to a "
                + Category.PRIMITIVE.toString().toLowerCase()
                + " type, but "
                + category.toString().toLowerCase()
                + " is found");
      }
    }

    converters = new ObjectInspectorConverters.Converter[arguments.length];
    for (int i = 0; i < arguments.length; i++) {
      if (i == 0) {
        converters[i] =
            ObjectInspectorConverters.getConverter(
                arguments[i], PrimitiveObjectInspectorFactory.writableIntObjectInspector);
      } else {
        converters[i] =
            ObjectInspectorConverters.getConverter(
                arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector);
      }
    }

    return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
  }
Exemplo n.º 7
0
  /**
   * Traverse all the partitions for a table, and get the OI for the table. Note that a conversion
   * is required if any of the partition OI is different from the table OI. For eg. if the query
   * references table T (partitions P1, P2), and P1's schema is same as T, whereas P2's scheme is
   * different from T, conversion might be needed for both P1 and P2, since SettableOI might be
   * needed for T
   */
  private Map<TableDesc, StructObjectInspector> getConvertedOI(Configuration hconf)
      throws HiveException {
    Map<TableDesc, StructObjectInspector> tableDescOI =
        new HashMap<TableDesc, StructObjectInspector>();
    Set<TableDesc> identityConverterTableDesc = new HashSet<TableDesc>();
    try {
      Map<ObjectInspector, Boolean> oiSettableProperties = new HashMap<ObjectInspector, Boolean>();

      for (String onefile : conf.getPathToAliases().keySet()) {
        PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
        TableDesc tableDesc = pd.getTableDesc();
        Deserializer partDeserializer = pd.getDeserializer(hconf);

        StructObjectInspector partRawRowObjectInspector;
        if (Utilities.isInputFileFormatSelfDescribing(pd)) {
          Deserializer tblDeserializer = tableDesc.getDeserializer(hconf);
          partRawRowObjectInspector = (StructObjectInspector) tblDeserializer.getObjectInspector();
        } else {
          partRawRowObjectInspector = (StructObjectInspector) partDeserializer.getObjectInspector();
        }

        StructObjectInspector tblRawRowObjectInspector = tableDescOI.get(tableDesc);
        if ((tblRawRowObjectInspector == null)
            || (identityConverterTableDesc.contains(tableDesc))) {
          Deserializer tblDeserializer = tableDesc.getDeserializer(hconf);
          tblRawRowObjectInspector =
              (StructObjectInspector)
                  ObjectInspectorConverters.getConvertedOI(
                      partRawRowObjectInspector,
                      tblDeserializer.getObjectInspector(),
                      oiSettableProperties);

          if (identityConverterTableDesc.contains(tableDesc)) {
            if (!partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {
              identityConverterTableDesc.remove(tableDesc);
            }
          } else if (partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {
            identityConverterTableDesc.add(tableDesc);
          }

          tableDescOI.put(tableDesc, tblRawRowObjectInspector);
        }
      }
    } catch (Exception e) {
      throw new HiveException(e);
    }
    return tableDescOI;
  }
Exemplo n.º 8
0
  @Override
  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length < 1 || arguments.length > 3) {
      throw new UDFArgumentLengthException(
          "The function sentences takes between 1 and 3 arguments.");
    }

    converters = new ObjectInspectorConverters.Converter[arguments.length];
    for (int i = 0; i < arguments.length; i++) {
      converters[i] =
          ObjectInspectorConverters.getConverter(
              arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    }

    return ObjectInspectorFactory.getStandardListObjectInspector(
        ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableStringObjectInspector));
  }
Exemplo n.º 9
0
  @Override
  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver;
    returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);

    if (arguments.length != 1) {
      throw new UDFArgumentLengthException(
          "The function SORT_ARRAY(array(obj1, obj2,...)) needs one argument.");
    }

    switch (arguments[0].getCategory()) {
      case LIST:
        if (((ListObjectInspector) (arguments[0]))
            .getListElementObjectInspector()
            .getCategory()
            .equals(Category.PRIMITIVE)) {
          break;
        }
      default:
        throw new UDFArgumentTypeException(
            0,
            "Argument 1"
                + " of function SORT_ARRAY must be "
                + serdeConstants.LIST_TYPE_NAME
                + "<"
                + Category.PRIMITIVE
                + ">, but "
                + arguments[0].getTypeName()
                + " was found.");
    }

    ObjectInspector elementObjectInspector =
        ((ListObjectInspector) (arguments[0])).getListElementObjectInspector();
    argumentOIs = arguments;
    converters = new Converter[arguments.length];
    ObjectInspector returnOI = returnOIResolver.get();
    if (returnOI == null) {
      returnOI = elementObjectInspector;
    }
    converters[0] = ObjectInspectorConverters.getConverter(elementObjectInspector, returnOI);

    return ObjectInspectorFactory.getStandardListObjectInspector(returnOI);
  }
Exemplo n.º 10
0
 public Symbol(ExprNodeEvaluator symbolExprEval, ObjectInspector symbolOI) {
   this.symbolExprEval = symbolExprEval;
   converter =
       ObjectInspectorConverters.getConverter(
           symbolOI, PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
 }
Exemplo n.º 11
0
  private MapOpCtx initObjectInspector(
      Configuration hconf, MapOpCtx opCtx, StructObjectInspector tableRowOI) throws Exception {
    PartitionDesc pd = opCtx.partDesc;
    TableDesc td = pd.getTableDesc();

    // Use table properties in case of unpartitioned tables,
    // and the union of table properties and partition properties, with partition
    // taking precedence, in the case of partitioned tables
    Properties overlayedProps =
        SerDeUtils.createOverlayedProperties(td.getProperties(), pd.getProperties());

    Map<String, String> partSpec = pd.getPartSpec();

    opCtx.tableName = String.valueOf(overlayedProps.getProperty("name"));
    opCtx.partName = String.valueOf(partSpec);
    opCtx.deserializer = pd.getDeserializer(hconf);

    StructObjectInspector partRawRowObjectInspector;
    if (Utilities.isInputFileFormatSelfDescribing(pd)) {
      partRawRowObjectInspector = tableRowOI;
    } else {
      partRawRowObjectInspector = (StructObjectInspector) opCtx.deserializer.getObjectInspector();
    }

    opCtx.partTblObjectInspectorConverter =
        ObjectInspectorConverters.getConverter(partRawRowObjectInspector, tableRowOI);

    // Next check if this table has partitions and if so
    // get the list of partition names as well as allocate
    // the serdes for the partition columns
    String pcols = overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS);

    if (pcols != null && pcols.length() > 0) {
      String[] partKeys = pcols.trim().split("/");
      String pcolTypes =
          overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES);
      String[] partKeyTypes = pcolTypes.trim().split(":");

      if (partKeys.length > partKeyTypes.length) {
        throw new HiveException(
            "Internal error : partKeys length, "
                + partKeys.length
                + " greater than partKeyTypes length, "
                + partKeyTypes.length);
      }

      List<String> partNames = new ArrayList<String>(partKeys.length);
      Object[] partValues = new Object[partKeys.length];
      List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>(partKeys.length);

      for (int i = 0; i < partKeys.length; i++) {
        String key = partKeys[i];
        partNames.add(key);
        ObjectInspector oi =
            PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
                TypeInfoFactory.getPrimitiveTypeInfo(partKeyTypes[i]));

        // Partitions do not exist for this table
        if (partSpec == null) {
          // for partitionless table, initialize partValue to null
          partValues[i] = null;
        } else {
          partValues[i] =
              ObjectInspectorConverters.getConverter(
                      PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi)
                  .convert(partSpec.get(key));
        }
        partObjectInspectors.add(oi);
      }
      opCtx.rowWithPart = new Object[] {null, partValues};
      opCtx.partObjectInspector =
          ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors);
    }

    // The op may not be a TableScan for mapjoins
    // Consider the query: select /*+MAPJOIN(a)*/ count(*) FROM T1 a JOIN T2 b ON a.key = b.key;
    // In that case, it will be a Select, but the rowOI need not be amended
    if (opCtx.op instanceof TableScanOperator) {
      TableScanOperator tsOp = (TableScanOperator) opCtx.op;
      TableScanDesc tsDesc = tsOp.getConf();
      if (tsDesc != null && tsDesc.hasVirtualCols()) {
        opCtx.vcs = tsDesc.getVirtualCols();
        opCtx.vcValues = new Object[opCtx.vcs.size()];
        opCtx.vcsObjectInspector = VirtualColumn.getVCSObjectInspector(opCtx.vcs);
        if (opCtx.isPartitioned()) {
          opCtx.rowWithPartAndVC = Arrays.copyOfRange(opCtx.rowWithPart, 0, 3);
        } else {
          opCtx.rowWithPartAndVC = new Object[2];
        }
      }
    }
    if (!opCtx.hasVC() && !opCtx.isPartitioned()) {
      opCtx.rowObjectInspector = tableRowOI;
      return opCtx;
    }
    List<StructObjectInspector> inspectors = new ArrayList<StructObjectInspector>();
    inspectors.add(tableRowOI);
    if (opCtx.isPartitioned()) {
      inspectors.add(opCtx.partObjectInspector);
    }
    if (opCtx.hasVC()) {
      inspectors.add(opCtx.vcsObjectInspector);
    }
    opCtx.rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(inspectors);
    return opCtx;
  }