private ObjectInspectorConverters.Converter getConverter(ObjectInspector arg) { switch (arg.getCategory()) { case PRIMITIVE: return ObjectInspectorConverters.getConverter(arg, arg); case LIST: case MAP: case STRUCT: return ObjectInspectorConverters.getConverter(arg, solveOi(arg)); default: return null; } }
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (arguments.length % 2 != 0) { throw new UDFArgumentLengthException("Arguments must be in key/value pairs"); } GenericUDFUtils.ReturnObjectInspectorResolver keyOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true); GenericUDFUtils.ReturnObjectInspectorResolver valueOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true); for (int i = 0; i < arguments.length; i++) { if (i % 2 == 0) { // Keys if (!(arguments[i] instanceof PrimitiveObjectInspector)) { throw new UDFArgumentTypeException( 1, "Primitive Type is expected but " + arguments[i].getTypeName() + "\" is found"); } if (!keyOIResolver.update(arguments[i])) { throw new UDFArgumentTypeException( i, "Key type \"" + arguments[i].getTypeName() + "\" is different from preceding key types. " + "Previous key type was \"" + arguments[i - 2].getTypeName() + "\""); } } else { // Values if (!valueOIResolver.update(arguments[i]) && !compatibleTypes(arguments[i], arguments[i - 2])) { throw new UDFArgumentTypeException( i, "Value type \"" + arguments[i].getTypeName() + "\" is different from preceding value types. " + "Previous value type was \"" + arguments[i - 2].getTypeName() + "\""); } } } ObjectInspector keyOI = keyOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector); ObjectInspector valueOI = valueOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector); converters = new Converter[arguments.length]; for (int i = 0; i < arguments.length; i++) { converters[i] = ObjectInspectorConverters.getConverter(arguments[i], i % 2 == 0 ? keyOI : valueOI); } return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI); }
/** * Converts the skewedValue available as a string in the metadata to the appropriate object by * using the type of the column from the join key. * * @param skewedValue * @param keyCol * @return an expression node descriptor of the appropriate constant */ private ExprNodeConstantDesc createConstDesc(String skewedValue, ExprNodeColumnDesc keyCol) { ObjectInspector inputOI = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(TypeInfoFactory.stringTypeInfo); ObjectInspector outputOI = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(keyCol.getTypeInfo()); Converter converter = ObjectInspectorConverters.getConverter(inputOI, outputOI); Object skewedValueObject = converter.convert(skewedValue); return new ExprNodeConstantDesc(keyCol.getTypeInfo(), skewedValueObject); }
public static void obtainBinaryConverter( ObjectInspector[] arguments, int i, PrimitiveCategory[] inputTypes, Converter[] converters) throws UDFArgumentTypeException { PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i]; PrimitiveCategory inputType = inOi.getPrimitiveCategory(); Converter converter = ObjectInspectorConverters.getConverter( arguments[i], PrimitiveObjectInspectorFactory.writableBinaryObjectInspector); converters[i] = converter; inputTypes[i] = inputType; }
private Converter checkArguments(ObjectInspector[] arguments, int i) throws UDFArgumentException { if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException( 0, "Only primitive type arguments are accepted but " + arguments[i].getTypeName() + " is passed. as first arguments"); } PrimitiveCategory inputType = ((PrimitiveObjectInspector) arguments[i]).getPrimitiveCategory(); Converter converter; switch (inputType) { case STRING: case VARCHAR: case CHAR: converter = ObjectInspectorConverters.getConverter( (PrimitiveObjectInspector) arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector); break; case TIMESTAMP: converter = new TimestampConverter( (PrimitiveObjectInspector) arguments[i], PrimitiveObjectInspectorFactory.writableTimestampObjectInspector); break; case DATE: converter = ObjectInspectorConverters.getConverter( (PrimitiveObjectInspector) arguments[i], PrimitiveObjectInspectorFactory.writableDateObjectInspector); break; default: throw new UDFArgumentException( " DATEDIFF() only takes STRING/TIMESTAMP/DATEWRITABLE types as " + (i + 1) + "-th argument, got " + inputType); } return converter; }
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (arguments.length < 2) { throw new UDFArgumentLengthException( "The function ELT(N,str1,str2,str3,...) needs at least two arguments."); } for (int i = 0; i < arguments.length; i++) { Category category = arguments[i].getCategory(); if (category != Category.PRIMITIVE) { throw new UDFArgumentTypeException( i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of function ELT is expected to a " + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } converters = new ObjectInspectorConverters.Converter[arguments.length]; for (int i = 0; i < arguments.length; i++) { if (i == 0) { converters[i] = ObjectInspectorConverters.getConverter( arguments[i], PrimitiveObjectInspectorFactory.writableIntObjectInspector); } else { converters[i] = ObjectInspectorConverters.getConverter( arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector); } } return PrimitiveObjectInspectorFactory.writableStringObjectInspector; }
/** * Traverse all the partitions for a table, and get the OI for the table. Note that a conversion * is required if any of the partition OI is different from the table OI. For eg. if the query * references table T (partitions P1, P2), and P1's schema is same as T, whereas P2's scheme is * different from T, conversion might be needed for both P1 and P2, since SettableOI might be * needed for T */ private Map<TableDesc, StructObjectInspector> getConvertedOI(Configuration hconf) throws HiveException { Map<TableDesc, StructObjectInspector> tableDescOI = new HashMap<TableDesc, StructObjectInspector>(); Set<TableDesc> identityConverterTableDesc = new HashSet<TableDesc>(); try { Map<ObjectInspector, Boolean> oiSettableProperties = new HashMap<ObjectInspector, Boolean>(); for (String onefile : conf.getPathToAliases().keySet()) { PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile); TableDesc tableDesc = pd.getTableDesc(); Deserializer partDeserializer = pd.getDeserializer(hconf); StructObjectInspector partRawRowObjectInspector; if (Utilities.isInputFileFormatSelfDescribing(pd)) { Deserializer tblDeserializer = tableDesc.getDeserializer(hconf); partRawRowObjectInspector = (StructObjectInspector) tblDeserializer.getObjectInspector(); } else { partRawRowObjectInspector = (StructObjectInspector) partDeserializer.getObjectInspector(); } StructObjectInspector tblRawRowObjectInspector = tableDescOI.get(tableDesc); if ((tblRawRowObjectInspector == null) || (identityConverterTableDesc.contains(tableDesc))) { Deserializer tblDeserializer = tableDesc.getDeserializer(hconf); tblRawRowObjectInspector = (StructObjectInspector) ObjectInspectorConverters.getConvertedOI( partRawRowObjectInspector, tblDeserializer.getObjectInspector(), oiSettableProperties); if (identityConverterTableDesc.contains(tableDesc)) { if (!partRawRowObjectInspector.equals(tblRawRowObjectInspector)) { identityConverterTableDesc.remove(tableDesc); } } else if (partRawRowObjectInspector.equals(tblRawRowObjectInspector)) { identityConverterTableDesc.add(tableDesc); } tableDescOI.put(tableDesc, tblRawRowObjectInspector); } } } catch (Exception e) { throw new HiveException(e); } return tableDescOI; }
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (arguments.length < 1 || arguments.length > 3) { throw new UDFArgumentLengthException( "The function sentences takes between 1 and 3 arguments."); } converters = new ObjectInspectorConverters.Converter[arguments.length]; for (int i = 0; i < arguments.length; i++) { converters[i] = ObjectInspectorConverters.getConverter( arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector); } return ObjectInspectorFactory.getStandardListObjectInspector( ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.writableStringObjectInspector)); }
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver; returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true); if (arguments.length != 1) { throw new UDFArgumentLengthException( "The function SORT_ARRAY(array(obj1, obj2,...)) needs one argument."); } switch (arguments[0].getCategory()) { case LIST: if (((ListObjectInspector) (arguments[0])) .getListElementObjectInspector() .getCategory() .equals(Category.PRIMITIVE)) { break; } default: throw new UDFArgumentTypeException( 0, "Argument 1" + " of function SORT_ARRAY must be " + serdeConstants.LIST_TYPE_NAME + "<" + Category.PRIMITIVE + ">, but " + arguments[0].getTypeName() + " was found."); } ObjectInspector elementObjectInspector = ((ListObjectInspector) (arguments[0])).getListElementObjectInspector(); argumentOIs = arguments; converters = new Converter[arguments.length]; ObjectInspector returnOI = returnOIResolver.get(); if (returnOI == null) { returnOI = elementObjectInspector; } converters[0] = ObjectInspectorConverters.getConverter(elementObjectInspector, returnOI); return ObjectInspectorFactory.getStandardListObjectInspector(returnOI); }
public Symbol(ExprNodeEvaluator symbolExprEval, ObjectInspector symbolOI) { this.symbolExprEval = symbolExprEval; converter = ObjectInspectorConverters.getConverter( symbolOI, PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); }
private MapOpCtx initObjectInspector( Configuration hconf, MapOpCtx opCtx, StructObjectInspector tableRowOI) throws Exception { PartitionDesc pd = opCtx.partDesc; TableDesc td = pd.getTableDesc(); // Use table properties in case of unpartitioned tables, // and the union of table properties and partition properties, with partition // taking precedence, in the case of partitioned tables Properties overlayedProps = SerDeUtils.createOverlayedProperties(td.getProperties(), pd.getProperties()); Map<String, String> partSpec = pd.getPartSpec(); opCtx.tableName = String.valueOf(overlayedProps.getProperty("name")); opCtx.partName = String.valueOf(partSpec); opCtx.deserializer = pd.getDeserializer(hconf); StructObjectInspector partRawRowObjectInspector; if (Utilities.isInputFileFormatSelfDescribing(pd)) { partRawRowObjectInspector = tableRowOI; } else { partRawRowObjectInspector = (StructObjectInspector) opCtx.deserializer.getObjectInspector(); } opCtx.partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(partRawRowObjectInspector, tableRowOI); // Next check if this table has partitions and if so // get the list of partition names as well as allocate // the serdes for the partition columns String pcols = overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS); if (pcols != null && pcols.length() > 0) { String[] partKeys = pcols.trim().split("/"); String pcolTypes = overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES); String[] partKeyTypes = pcolTypes.trim().split(":"); if (partKeys.length > partKeyTypes.length) { throw new HiveException( "Internal error : partKeys length, " + partKeys.length + " greater than partKeyTypes length, " + partKeyTypes.length); } List<String> partNames = new ArrayList<String>(partKeys.length); Object[] partValues = new Object[partKeys.length]; List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>(partKeys.length); for (int i = 0; i < partKeys.length; i++) { String key = partKeys[i]; partNames.add(key); ObjectInspector oi = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( TypeInfoFactory.getPrimitiveTypeInfo(partKeyTypes[i])); // Partitions do not exist for this table if (partSpec == null) { // for partitionless table, initialize partValue to null partValues[i] = null; } else { partValues[i] = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi) .convert(partSpec.get(key)); } partObjectInspectors.add(oi); } opCtx.rowWithPart = new Object[] {null, partValues}; opCtx.partObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors); } // The op may not be a TableScan for mapjoins // Consider the query: select /*+MAPJOIN(a)*/ count(*) FROM T1 a JOIN T2 b ON a.key = b.key; // In that case, it will be a Select, but the rowOI need not be amended if (opCtx.op instanceof TableScanOperator) { TableScanOperator tsOp = (TableScanOperator) opCtx.op; TableScanDesc tsDesc = tsOp.getConf(); if (tsDesc != null && tsDesc.hasVirtualCols()) { opCtx.vcs = tsDesc.getVirtualCols(); opCtx.vcValues = new Object[opCtx.vcs.size()]; opCtx.vcsObjectInspector = VirtualColumn.getVCSObjectInspector(opCtx.vcs); if (opCtx.isPartitioned()) { opCtx.rowWithPartAndVC = Arrays.copyOfRange(opCtx.rowWithPart, 0, 3); } else { opCtx.rowWithPartAndVC = new Object[2]; } } } if (!opCtx.hasVC() && !opCtx.isPartitioned()) { opCtx.rowObjectInspector = tableRowOI; return opCtx; } List<StructObjectInspector> inspectors = new ArrayList<StructObjectInspector>(); inspectors.add(tableRowOI); if (opCtx.isPartitioned()) { inspectors.add(opCtx.partObjectInspector); } if (opCtx.hasVC()) { inspectors.add(opCtx.vcsObjectInspector); } opCtx.rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(inspectors); return opCtx; }