private ObjectInspectorConverters.Converter getConverter(ObjectInspector arg) {
switch (arg.getCategory()) {
case PRIMITIVE:
return ObjectInspectorConverters.getConverter(arg, arg);
case LIST:
case MAP:
case STRUCT:
return ObjectInspectorConverters.getConverter(arg, solveOi(arg));
default:
return null;
}
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length % 2 != 0) {
throw new UDFArgumentLengthException("Arguments must be in key/value pairs");
}
GenericUDFUtils.ReturnObjectInspectorResolver keyOIResolver =
new GenericUDFUtils.ReturnObjectInspectorResolver(true);
GenericUDFUtils.ReturnObjectInspectorResolver valueOIResolver =
new GenericUDFUtils.ReturnObjectInspectorResolver(true);
for (int i = 0; i < arguments.length; i++) {
if (i % 2 == 0) {
// Keys
if (!(arguments[i] instanceof PrimitiveObjectInspector)) {
throw new UDFArgumentTypeException(
1, "Primitive Type is expected but " + arguments[i].getTypeName() + "\" is found");
}
if (!keyOIResolver.update(arguments[i])) {
throw new UDFArgumentTypeException(
i,
"Key type \""
+ arguments[i].getTypeName()
+ "\" is different from preceding key types. "
+ "Previous key type was \""
+ arguments[i - 2].getTypeName()
+ "\"");
}
} else {
// Values
if (!valueOIResolver.update(arguments[i])
&& !compatibleTypes(arguments[i], arguments[i - 2])) {
throw new UDFArgumentTypeException(
i,
"Value type \""
+ arguments[i].getTypeName()
+ "\" is different from preceding value types. "
+ "Previous value type was \""
+ arguments[i - 2].getTypeName()
+ "\"");
}
}
}
ObjectInspector keyOI =
keyOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
ObjectInspector valueOI =
valueOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
converters = new Converter[arguments.length];
for (int i = 0; i < arguments.length; i++) {
converters[i] =
ObjectInspectorConverters.getConverter(arguments[i], i % 2 == 0 ? keyOI : valueOI);
}
return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI);
}
/**
* Converts the skewedValue available as a string in the metadata to the appropriate object by
* using the type of the column from the join key.
*
* @param skewedValue
* @param keyCol
* @return an expression node descriptor of the appropriate constant
*/
private ExprNodeConstantDesc createConstDesc(String skewedValue, ExprNodeColumnDesc keyCol) {
ObjectInspector inputOI =
TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(TypeInfoFactory.stringTypeInfo);
ObjectInspector outputOI =
TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(keyCol.getTypeInfo());
Converter converter = ObjectInspectorConverters.getConverter(inputOI, outputOI);
Object skewedValueObject = converter.convert(skewedValue);
return new ExprNodeConstantDesc(keyCol.getTypeInfo(), skewedValueObject);
}
public static void obtainBinaryConverter(
ObjectInspector[] arguments, int i, PrimitiveCategory[] inputTypes, Converter[] converters)
throws UDFArgumentTypeException {
PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
PrimitiveCategory inputType = inOi.getPrimitiveCategory();
Converter converter =
ObjectInspectorConverters.getConverter(
arguments[i], PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
converters[i] = converter;
inputTypes[i] = inputType;
}
private Converter checkArguments(ObjectInspector[] arguments, int i) throws UDFArgumentException {
if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) {
throw new UDFArgumentTypeException(
0,
"Only primitive type arguments are accepted but "
+ arguments[i].getTypeName()
+ " is passed. as first arguments");
}
PrimitiveCategory inputType = ((PrimitiveObjectInspector) arguments[i]).getPrimitiveCategory();
Converter converter;
switch (inputType) {
case STRING:
case VARCHAR:
case CHAR:
converter =
ObjectInspectorConverters.getConverter(
(PrimitiveObjectInspector) arguments[i],
PrimitiveObjectInspectorFactory.writableStringObjectInspector);
break;
case TIMESTAMP:
converter =
new TimestampConverter(
(PrimitiveObjectInspector) arguments[i],
PrimitiveObjectInspectorFactory.writableTimestampObjectInspector);
break;
case DATE:
converter =
ObjectInspectorConverters.getConverter(
(PrimitiveObjectInspector) arguments[i],
PrimitiveObjectInspectorFactory.writableDateObjectInspector);
break;
default:
throw new UDFArgumentException(
" DATEDIFF() only takes STRING/TIMESTAMP/DATEWRITABLE types as "
+ (i + 1)
+ "-th argument, got "
+ inputType);
}
return converter;
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length < 2) {
throw new UDFArgumentLengthException(
"The function ELT(N,str1,str2,str3,...) needs at least two arguments.");
}
for (int i = 0; i < arguments.length; i++) {
Category category = arguments[i].getCategory();
if (category != Category.PRIMITIVE) {
throw new UDFArgumentTypeException(
i,
"The "
+ GenericUDFUtils.getOrdinal(i + 1)
+ " argument of function ELT is expected to a "
+ Category.PRIMITIVE.toString().toLowerCase()
+ " type, but "
+ category.toString().toLowerCase()
+ " is found");
}
}
converters = new ObjectInspectorConverters.Converter[arguments.length];
for (int i = 0; i < arguments.length; i++) {
if (i == 0) {
converters[i] =
ObjectInspectorConverters.getConverter(
arguments[i], PrimitiveObjectInspectorFactory.writableIntObjectInspector);
} else {
converters[i] =
ObjectInspectorConverters.getConverter(
arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector);
}
}
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
}
/**
* Traverse all the partitions for a table, and get the OI for the table. Note that a conversion
* is required if any of the partition OI is different from the table OI. For eg. if the query
* references table T (partitions P1, P2), and P1's schema is same as T, whereas P2's scheme is
* different from T, conversion might be needed for both P1 and P2, since SettableOI might be
* needed for T
*/
private Map<TableDesc, StructObjectInspector> getConvertedOI(Configuration hconf)
throws HiveException {
Map<TableDesc, StructObjectInspector> tableDescOI =
new HashMap<TableDesc, StructObjectInspector>();
Set<TableDesc> identityConverterTableDesc = new HashSet<TableDesc>();
try {
Map<ObjectInspector, Boolean> oiSettableProperties = new HashMap<ObjectInspector, Boolean>();
for (String onefile : conf.getPathToAliases().keySet()) {
PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
TableDesc tableDesc = pd.getTableDesc();
Deserializer partDeserializer = pd.getDeserializer(hconf);
StructObjectInspector partRawRowObjectInspector;
if (Utilities.isInputFileFormatSelfDescribing(pd)) {
Deserializer tblDeserializer = tableDesc.getDeserializer(hconf);
partRawRowObjectInspector = (StructObjectInspector) tblDeserializer.getObjectInspector();
} else {
partRawRowObjectInspector = (StructObjectInspector) partDeserializer.getObjectInspector();
}
StructObjectInspector tblRawRowObjectInspector = tableDescOI.get(tableDesc);
if ((tblRawRowObjectInspector == null)
|| (identityConverterTableDesc.contains(tableDesc))) {
Deserializer tblDeserializer = tableDesc.getDeserializer(hconf);
tblRawRowObjectInspector =
(StructObjectInspector)
ObjectInspectorConverters.getConvertedOI(
partRawRowObjectInspector,
tblDeserializer.getObjectInspector(),
oiSettableProperties);
if (identityConverterTableDesc.contains(tableDesc)) {
if (!partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {
identityConverterTableDesc.remove(tableDesc);
}
} else if (partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {
identityConverterTableDesc.add(tableDesc);
}
tableDescOI.put(tableDesc, tblRawRowObjectInspector);
}
}
} catch (Exception e) {
throw new HiveException(e);
}
return tableDescOI;
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length < 1 || arguments.length > 3) {
throw new UDFArgumentLengthException(
"The function sentences takes between 1 and 3 arguments.");
}
converters = new ObjectInspectorConverters.Converter[arguments.length];
for (int i = 0; i < arguments.length; i++) {
converters[i] =
ObjectInspectorConverters.getConverter(
arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector);
}
return ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableStringObjectInspector));
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver;
returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
if (arguments.length != 1) {
throw new UDFArgumentLengthException(
"The function SORT_ARRAY(array(obj1, obj2,...)) needs one argument.");
}
switch (arguments[0].getCategory()) {
case LIST:
if (((ListObjectInspector) (arguments[0]))
.getListElementObjectInspector()
.getCategory()
.equals(Category.PRIMITIVE)) {
break;
}
default:
throw new UDFArgumentTypeException(
0,
"Argument 1"
+ " of function SORT_ARRAY must be "
+ serdeConstants.LIST_TYPE_NAME
+ "<"
+ Category.PRIMITIVE
+ ">, but "
+ arguments[0].getTypeName()
+ " was found.");
}
ObjectInspector elementObjectInspector =
((ListObjectInspector) (arguments[0])).getListElementObjectInspector();
argumentOIs = arguments;
converters = new Converter[arguments.length];
ObjectInspector returnOI = returnOIResolver.get();
if (returnOI == null) {
returnOI = elementObjectInspector;
}
converters[0] = ObjectInspectorConverters.getConverter(elementObjectInspector, returnOI);
return ObjectInspectorFactory.getStandardListObjectInspector(returnOI);
}
public Symbol(ExprNodeEvaluator symbolExprEval, ObjectInspector symbolOI) {
this.symbolExprEval = symbolExprEval;
converter =
ObjectInspectorConverters.getConverter(
symbolOI, PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
}
private MapOpCtx initObjectInspector(
Configuration hconf, MapOpCtx opCtx, StructObjectInspector tableRowOI) throws Exception {
PartitionDesc pd = opCtx.partDesc;
TableDesc td = pd.getTableDesc();
// Use table properties in case of unpartitioned tables,
// and the union of table properties and partition properties, with partition
// taking precedence, in the case of partitioned tables
Properties overlayedProps =
SerDeUtils.createOverlayedProperties(td.getProperties(), pd.getProperties());
Map<String, String> partSpec = pd.getPartSpec();
opCtx.tableName = String.valueOf(overlayedProps.getProperty("name"));
opCtx.partName = String.valueOf(partSpec);
opCtx.deserializer = pd.getDeserializer(hconf);
StructObjectInspector partRawRowObjectInspector;
if (Utilities.isInputFileFormatSelfDescribing(pd)) {
partRawRowObjectInspector = tableRowOI;
} else {
partRawRowObjectInspector = (StructObjectInspector) opCtx.deserializer.getObjectInspector();
}
opCtx.partTblObjectInspectorConverter =
ObjectInspectorConverters.getConverter(partRawRowObjectInspector, tableRowOI);
// Next check if this table has partitions and if so
// get the list of partition names as well as allocate
// the serdes for the partition columns
String pcols = overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS);
if (pcols != null && pcols.length() > 0) {
String[] partKeys = pcols.trim().split("/");
String pcolTypes =
overlayedProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES);
String[] partKeyTypes = pcolTypes.trim().split(":");
if (partKeys.length > partKeyTypes.length) {
throw new HiveException(
"Internal error : partKeys length, "
+ partKeys.length
+ " greater than partKeyTypes length, "
+ partKeyTypes.length);
}
List<String> partNames = new ArrayList<String>(partKeys.length);
Object[] partValues = new Object[partKeys.length];
List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>(partKeys.length);
for (int i = 0; i < partKeys.length; i++) {
String key = partKeys[i];
partNames.add(key);
ObjectInspector oi =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
TypeInfoFactory.getPrimitiveTypeInfo(partKeyTypes[i]));
// Partitions do not exist for this table
if (partSpec == null) {
// for partitionless table, initialize partValue to null
partValues[i] = null;
} else {
partValues[i] =
ObjectInspectorConverters.getConverter(
PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi)
.convert(partSpec.get(key));
}
partObjectInspectors.add(oi);
}
opCtx.rowWithPart = new Object[] {null, partValues};
opCtx.partObjectInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors);
}
// The op may not be a TableScan for mapjoins
// Consider the query: select /*+MAPJOIN(a)*/ count(*) FROM T1 a JOIN T2 b ON a.key = b.key;
// In that case, it will be a Select, but the rowOI need not be amended
if (opCtx.op instanceof TableScanOperator) {
TableScanOperator tsOp = (TableScanOperator) opCtx.op;
TableScanDesc tsDesc = tsOp.getConf();
if (tsDesc != null && tsDesc.hasVirtualCols()) {
opCtx.vcs = tsDesc.getVirtualCols();
opCtx.vcValues = new Object[opCtx.vcs.size()];
opCtx.vcsObjectInspector = VirtualColumn.getVCSObjectInspector(opCtx.vcs);
if (opCtx.isPartitioned()) {
opCtx.rowWithPartAndVC = Arrays.copyOfRange(opCtx.rowWithPart, 0, 3);
} else {
opCtx.rowWithPartAndVC = new Object[2];
}
}
}
if (!opCtx.hasVC() && !opCtx.isPartitioned()) {
opCtx.rowObjectInspector = tableRowOI;
return opCtx;
}
List<StructObjectInspector> inspectors = new ArrayList<StructObjectInspector>();
inspectors.add(tableRowOI);
if (opCtx.isPartitioned()) {
inspectors.add(opCtx.partObjectInspector);
}
if (opCtx.hasVC()) {
inspectors.add(opCtx.vcsObjectInspector);
}
opCtx.rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(inspectors);
return opCtx;
}
