public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
super.init(m, parameters);
// init output object inspectors
/// input will be key, value and batch size
LOG.info(" Init mode = " + m);
System.out.println(" Init mode = " + m);
System.out.println(" parameters = = " + parameters + " Length = " + parameters.length);
configMap = new HashMap<String, String>();
for (int k = 0; k < parameters.length; ++k) {
LOG.info("Param " + k + " is " + parameters[k]);
System.out.println("Param " + k + " is " + parameters[k]);
}
if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) {
configMap = HTableFactory.getConfigFromConstMapInspector(parameters[0]);
checkConfig(configMap);
inputKeyOI = (PrimitiveObjectInspector) parameters[1];
try {
LOG.info(" Initializing HTable ");
table = HTableFactory.getHTable(configMap);
if (configMap.containsKey(BATCH_SIZE_TAG)) {
batchSize = Integer.parseInt(configMap.get(BATCH_SIZE_TAG));
}
if (configMap.containsKey(DISABLE_AUTO_FLUSH)) {
disableAutoFlush = Boolean.valueOf(configMap.get(DISABLE_AUTO_FLUSH));
LOG.info("Disabling auto flush on hbase deletes");
}
// if (configMap.containsKey(DISABLE_WAL)) {
// disableWAL = Boolean.valueOf(configMap.get(DISABLE_WAL));
// LOG.info("Disabling WAL writes on hbase deletes");
// }
//
// if (configMap.containsKey(WRITE_BUFFER_SIZE_MB)) {
// writeBufferSizeBytes =
// Integer.parseInt(configMap.get(WRITE_BUFFER_SIZE_MB)) * 1024 * 1024;
// LOG.info("Setting habase write buffer size to: " +
// writeBufferSizeBytes);
// }
} catch (IOException e) {
throw new HiveException(e);
}
} else {
listKVOI = (StandardListObjectInspector) parameters[0];
}
if (m == Mode.PARTIAL1 || m == Mode.PARTIAL2) {
return ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector));
} else {
/// Otherwise return a message
return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
}
}
@Override
public ObjectInspector initialize(@Nonnull final ObjectInspector[] argOIs)
throws UDFArgumentException {
final int numArgOIs = argOIs.length;
if (numArgOIs < 2) {
throw new UDFArgumentException(
"argOIs.length must be greater that or equals to 2: " + numArgOIs);
}
this.featureNames = HiveUtils.getConstStringArray(argOIs[0]);
if (featureNames == null) {
throw new UDFArgumentException("#featureNames should not be null");
}
int numFeatureNames = featureNames.length;
if (numFeatureNames < 1) {
throw new UDFArgumentException(
"#featureNames must be greater than or equals to 1: " + numFeatureNames);
}
int numFeatures = numArgOIs - 1;
if (numFeatureNames != numFeatures) {
throw new UDFArgumentException(
"#featureNames '" + numFeatureNames + "' != #arguments '" + numFeatures + "'");
}
this.inputOIs = new PrimitiveObjectInspector[numFeatures];
for (int i = 0; i < numFeatures; i++) {
ObjectInspector oi = argOIs[i + 1];
inputOIs[i] = HiveUtils.asPrimitiveObjectInspector(oi);
}
this.result = new ArrayList<Text>(numFeatures);
return ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableStringObjectInspector);
}
private ObjectInspector solveOi(ObjectInspector arg) {
switch (arg.getCategory()) {
case PRIMITIVE:
// VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
// UNKNOWN
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getPrimitiveCategory());
case LIST:
return ObjectInspectorFactory.getStandardListObjectInspector(
solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
case MAP:
return ObjectInspectorFactory.getStandardMapObjectInspector(
solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) arg;
int size = soi.getAllStructFieldRefs().size();
ArrayList<String> fnl = new ArrayList<String>(size);
ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);
for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
fnl.add(sf.getFieldName());
foil.add(solveOi(sf.getFieldObjectInspector()));
}
return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
default:
return arg;
}
}
protected static StructObjectInspector createSelectListOI(
MatchPath evaluator, PTFInputDef inpDef) {
StructObjectInspector inOI = inpDef.getOutputShape().getOI();
ArrayList<String> inputColumnNames = new ArrayList<String>();
ArrayList<String> selectListNames = new ArrayList<String>();
ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
for (StructField f : inOI.getAllStructFieldRefs()) {
String inputColName = evaluator.inputColumnNamesMap.get(f.getFieldName());
if (inputColName != null) {
inputColumnNames.add(inputColName);
selectListNames.add(f.getFieldName());
fieldOIs.add(f.getFieldObjectInspector());
}
}
StandardListObjectInspector pathAttrOI =
ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, fieldOIs));
ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>();
selectFieldOIs.addAll(fieldOIs);
selectFieldOIs.add(pathAttrOI);
selectListNames.add(MatchPath.PATHATTR_NAME);
return ObjectInspectorFactory.getStandardStructObjectInspector(selectListNames, selectFieldOIs);
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length < 1 || arguments.length > 3) {
throw new UDFArgumentLengthException(
"The function sentences takes between 1 and 3 arguments.");
}
converters = new ObjectInspectorConverters.Converter[arguments.length];
for (int i = 0; i < arguments.length; i++) {
converters[i] =
ObjectInspectorConverters.getConverter(
arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector);
}
return ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableStringObjectInspector));
}
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
super.init(m, parameters);
if (m == Mode.PARTIAL1) {
inputOI = (PrimitiveObjectInspector) parameters[0];
return ObjectInspectorFactory.getStandardListObjectInspector(
(PrimitiveObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(inputOI));
} else {
if (!(parameters[0] instanceof StandardListObjectInspector)) {
inputOI =
(PrimitiveObjectInspector)
ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
return (StandardListObjectInspector)
ObjectInspectorFactory.getStandardListObjectInspector(inputOI);
} else {
internalMergeOI = (StandardListObjectInspector) parameters[0];
inputOI = (PrimitiveObjectInspector) internalMergeOI.getListElementObjectInspector();
loi =
(StandardListObjectInspector)
ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI);
return loi;
}
}
}
/*
* add array<struct> to the list of columns
*/
protected static RowResolver createSelectListRR(MatchPath evaluator, PTFInputDef inpDef)
throws SemanticException {
RowResolver rr = new RowResolver();
RowResolver inputRR = inpDef.getOutputShape().getRr();
evaluator.inputColumnNamesMap = new HashMap<String, String>();
ArrayList<String> inputColumnNames = new ArrayList<String>();
ArrayList<ObjectInspector> inpColOIs = new ArrayList<ObjectInspector>();
for (ColumnInfo inpCInfo : inputRR.getColumnInfos()) {
ColumnInfo cInfo = new ColumnInfo(inpCInfo);
String colAlias = cInfo.getAlias();
String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName());
if (tabColAlias != null) {
colAlias = tabColAlias[1];
}
ASTNode inExpr = null;
inExpr = PTFTranslator.getASTNode(inpCInfo, inputRR);
if (inExpr != null) {
rr.putExpression(inExpr, cInfo);
colAlias = inExpr.toStringTree().toLowerCase();
} else {
colAlias = colAlias == null ? cInfo.getInternalName() : colAlias;
rr.put(cInfo.getTabAlias(), colAlias, cInfo);
}
evaluator.inputColumnNamesMap.put(cInfo.getInternalName(), colAlias);
inputColumnNames.add(colAlias);
inpColOIs.add(cInfo.getObjectInspector());
}
StandardListObjectInspector pathAttrOI =
ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, inpColOIs));
ColumnInfo pathColumn =
new ColumnInfo(
PATHATTR_NAME,
TypeInfoUtils.getTypeInfoFromObjectInspector(pathAttrOI),
null,
false,
false);
rr.put(null, PATHATTR_NAME, pathColumn);
return rr;
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length != 1) {
throw new UDFArgumentLengthException("The function MAP_VALUES only accepts 1 argument.");
} else if (!(arguments[0] instanceof MapObjectInspector)) {
throw new UDFArgumentTypeException(
0,
"\""
+ Category.MAP.toString().toLowerCase()
+ "\" is expected at function MAP_VALUES, "
+ "but \""
+ arguments[0].getTypeName()
+ "\" is found");
}
mapOI = (MapObjectInspector) arguments[0];
ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();
return ObjectInspectorFactory.getStandardListObjectInspector(mapValueOI);
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver;
returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
if (arguments.length != 1) {
throw new UDFArgumentLengthException(
"The function SORT_ARRAY(array(obj1, obj2,...)) needs one argument.");
}
switch (arguments[0].getCategory()) {
case LIST:
if (((ListObjectInspector) (arguments[0]))
.getListElementObjectInspector()
.getCategory()
.equals(Category.PRIMITIVE)) {
break;
}
default:
throw new UDFArgumentTypeException(
0,
"Argument 1"
+ " of function SORT_ARRAY must be "
+ serdeConstants.LIST_TYPE_NAME
+ "<"
+ Category.PRIMITIVE
+ ">, but "
+ arguments[0].getTypeName()
+ " was found.");
}
ObjectInspector elementObjectInspector =
((ListObjectInspector) (arguments[0])).getListElementObjectInspector();
argumentOIs = arguments;
converters = new Converter[arguments.length];
ObjectInspector returnOI = returnOIResolver.get();
if (returnOI == null) {
returnOI = elementObjectInspector;
}
converters[0] = ObjectInspectorConverters.getConverter(elementObjectInspector, returnOI);
return ObjectInspectorFactory.getStandardListObjectInspector(returnOI);
}
private static ObjectInspector getObjectInspector(TypeInfo type) throws IOException {
switch (type.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
primitiveType.getPrimitiveCategory());
case MAP:
MapTypeInfo mapType = (MapTypeInfo) type;
MapObjectInspector mapInspector =
ObjectInspectorFactory.getStandardMapObjectInspector(
getObjectInspector(mapType.getMapKeyTypeInfo()),
getObjectInspector(mapType.getMapValueTypeInfo()));
return mapInspector;
case LIST:
ListTypeInfo listType = (ListTypeInfo) type;
ListObjectInspector listInspector =
ObjectInspectorFactory.getStandardListObjectInspector(
getObjectInspector(listType.getListElementTypeInfo()));
return listInspector;
case STRUCT:
StructTypeInfo structType = (StructTypeInfo) type;
List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();
List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
for (TypeInfo fieldType : fieldTypes) {
fieldInspectors.add(getObjectInspector(fieldType));
}
StructObjectInspector structInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(
structType.getAllStructFieldNames(), fieldInspectors);
return structInspector;
default:
throw new IOException("Unknown field schema type");
}
}
/**
* The initialize method is called only once during the lifetime of the UDF.
*
* <p>Method checks for the validity (number, type, etc) of the arguments being passed to the UDF.
* It also sets the return type of the result of the UDF, in this case the ObjectInspector
* equivalent of Map<String,Object>
*
* @param arguments
* @return ObjectInspector Map<String,Object>
* @throws UDFArgumentException
*/
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length != 1) {
throw new UDFArgumentLengthException(
"The HostNormalizerUDF takes an array with only 1 element as argument");
}
// we are expecting the parameter to be of String type.
ObjectInspector arg = arguments[0];
int argIndex = 0;
if (arg.getCategory() != Category.PRIMITIVE) {
throw new UDFArgumentTypeException(
argIndex,
"A string argument was expected but an argument of type "
+ arg.getTypeName()
+ " was given.");
}
// Now that we have made sure that the argument is of primitive type, we can get the primitive
// category
PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) arg).getPrimitiveCategory();
if (primitiveCategory != PrimitiveCategory.STRING) {
throw new UDFArgumentTypeException(
argIndex,
"A string argument was expected but an argument of type "
+ arg.getTypeName()
+ " was given.");
}
// Instantiate the Webrequest
webrequest = Webrequest.getInstance();
argumentOI = (StringObjectInspector) arg;
List<String> fieldNames = new LinkedList<>();
List<ObjectInspector> fieldOIs = new LinkedList<>();
int idx = 0;
fieldNames.add("project_class");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
IDX_PROJECT_CLASS = idx++;
fieldNames.add("project");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
IDX_PROJECT = idx++;
fieldNames.add("qualifiers");
fieldOIs.add(
ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector));
IDX_QUALIFIERS = idx++;
fieldNames.add("tld");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
IDX_TLD = idx++;
result = new Object[idx];
return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
