protected static StructObjectInspector createSelectListOI(
MatchPath evaluator, PTFInputDef inpDef) {
StructObjectInspector inOI = inpDef.getOutputShape().getOI();
ArrayList<String> inputColumnNames = new ArrayList<String>();
ArrayList<String> selectListNames = new ArrayList<String>();
ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
for (StructField f : inOI.getAllStructFieldRefs()) {
String inputColName = evaluator.inputColumnNamesMap.get(f.getFieldName());
if (inputColName != null) {
inputColumnNames.add(inputColName);
selectListNames.add(f.getFieldName());
fieldOIs.add(f.getFieldObjectInspector());
}
}
StandardListObjectInspector pathAttrOI =
ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, fieldOIs));
ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>();
selectFieldOIs.addAll(fieldOIs);
selectFieldOIs.add(pathAttrOI);
selectListNames.add(MatchPath.PATHATTR_NAME);
return ObjectInspectorFactory.getStandardStructObjectInspector(selectListNames, selectFieldOIs);
}
private ObjectInspector solveOi(ObjectInspector arg) {
switch (arg.getCategory()) {
case PRIMITIVE:
// VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
// UNKNOWN
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getPrimitiveCategory());
case LIST:
return ObjectInspectorFactory.getStandardListObjectInspector(
solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
case MAP:
return ObjectInspectorFactory.getStandardMapObjectInspector(
solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) arg;
int size = soi.getAllStructFieldRefs().size();
ArrayList<String> fnl = new ArrayList<String>(size);
ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);
for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
fnl.add(sf.getFieldName());
foil.add(solveOi(sf.getFieldObjectInspector()));
}
return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
default:
return arg;
}
}
/**
* Initializes array of ExprNodeEvaluator. Adds Union field for distinct column indices for group
* by. Puts the return values into a StructObjectInspector with output column names.
*
* <p>If distinctColIndices is empty, the object inspector is same as {@link
* Operator#initEvaluatorsAndReturnStruct(ExprNodeEvaluator[], List, ObjectInspector)}
*/
protected static StructObjectInspector initEvaluatorsAndReturnStruct(
ExprNodeEvaluator[] evals,
List<List<Integer>> distinctColIndices,
List<String> outputColNames,
int length,
ObjectInspector rowInspector)
throws HiveException {
int inspectorLen = evals.length > length ? length + 1 : evals.length;
List<ObjectInspector> sois = new ArrayList<ObjectInspector>(inspectorLen);
// keys
ObjectInspector[] fieldObjectInspectors = initEvaluators(evals, 0, length, rowInspector);
sois.addAll(Arrays.asList(fieldObjectInspectors));
if (outputColNames.size() > length) {
// union keys
assert distinctColIndices != null;
List<ObjectInspector> uois = new ArrayList<ObjectInspector>();
for (List<Integer> distinctCols : distinctColIndices) {
List<String> names = new ArrayList<String>();
List<ObjectInspector> eois = new ArrayList<ObjectInspector>();
int numExprs = 0;
for (int i : distinctCols) {
names.add(HiveConf.getColumnInternalName(numExprs));
eois.add(evals[i].initialize(rowInspector));
numExprs++;
}
uois.add(ObjectInspectorFactory.getStandardStructObjectInspector(names, eois));
}
UnionObjectInspector uoi = ObjectInspectorFactory.getStandardUnionObjectInspector(uois);
sois.add(uoi);
}
return ObjectInspectorFactory.getStandardStructObjectInspector(outputColNames, sois);
}
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
super.init(m, parameters);
// init output object inspectors
/// input will be key, value and batch size
LOG.info(" Init mode = " + m);
System.out.println(" Init mode = " + m);
System.out.println(" parameters = = " + parameters + " Length = " + parameters.length);
configMap = new HashMap<String, String>();
for (int k = 0; k < parameters.length; ++k) {
LOG.info("Param " + k + " is " + parameters[k]);
System.out.println("Param " + k + " is " + parameters[k]);
}
if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) {
configMap = HTableFactory.getConfigFromConstMapInspector(parameters[0]);
checkConfig(configMap);
inputKeyOI = (PrimitiveObjectInspector) parameters[1];
try {
LOG.info(" Initializing HTable ");
table = HTableFactory.getHTable(configMap);
if (configMap.containsKey(BATCH_SIZE_TAG)) {
batchSize = Integer.parseInt(configMap.get(BATCH_SIZE_TAG));
}
if (configMap.containsKey(DISABLE_AUTO_FLUSH)) {
disableAutoFlush = Boolean.valueOf(configMap.get(DISABLE_AUTO_FLUSH));
LOG.info("Disabling auto flush on hbase deletes");
}
// if (configMap.containsKey(DISABLE_WAL)) {
// disableWAL = Boolean.valueOf(configMap.get(DISABLE_WAL));
// LOG.info("Disabling WAL writes on hbase deletes");
// }
//
// if (configMap.containsKey(WRITE_BUFFER_SIZE_MB)) {
// writeBufferSizeBytes =
// Integer.parseInt(configMap.get(WRITE_BUFFER_SIZE_MB)) * 1024 * 1024;
// LOG.info("Setting habase write buffer size to: " +
// writeBufferSizeBytes);
// }
} catch (IOException e) {
throw new HiveException(e);
}
} else {
listKVOI = (StandardListObjectInspector) parameters[0];
}
if (m == Mode.PARTIAL1 || m == Mode.PARTIAL2) {
return ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector));
} else {
/// Otherwise return a message
return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
}
}
/*
* add array<struct> to the list of columns
*/
protected static RowResolver createSelectListRR(MatchPath evaluator, PTFInputDef inpDef)
throws SemanticException {
RowResolver rr = new RowResolver();
RowResolver inputRR = inpDef.getOutputShape().getRr();
evaluator.inputColumnNamesMap = new HashMap<String, String>();
ArrayList<String> inputColumnNames = new ArrayList<String>();
ArrayList<ObjectInspector> inpColOIs = new ArrayList<ObjectInspector>();
for (ColumnInfo inpCInfo : inputRR.getColumnInfos()) {
ColumnInfo cInfo = new ColumnInfo(inpCInfo);
String colAlias = cInfo.getAlias();
String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName());
if (tabColAlias != null) {
colAlias = tabColAlias[1];
}
ASTNode inExpr = null;
inExpr = PTFTranslator.getASTNode(inpCInfo, inputRR);
if (inExpr != null) {
rr.putExpression(inExpr, cInfo);
colAlias = inExpr.toStringTree().toLowerCase();
} else {
colAlias = colAlias == null ? cInfo.getInternalName() : colAlias;
rr.put(cInfo.getTabAlias(), colAlias, cInfo);
}
evaluator.inputColumnNamesMap.put(cInfo.getInternalName(), colAlias);
inputColumnNames.add(colAlias);
inpColOIs.add(cInfo.getObjectInspector());
}
StandardListObjectInspector pathAttrOI =
ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, inpColOIs));
ColumnInfo pathColumn =
new ColumnInfo(
PATHATTR_NAME,
TypeInfoUtils.getTypeInfoFromObjectInspector(pathAttrOI),
null,
false,
false);
rr.put(null, PATHATTR_NAME, pathColumn);
return rr;
}
public void initialize(Configuration job, Properties tbl) throws SerDeException {
separator = DefaultSeparator;
String alt_sep = tbl.getProperty("testserde.default.serialization.format");
if (alt_sep != null && alt_sep.length() > 0) {
try {
byte b[] = new byte[1];
b[0] = Byte.valueOf(alt_sep).byteValue();
separator = new String(b);
} catch (NumberFormatException e) {
separator = alt_sep;
}
}
String columnProperty = tbl.getProperty("columns");
if (columnProperty == null || columnProperty.length() == 0) {
// Hack for tables with no columns
// Treat it as a table with a single column called "col"
cachedObjectInspector =
ObjectInspectorFactory.getReflectionObjectInspector(
ColumnSet.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
} else {
columnNames = Arrays.asList(columnProperty.split(","));
cachedObjectInspector = MetadataListStructObjectInspector.getInstance(columnNames);
}
LOG.info(getClass().getName() + ": initialized with columnNames: " + columnNames);
}
private void buildSelectListEvaluators() throws SemanticException, HiveException {
resultExprInfo = new ResultExprInfo();
resultExprInfo.resultExprEvals = new ArrayList<ExprNodeEvaluator>();
resultExprInfo.resultExprNames = new ArrayList<String>();
resultExprInfo.resultExprNodes = new ArrayList<ExprNodeDesc>();
// result
ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>();
int i = 0;
for (WindowExpressionSpec expr : selectSpec) {
String selectColName = expr.getAlias();
ASTNode selectColumnNode = expr.getExpression();
ExprNodeDesc selectColumnExprNode =
ResultExpressionParser.buildExprNode(selectColumnNode, selectListInputTypeCheckCtx);
ExprNodeEvaluator selectColumnExprEval = ExprNodeEvaluatorFactory.get(selectColumnExprNode);
ObjectInspector selectColumnOI = null;
selectColumnOI = selectColumnExprEval.initialize(selectListInputOI);
selectColName = getColumnName(selectColName, selectColumnExprNode, i);
resultExprInfo.resultExprEvals.add(selectColumnExprEval);
selectListExprOIs.add(selectColumnOI);
resultExprInfo.resultExprNodes.add(selectColumnExprNode);
resultExprInfo.resultExprNames.add(selectColName);
i++;
}
resultExprInfo.resultOI =
ObjectInspectorFactory.getStandardStructObjectInspector(
resultExprInfo.resultExprNames, selectListExprOIs);
}
@Override
public ObjectInspector initialize(@Nonnull final ObjectInspector[] argOIs)
throws UDFArgumentException {
final int numArgOIs = argOIs.length;
if (numArgOIs < 2) {
throw new UDFArgumentException(
"argOIs.length must be greater that or equals to 2: " + numArgOIs);
}
this.featureNames = HiveUtils.getConstStringArray(argOIs[0]);
if (featureNames == null) {
throw new UDFArgumentException("#featureNames should not be null");
}
int numFeatureNames = featureNames.length;
if (numFeatureNames < 1) {
throw new UDFArgumentException(
"#featureNames must be greater than or equals to 1: " + numFeatureNames);
}
int numFeatures = numArgOIs - 1;
if (numFeatureNames != numFeatures) {
throw new UDFArgumentException(
"#featureNames '" + numFeatureNames + "' != #arguments '" + numFeatures + "'");
}
this.inputOIs = new PrimitiveObjectInspector[numFeatures];
for (int i = 0; i < numFeatures; i++) {
ObjectInspector oi = argOIs[i + 1];
inputOIs[i] = HiveUtils.asPrimitiveObjectInspector(oi);
}
this.result = new ArrayList<Text>(numFeatures);
return ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableStringObjectInspector);
}
// Initialize this SerDe with the system properties and table properties
@Override
public void initialize(Configuration sysProps, Properties tblProps) throws SerDeException {
LOG.debug("Initializing QueryStringSerDe");
// Get the names of the columns for the table this SerDe is being used
// with
String columnNameProperty = tblProps.getProperty(serdeConstants.LIST_COLUMNS);
columnNames = Arrays.asList(columnNameProperty.split(","));
// Convert column types from text to TypeInfo objects
String columnTypeProperty = tblProps.getProperty(serdeConstants.LIST_COLUMN_TYPES);
columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
assert columnNames.size() == columnTypes.size();
numColumns = columnNames.size();
// Create ObjectInspectors from the type information for each column
List<ObjectInspector> columnOIs = new ArrayList<ObjectInspector>(columnNames.size());
ObjectInspector oi;
for (int c = 0; c < numColumns; c++) {
oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(columnTypes.get(c));
columnOIs.add(oi);
}
rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs);
// Create an empty row object to be reused during deserialization
row = new ArrayList<Object>(numColumns);
for (int c = 0; c < numColumns; c++) {
row.add(null);
}
LOG.debug("QueryStringSerDe initialization complete");
}
@Override
public void initialize(final Configuration conf, final Properties tbl) throws SerDeException {
List<String> columnNames = Arrays.asList(tbl.getProperty(Constants.LIST_COLUMNS).split(","));
List<TypeInfo> columnTypes =
TypeInfoUtils.getTypeInfosFromTypeString(tbl.getProperty(Constants.LIST_COLUMN_TYPES));
numCols = columnNames.size();
List<ObjectInspector> columnOIs = new ArrayList<ObjectInspector>(numCols);
for (int i = 0; i < numCols; i++) {
columnOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
}
this.inspector =
ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs);
this.outputFields = new String[numCols];
row = new ArrayList<String>(numCols);
for (int i = 0; i < numCols; i++) {
row.add(null);
}
separatorChar = getProperty(tbl, "separatorChar", CSVWriter.DEFAULT_SEPARATOR);
quoteChar = getProperty(tbl, "quoteChar", CSVWriter.DEFAULT_QUOTE_CHARACTER);
escapeChar = getProperty(tbl, "escapeChar", CSVWriter.DEFAULT_ESCAPE_CHARACTER);
}
@Test
public void testBufferSizeFor1000Col() throws IOException {
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector =
ObjectInspectorFactory.getReflectionObjectInspector(
Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
int bufferSize = 128 * 1024;
String columns = getRandomColumnNames(1000);
// just for testing. manually write the column names
conf.set(IOConstants.COLUMNS, columns);
Writer writer =
OrcFile.createWriter(
testFilePath,
OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(100000)
.compress(CompressionKind.NONE)
.bufferSize(bufferSize));
final int newBufferSize;
if (writer instanceof WriterImpl) {
WriterImpl orcWriter = (WriterImpl) writer;
newBufferSize = orcWriter.getEstimatedBufferSize(bufferSize);
assertEquals(bufferSize, newBufferSize);
}
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length % 2 != 0) {
throw new UDFArgumentLengthException("Arguments must be in key/value pairs");
}
GenericUDFUtils.ReturnObjectInspectorResolver keyOIResolver =
new GenericUDFUtils.ReturnObjectInspectorResolver(true);
GenericUDFUtils.ReturnObjectInspectorResolver valueOIResolver =
new GenericUDFUtils.ReturnObjectInspectorResolver(true);
for (int i = 0; i < arguments.length; i++) {
if (i % 2 == 0) {
// Keys
if (!(arguments[i] instanceof PrimitiveObjectInspector)) {
throw new UDFArgumentTypeException(
1, "Primitive Type is expected but " + arguments[i].getTypeName() + "\" is found");
}
if (!keyOIResolver.update(arguments[i])) {
throw new UDFArgumentTypeException(
i,
"Key type \""
+ arguments[i].getTypeName()
+ "\" is different from preceding key types. "
+ "Previous key type was \""
+ arguments[i - 2].getTypeName()
+ "\"");
}
} else {
// Values
if (!valueOIResolver.update(arguments[i])
&& !compatibleTypes(arguments[i], arguments[i - 2])) {
throw new UDFArgumentTypeException(
i,
"Value type \""
+ arguments[i].getTypeName()
+ "\" is different from preceding value types. "
+ "Previous value type was \""
+ arguments[i - 2].getTypeName()
+ "\"");
}
}
}
ObjectInspector keyOI =
keyOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
ObjectInspector valueOI =
valueOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
converters = new Converter[arguments.length];
for (int i = 0; i < arguments.length; i++) {
converters[i] =
ObjectInspectorConverters.getConverter(arguments[i], i % 2 == 0 ? keyOI : valueOI);
}
return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI);
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length < 1 || arguments.length > 3) {
throw new UDFArgumentLengthException(
"The function sentences takes between 1 and 3 arguments.");
}
converters = new ObjectInspectorConverters.Converter[arguments.length];
for (int i = 0; i < arguments.length; i++) {
converters[i] =
ObjectInspectorConverters.getConverter(
arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector);
}
return ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableStringObjectInspector));
}
public static ObjectInspector createColumnarStructInspector(
List<String> columnNames, List<TypeInfo> columnTypes) {
ArrayList<ObjectInspector> columnObjectInspectors =
new ArrayList<ObjectInspector>(columnTypes.size());
for (int i = 0; i < columnTypes.size(); i++) {
columnObjectInspectors.add(
LazyBinaryUtils.getLazyBinaryObjectInspectorFromTypeInfo(columnTypes.get(i)));
}
return ObjectInspectorFactory.getColumnarStructObjectInspector(
columnNames, columnObjectInspectors);
}
private static ObjectInspector getObjectInspector(TypeInfo type) throws IOException {
switch (type.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
primitiveType.getPrimitiveCategory());
case MAP:
MapTypeInfo mapType = (MapTypeInfo) type;
MapObjectInspector mapInspector =
ObjectInspectorFactory.getStandardMapObjectInspector(
getObjectInspector(mapType.getMapKeyTypeInfo()),
getObjectInspector(mapType.getMapValueTypeInfo()));
return mapInspector;
case LIST:
ListTypeInfo listType = (ListTypeInfo) type;
ListObjectInspector listInspector =
ObjectInspectorFactory.getStandardListObjectInspector(
getObjectInspector(listType.getListElementTypeInfo()));
return listInspector;
case STRUCT:
StructTypeInfo structType = (StructTypeInfo) type;
List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();
List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
for (TypeInfo fieldType : fieldTypes) {
fieldInspectors.add(getObjectInspector(fieldType));
}
StructObjectInspector structInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(
structType.getAllStructFieldNames(), fieldInspectors);
return structInspector;
default:
throw new IOException("Unknown field schema type");
}
}
/** Return (string rowid, int label, float probability) as a result */
@Override
public StructObjectInspector getReturnOI() {
final ArrayList fieldNames = new ArrayList(3);
final ArrayList fieldOIs = new ArrayList(3);
fieldNames.add("rowid");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldNames.add("label");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldNames.add("probability");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaFloatObjectInspector);
return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
/* Initialize the UDAF on the map side. */
private void initMapSide(ObjectInspector[] parameters) throws HiveException {
int length = parameters.length;
outputOIs = new ObjectInspector[length];
List<String> fieldNames = new ArrayList<String>(length);
List<ObjectInspector> fieldOIs = Arrays.asList(outputOIs);
for (int i = 0; i < length; i++) {
fieldNames.add("col" + i); // field names are not made available! :(
outputOIs[i] = ObjectInspectorUtils.getStandardObjectInspector(parameters[i]);
}
inputOIs = parameters;
structOI = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
super.init(m, parameters);
if (m == Mode.PARTIAL1) {
inputOI = (PrimitiveObjectInspector) parameters[0];
return ObjectInspectorFactory.getStandardListObjectInspector(
(PrimitiveObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(inputOI));
} else {
if (!(parameters[0] instanceof StandardListObjectInspector)) {
inputOI =
(PrimitiveObjectInspector)
ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
return (StandardListObjectInspector)
ObjectInspectorFactory.getStandardListObjectInspector(inputOI);
} else {
internalMergeOI = (StandardListObjectInspector) parameters[0];
inputOI = (PrimitiveObjectInspector) internalMergeOI.getListElementObjectInspector();
loi =
(StandardListObjectInspector)
ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI);
return loi;
}
}
}
@Override
public void close(boolean b) throws IOException {
// if we haven't written any rows, we need to create a file with a
// generic schema.
if (writer == null) {
// a row with no columns
ObjectInspector inspector =
ObjectInspectorFactory.getStandardStructObjectInspector(
new ArrayList<String>(), new ArrayList<ObjectInspector>());
options.inspector(inspector);
writer = OrcFile.createWriter(path, options);
}
writer.close();
}
@Override
public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException {
if (args.length != 1) {
throw new UDFArgumentException("explode() takes only one argument");
}
if (args[0].getCategory() != ObjectInspector.Category.LIST) {
throw new UDFArgumentException("explode() takes an array as a parameter");
}
listOI = (ListObjectInspector) args[0];
ArrayList<String> fieldNames = new ArrayList<String>();
ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
fieldNames.add("col");
fieldOIs.add(listOI.getListElementObjectInspector());
return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver;
returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
if (arguments.length != 1) {
throw new UDFArgumentLengthException(
"The function SORT_ARRAY(array(obj1, obj2,...)) needs one argument.");
}
switch (arguments[0].getCategory()) {
case LIST:
if (((ListObjectInspector) (arguments[0]))
.getListElementObjectInspector()
.getCategory()
.equals(Category.PRIMITIVE)) {
break;
}
default:
throw new UDFArgumentTypeException(
0,
"Argument 1"
+ " of function SORT_ARRAY must be "
+ serdeConstants.LIST_TYPE_NAME
+ "<"
+ Category.PRIMITIVE
+ ">, but "
+ arguments[0].getTypeName()
+ " was found.");
}
ObjectInspector elementObjectInspector =
((ListObjectInspector) (arguments[0])).getListElementObjectInspector();
argumentOIs = arguments;
converters = new Converter[arguments.length];
ObjectInspector returnOI = returnOIResolver.get();
if (returnOI == null) {
returnOI = elementObjectInspector;
}
converters[0] = ObjectInspectorConverters.getConverter(elementObjectInspector, returnOI);
return ObjectInspectorFactory.getStandardListObjectInspector(returnOI);
}
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length != 1) {
throw new UDFArgumentLengthException("The function MAP_VALUES only accepts 1 argument.");
} else if (!(arguments[0] instanceof MapObjectInspector)) {
throw new UDFArgumentTypeException(
0,
"\""
+ Category.MAP.toString().toLowerCase()
+ "\" is expected at function MAP_VALUES, "
+ "but \""
+ arguments[0].getTypeName()
+ "\" is found");
}
mapOI = (MapObjectInspector) arguments[0];
ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();
return ObjectInspectorFactory.getStandardListObjectInspector(mapValueOI);
}
@Override
protected void initializeOp(Configuration hconf) throws HiveException {
super.initializeOp(hconf);
// Just forward the row as is
if (conf.isSelStarNoCompute()) {
return;
}
List<ObjectInspector> objectInspectors = new ArrayList<ObjectInspector>();
List<ExprNodeDesc> colList = conf.getColList();
valueWriters = VectorExpressionWriterFactory.getExpressionWriters(colList);
for (VectorExpressionWriter vew : valueWriters) {
objectInspectors.add(vew.getObjectInspector());
}
List<String> outputFieldNames = conf.getOutputColumnNames();
outputObjInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(outputFieldNames, objectInspectors);
}
/*
* (non-Javadoc)
*
* @see org.apache.hadoop.hive.serde2.AbstractSerDe#initialize(org.apache.hadoop.conf.Configuration,
* java.util.Properties)
*/
@Override
public void initialize(Configuration conf, Properties tbl) throws SerDeException {
String columnNameProperty = tbl.getProperty(LIST_COLUMNS);
String columnTypeProperty = tbl.getProperty(LIST_COLUMN_TYPES);
List<String> columnNames = Arrays.asList(columnNameProperty.split(","));
List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
List<ObjectInspector> columnObjectInspectors =
new ArrayList<ObjectInspector>(columnNames.size());
ObjectInspector colObjectInspector;
for (int col = 0; col < columnNames.size(); col++) {
colObjectInspector =
TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(columnTypes.get(col));
columnObjectInspectors.add(colObjectInspector);
}
cachedObjectInspector =
ObjectInspectorFactory.getColumnarStructObjectInspector(
columnNames, columnObjectInspectors);
}
static StandardStructObjectInspector structObjectInspector(Properties tableProperties) {
// extract column info - don't use Hive constants as they were renamed in 0.9 breaking
// compatibility
// the column names are saved as the given inspector to #serialize doesn't preserves them (maybe
// because it's an external table)
// use the class since StructType requires it ...
List<String> columnNames =
StringUtils.tokenize(tableProperties.getProperty(HiveConstants.COLUMNS), ",");
List<TypeInfo> colTypes =
TypeInfoUtils.getTypeInfosFromTypeString(
tableProperties.getProperty(HiveConstants.COLUMNS_TYPES));
// create a standard writable Object Inspector - used later on by serialization/deserialization
List<ObjectInspector> inspectors = new ArrayList<ObjectInspector>();
for (TypeInfo typeInfo : colTypes) {
inspectors.add(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo));
}
return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors);
}
static StructObjectInspector createStructObjectInspector(HCatSchema outputSchema)
throws IOException {
if (outputSchema == null) {
throw new IOException("Invalid output schema specified");
}
List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
List<String> fieldNames = new ArrayList<String>();
for (HCatFieldSchema hcatFieldSchema : outputSchema.getFields()) {
TypeInfo type = TypeInfoUtils.getTypeInfoFromTypeString(hcatFieldSchema.getTypeString());
fieldNames.add(hcatFieldSchema.getName());
fieldInspectors.add(getObjectInspector(type));
}
StructObjectInspector structInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors);
return structInspector;
}
@Override
public void initializeOutputOI() throws HiveException {
try {
MatchPath evaluator = (MatchPath) getEvaluator();
PartitionedTableFunctionDef tDef = evaluator.getTableDef();
List<PTFExpressionDef> args = tDef.getArgs();
int argsNum = args.size();
validateAndSetupPatternStr(evaluator, args);
validateAndSetupSymbolInfo(evaluator, args, argsNum);
validateAndSetupResultExprStr(evaluator, args, argsNum);
setupSymbolFunctionChain(evaluator);
/*
* setup OI for input to resultExpr select list
*/
StructObjectInspector selectListInputOI =
MatchPath.createSelectListOI(evaluator, tDef.getInput());
ResultExprInfo resultExprInfo = evaluator.resultExprInfo;
ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>();
resultExprInfo.resultExprEvals = new ArrayList<ExprNodeEvaluator>();
for (int i = 0; i < resultExprInfo.resultExprNodes.size(); i++) {
ExprNodeDesc selectColumnExprNode = resultExprInfo.resultExprNodes.get(i);
ExprNodeEvaluator selectColumnExprEval =
ExprNodeEvaluatorFactory.get(selectColumnExprNode);
ObjectInspector selectColumnOI = selectColumnExprEval.initialize(selectListInputOI);
resultExprInfo.resultExprEvals.add(selectColumnExprEval);
selectListExprOIs.add(selectColumnOI);
}
resultExprInfo.resultOI =
ObjectInspectorFactory.getStandardStructObjectInspector(
resultExprInfo.resultExprNames, selectListExprOIs);
setOutputOI(resultExprInfo.resultOI);
} catch (SemanticException se) {
throw new HiveException(se);
}
}
@Override
public void initialize(Configuration cfg, Properties props) throws SerDeException {
String columnNameProperty = props.getProperty(serdeConstants.LIST_COLUMNS);
columnNames = Arrays.asList(columnNameProperty.split(","));
numColumns = columnNames.size();
String columnTypeProperty = props.getProperty(serdeConstants.LIST_COLUMN_TYPES);
List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
// Ensure we have the same number of column names and types
assert numColumns == columnTypes.size();
List<ObjectInspector> inspectors = new ArrayList<ObjectInspector>(numColumns);
row = new ArrayList<Object>(numColumns);
for (int c = 0; c < numColumns; c++) {
ObjectInspector oi =
TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(columnTypes.get(c));
inspectors.add(oi);
row.add(null);
}
inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors);
}
public Handler(
ObjectInspector inputObjInspector,
List<ExprNodeDesc> keyCols,
List<ExprNodeDesc> valueCols,
List<String> outputKeyColumnNames,
List<String> outputValueColumnNames,
Integer tag)
throws HiveException {
keyEval = new ExprNodeEvaluator[keyCols.size()];
int i = 0;
for (ExprNodeDesc e : keyCols) {
keyEval[i++] = ExprNodeEvaluatorFactory.get(e);
}
outputKey = new Object[keyEval.length];
valueEval = new ExprNodeEvaluator[valueCols.size()];
i = 0;
for (ExprNodeDesc e : valueCols) {
valueEval[i++] = ExprNodeEvaluatorFactory.get(e);
}
outputValue = new Object[valueEval.length];
this.tag = tag;
ObjectInspector keyObjectInspector =
initEvaluatorsAndReturnStruct(keyEval, outputKeyColumnNames, inputObjInspector);
ObjectInspector valueObjectInspector =
initEvaluatorsAndReturnStruct(valueEval, outputValueColumnNames, inputObjInspector);
List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
ois.add(keyObjectInspector);
ois.add(valueObjectInspector);
this.outputObjInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(
Utilities.reduceFieldNameList, ois);
this.forwardedRow = new ArrayList<Object>(Utilities.reduceFieldNameList.size());
}
@Test
public void testBufferSizeFor1Col() throws IOException {
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector =
ObjectInspectorFactory.getReflectionObjectInspector(
Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
int bufferSize = 128 * 1024;
Writer writer =
OrcFile.createWriter(
testFilePath,
OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(100000)
.compress(CompressionKind.NONE)
.bufferSize(bufferSize));
final int newBufferSize;
if (writer instanceof WriterImpl) {
WriterImpl orcWriter = (WriterImpl) writer;
newBufferSize = orcWriter.getEstimatedBufferSize(bufferSize);
assertEquals(bufferSize, newBufferSize);
}
}