protected static StructObjectInspector createSelectListOI(
MatchPath evaluator, PTFInputDef inpDef) {
StructObjectInspector inOI = inpDef.getOutputShape().getOI();
ArrayList<String> inputColumnNames = new ArrayList<String>();
ArrayList<String> selectListNames = new ArrayList<String>();
ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
for (StructField f : inOI.getAllStructFieldRefs()) {
String inputColName = evaluator.inputColumnNamesMap.get(f.getFieldName());
if (inputColName != null) {
inputColumnNames.add(inputColName);
selectListNames.add(f.getFieldName());
fieldOIs.add(f.getFieldObjectInspector());
}
}
StandardListObjectInspector pathAttrOI =
ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, fieldOIs));
ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>();
selectFieldOIs.addAll(fieldOIs);
selectFieldOIs.add(pathAttrOI);
selectListNames.add(MatchPath.PATHATTR_NAME);
return ObjectInspectorFactory.getStandardStructObjectInspector(selectListNames, selectFieldOIs);
}
private ObjectInspector solveOi(ObjectInspector arg) {
switch (arg.getCategory()) {
case PRIMITIVE:
// VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
// UNKNOWN
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getPrimitiveCategory());
case LIST:
return ObjectInspectorFactory.getStandardListObjectInspector(
solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
case MAP:
return ObjectInspectorFactory.getStandardMapObjectInspector(
solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) arg;
int size = soi.getAllStructFieldRefs().size();
ArrayList<String> fnl = new ArrayList<String>(size);
ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);
for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
fnl.add(sf.getFieldName());
foil.add(solveOi(sf.getFieldObjectInspector()));
}
return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
default:
return arg;
}
}
@Override
protected String extractField(Object target) {
if (target instanceof HiveType) {
HiveType type = (HiveType) target;
ObjectInspector inspector = type.getObjectInspector();
if (inspector instanceof StructObjectInspector) {
StructObjectInspector soi = (StructObjectInspector) inspector;
StructField field = soi.getStructFieldRef(fieldName);
ObjectInspector foi = field.getFieldObjectInspector();
Assert.isTrue(
foi.getCategory() == ObjectInspector.Category.PRIMITIVE,
String.format(
"Field [%s] needs to be a primitive; found [%s]", fieldName, foi.getTypeName()));
// expecting a writeable - simply do a toString
Object data = soi.getStructFieldData(type.getObject(), field);
if (data == null || data instanceof NullWritable) {
return StringUtils.EMPTY;
}
return data.toString();
}
}
return null;
}
/**
* Whether comparison is supported for this type. Currently all types that references any map are
* not comparable.
*/
public static boolean compareSupported(ObjectInspector oi) {
switch (oi.getCategory()) {
case PRIMITIVE:
return true;
case LIST:
ListObjectInspector loi = (ListObjectInspector) oi;
return compareSupported(loi.getListElementObjectInspector());
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int f = 0; f < fields.size(); f++) {
if (!compareSupported(fields.get(f).getFieldObjectInspector())) {
return false;
}
}
return true;
case MAP:
return false;
case UNION:
UnionObjectInspector uoi = (UnionObjectInspector) oi;
for (ObjectInspector eoi : uoi.getObjectInspectors()) {
if (!compareSupported(eoi)) {
return false;
}
}
return true;
default:
return false;
}
}
public void printFileInfo() throws Exception {
System.out.println("Reader: " + m_reader);
System.out.println("# Rows: " + m_reader.getNumberOfRows());
System.out.println("# Types in the file: " + m_types.size());
for (int i = 0; i < m_types.size(); i++) {
System.out.println("Type " + i + ": " + m_types.get(i).getKind());
}
System.out.println("Compression: " + m_reader.getCompression());
if (m_reader.getCompression() != CompressionKind.NONE) {
System.out.println("Compression size: " + m_reader.getCompressionSize());
}
m_oi = (StructObjectInspector) m_reader.getObjectInspector();
System.out.println("object inspector type category: " + m_oi.getCategory());
System.out.println("object inspector type name : " + m_oi.getTypeName());
System.out.println("Number of columns in the table: " + m_fields.size());
// Print the type info:
for (int i = 0; i < m_fields.size(); i++) {
System.out.println("Column " + i + " name: " + m_fields.get(i).getFieldName());
ObjectInspector lv_foi = m_fields.get(i).getFieldObjectInspector();
System.out.println("Column " + i + " type category: " + lv_foi.getCategory());
System.out.println("Column " + i + " type name: " + lv_foi.getTypeName());
}
}
/**
* Serializing means getting every field, and setting the appropriate JSONObject field. Actual
* serialization is done at the end when the whole JSON object is built
*
* @param serializer
* @param obj
* @param structObjectInspector
*/
private JSONObject serializeStruct(
Object obj, StructObjectInspector soi, List<String> columnNames) {
// do nothing for null struct
if (null == obj) {
return null;
}
JSONObject result = new JSONObject();
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int i = 0; i < fields.size(); i++) {
StructField sf = fields.get(i);
Object data = soi.getStructFieldData(obj, sf);
if (null != data) {
try {
// we want to serialize columns with their proper HIVE name,
// not the _col2 kind of name usually generated upstream
result.put(
getSerializedFieldName(columnNames, i, sf),
serializeField(data, sf.getFieldObjectInspector()));
} catch (JSONException ex) {
LOG.warn("Problem serializing", ex);
throw new RuntimeException(ex);
}
}
}
return result;
}
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
if (objInspector.getCategory() != Category.STRUCT) {
throw new SerDeException(
getClass().toString()
+ " can only serialize struct types, but we got: "
+ objInspector.getTypeName());
}
StructObjectInspector soi = (StructObjectInspector) objInspector;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < fields.size(); i++) {
if (i > 0) sb.append(separator);
Object column = soi.getStructFieldData(obj, fields.get(i));
if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) {
// For primitive object, serialize to plain string
sb.append(column == null ? nullString : column.toString());
} else {
// For complex object, serialize to JSON format
sb.append(SerDeUtils.getJSONString(column, fields.get(i).getFieldObjectInspector()));
}
}
serializeCache.set(sb.toString());
return serializeCache;
}
private void stringifyObject(StringBuilder buffer, Object obj, ObjectInspector inspector)
throws IOException {
if (inspector instanceof StructObjectInspector) {
buffer.append("{ ");
StructObjectInspector soi = (StructObjectInspector) inspector;
boolean isFirst = true;
for (StructField field : soi.getAllStructFieldRefs()) {
if (isFirst) {
isFirst = false;
} else {
buffer.append(", ");
}
buffer.append(field.getFieldName());
buffer.append(": ");
stringifyObject(
buffer, soi.getStructFieldData(obj, field), field.getFieldObjectInspector());
}
buffer.append(" }");
} else if (inspector instanceof PrimitiveObjectInspector) {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
buffer.append(poi.getPrimitiveJavaObject(obj).toString());
} else {
buffer.append("*unknown*");
}
}
private RowSet decodeFromString(List<Object> rows, RowSet rowSet)
throws SQLException, SerDeException {
getSerDe();
StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
Object[] deserializedFields = new Object[fieldRefs.size()];
Object rowObj;
ObjectInspector fieldOI;
int protocol = getProtocolVersion().getValue();
for (Object rowString : rows) {
try {
rowObj = serde.deserialize(new BytesWritable(((String) rowString).getBytes("UTF-8")));
} catch (UnsupportedEncodingException e) {
throw new SerDeException(e);
}
for (int i = 0; i < fieldRefs.size(); i++) {
StructField fieldRef = fieldRefs.get(i);
fieldOI = fieldRef.getFieldObjectInspector();
Object fieldData = soi.getStructFieldData(rowObj, fieldRef);
deserializedFields[i] = SerDeUtils.toThriftPayload(fieldData, fieldOI, protocol);
}
rowSet.addRow(deserializedFields);
}
return rowSet;
}
@Override
public Writable serialize(Object obj, ObjectInspector oi) throws SerDeException {
if (oi.getCategory() != Category.STRUCT) {
throw new VoltSerdeException(
getClass().toString()
+ " can only serialize struct types, but we got: "
+ oi.getTypeName());
}
VoltRecord vr = new VoltRecord(m_voltConf.getTableName());
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> structFields = soi.getAllStructFieldRefs();
List<Object> fieldValues = soi.getStructFieldsDataAsList(obj);
final int size = m_oig.getColumnTypes().size();
for (int i = 0; i < size; ++i) {
ObjectInspector fieldOI = structFields.get(i).getFieldObjectInspector();
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) fieldOI;
Object fieldValue = poi.getPrimitiveJavaObject(fieldValues.get(i));
if (poi.getTypeInfo().equals(TypeInfoFactory.timestampTypeInfo)) {
fieldValue = fieldValue != null ? new Date(((Timestamp) fieldValue).getTime()) : null;
}
vr.add(fieldValue);
}
return vr;
}
private static void addInputColumnsToList(
ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) {
StructObjectInspector OI = shape.getOI();
for (StructField f : OI.getAllStructFieldRefs()) {
fieldNames.add(f.getFieldName());
fieldOIs.add(f.getFieldObjectInspector());
}
}
/**
* @param oi - Input object inspector
* @param oiSettableProperties - Lookup map to cache the result.(If no caching, pass null)
* @return - true if : (1) oi is an instance of settable<DataType>OI. (2) All the embedded object
* inspectors are instances of settable<DataType>OI. If (1) or (2) is false, return false.
*/
public static boolean hasAllFieldsSettable(
ObjectInspector oi, Map<ObjectInspector, Boolean> oiSettableProperties) {
// If the result is already present in the cache, return it.
if (!(oiSettableProperties == null) && oiSettableProperties.containsKey(oi)) {
return oiSettableProperties.get(oi).booleanValue();
}
// If the top-level object inspector is non-settable return false
if (!(isInstanceOfSettableOI(oi))) {
return setOISettablePropertiesMap(oi, oiSettableProperties, false);
}
Boolean returnValue = true;
switch (oi.getCategory()) {
case PRIMITIVE:
break;
case STRUCT:
StructObjectInspector structOutputOI = (StructObjectInspector) oi;
List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
for (StructField listField : listFields) {
if (!hasAllFieldsSettable(listField.getFieldObjectInspector(), oiSettableProperties)) {
returnValue = false;
break;
}
}
break;
case LIST:
ListObjectInspector listOutputOI = (ListObjectInspector) oi;
returnValue =
hasAllFieldsSettable(
listOutputOI.getListElementObjectInspector(), oiSettableProperties);
break;
case MAP:
MapObjectInspector mapOutputOI = (MapObjectInspector) oi;
returnValue =
hasAllFieldsSettable(mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties)
&& hasAllFieldsSettable(
mapOutputOI.getMapValueObjectInspector(), oiSettableProperties);
break;
case UNION:
UnionObjectInspector unionOutputOI = (UnionObjectInspector) oi;
List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
for (ObjectInspector listField : unionListFields) {
if (!hasAllFieldsSettable(listField, oiSettableProperties)) {
returnValue = false;
break;
}
}
break;
default:
throw new RuntimeException(
"Hive internal error inside hasAllFieldsSettable : "
+ oi.getTypeName()
+ " not supported yet.");
}
return setOISettablePropertiesMap(oi, oiSettableProperties, returnValue);
}
@Override
public Writable serialize(final Object obj, final ObjectInspector inspector)
throws SerDeException {
final StructObjectInspector structInspector = (StructObjectInspector) inspector;
final List<? extends StructField> fields = structInspector.getAllStructFieldRefs();
if (fields.size() != columnNames.size()) {
throw new SerDeException(
String.format("Required %d columns, received %d.", columnNames.size(), fields.size()));
}
cachedWritable.clear();
for (int c = 0; c < fieldCount; c++) {
StructField structField = fields.get(c);
LOG.debug("fieldId=" + c + ",structField=" + structField.toString());
if (structField != null) {
final Object field = structInspector.getStructFieldData(obj, fields.get(c));
final AbstractPrimitiveObjectInspector fieldOI =
(AbstractPrimitiveObjectInspector) fields.get(c).getFieldObjectInspector();
Writable value = (Writable) fieldOI.getPrimitiveWritableObject(field);
if (value == null) {
continue;
}
LOG.debug("fieldCount=" + fieldCount + ",value=" + value.toString());
if (value instanceof IntWritable) {
cachedWritable.put(new Text(columnNames.get(c)), value);
} else if (value instanceof Text) {
cachedWritable.put(new Text(columnNames.get(c)), ((Text) value));
} else if (value instanceof LongWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((LongWritable) value));
} else if (value instanceof DoubleWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((DoubleWritable) value));
} else if (value instanceof FloatWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((FloatWritable) value));
} else if (value instanceof BooleanWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((BooleanWritable) value));
} else if (value instanceof ByteWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((ByteWritable) value));
} else if (value instanceof BytesWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((BytesWritable) value));
} else {
LOG.warn("fieldCount=" + fieldCount + ",type=" + value.getClass().getName());
}
}
}
return cachedWritable;
}
/**
* Copy fields in the input row to the output array of standard objects.
*
* @param result output list of standard objects.
* @param row input row.
* @param soi Object inspector for the to-be-copied columns.
* @param objectInspectorOption
*/
public static void copyToStandardObject(
List<Object> result,
Object row,
StructObjectInspector soi,
ObjectInspectorCopyOption objectInspectorOption) {
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (StructField f : fields) {
result.add(
copyToStandardObject(
soi.getStructFieldData(row, f), f.getFieldObjectInspector(), objectInspectorOption));
}
}
@Override
public void setKeyValue(Writable key, Writable val) throws SerDeException {
Object keyObj = keySerDe.deserialize(key), valObj = valSerDe.deserialize(val);
List<? extends StructField> keyFields = keySoi.getAllStructFieldRefs(),
valFields = valSoi.getAllStructFieldRefs();
for (int i = 0; i < keyFields.size(); ++i) {
keyObjs[i] = keySoi.getStructFieldData(keyObj, keyFields.get(i));
}
for (int i = 0; i < valFields.size(); ++i) {
valObjs[i] = valSoi.getStructFieldData(valObj, valFields.get(i));
}
}
private ArrayWritable createStruct(final Object obj, final StructObjectInspector inspector)
throws SerDeException {
final List<? extends StructField> fields = inspector.getAllStructFieldRefs();
final Writable[] arr = new Writable[fields.size()];
for (int i = 0; i < fields.size(); i++) {
final StructField field = fields.get(i);
final Object subObj = inspector.getStructFieldData(obj, field);
final ObjectInspector subInspector = field.getFieldObjectInspector();
arr[i] = createObject(subObj, subInspector);
}
return new ArrayWritable(Writable.class, arr);
}
/*
* from the prunedCols list filter out columns that refer to WindowFns or WindowExprs
* the returned list is set as the prunedList needed by the PTFOp.
*/
private ArrayList<String> prunedInputList(
List<String> prunedCols, WindowTableFunctionDef tDef) {
ArrayList<String> prunedInputCols = new ArrayList<String>();
StructObjectInspector OI = tDef.getInput().getOutputShape().getOI();
for (StructField f : OI.getAllStructFieldRefs()) {
String fName = f.getFieldName();
if (prunedCols.contains(fName)) {
prunedInputCols.add(fName);
}
}
return prunedInputCols;
}
/**
* Traverse all the partitions for a table, and get the OI for the table. Note that a conversion
* is required if any of the partition OI is different from the table OI. For eg. if the query
* references table T (partitions P1, P2), and P1's schema is same as T, whereas P2's scheme is
* different from T, conversion might be needed for both P1 and P2, since SettableOI might be
* needed for T
*/
private Map<TableDesc, StructObjectInspector> getConvertedOI(Configuration hconf)
throws HiveException {
Map<TableDesc, StructObjectInspector> tableDescOI =
new HashMap<TableDesc, StructObjectInspector>();
Set<TableDesc> identityConverterTableDesc = new HashSet<TableDesc>();
try {
Map<ObjectInspector, Boolean> oiSettableProperties = new HashMap<ObjectInspector, Boolean>();
for (String onefile : conf.getPathToAliases().keySet()) {
PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
TableDesc tableDesc = pd.getTableDesc();
Deserializer partDeserializer = pd.getDeserializer(hconf);
StructObjectInspector partRawRowObjectInspector;
if (Utilities.isInputFileFormatSelfDescribing(pd)) {
Deserializer tblDeserializer = tableDesc.getDeserializer(hconf);
partRawRowObjectInspector = (StructObjectInspector) tblDeserializer.getObjectInspector();
} else {
partRawRowObjectInspector = (StructObjectInspector) partDeserializer.getObjectInspector();
}
StructObjectInspector tblRawRowObjectInspector = tableDescOI.get(tableDesc);
if ((tblRawRowObjectInspector == null)
|| (identityConverterTableDesc.contains(tableDesc))) {
Deserializer tblDeserializer = tableDesc.getDeserializer(hconf);
tblRawRowObjectInspector =
(StructObjectInspector)
ObjectInspectorConverters.getConvertedOI(
partRawRowObjectInspector,
tblDeserializer.getObjectInspector(),
oiSettableProperties);
if (identityConverterTableDesc.contains(tableDesc)) {
if (!partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {
identityConverterTableDesc.remove(tableDesc);
}
} else if (partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {
identityConverterTableDesc.add(tableDesc);
}
tableDescOI.put(tableDesc, tblRawRowObjectInspector);
}
}
} catch (Exception e) {
throw new HiveException(e);
}
return tableDescOI;
}
private void parseStringColumn(int column) {
// don't include column number in message because it causes boxing which is expensive here
checkArgument(!isPartitionColumn[column], "Column is a partition key");
loaded[column] = true;
Object fieldData = rowInspector.getStructFieldData(rowData, structFields[column]);
if (fieldData == null) {
nulls[column] = true;
} else if (hiveTypes[column] == HiveType.MAP
|| hiveTypes[column] == HiveType.LIST
|| hiveTypes[column] == HiveType.STRUCT) {
// temporarily special case MAP, LIST, and STRUCT types as strings
slices[column] =
Slices.wrappedBuffer(
SerDeUtils.getJsonBytes(sessionTimeZone, fieldData, fieldInspectors[column]));
nulls[column] = false;
} else {
Object fieldValue =
((PrimitiveObjectInspector) fieldInspectors[column]).getPrimitiveJavaObject(fieldData);
checkState(fieldValue != null, "fieldValue should not be null");
if (fieldValue instanceof String) {
slices[column] = Slices.utf8Slice((String) fieldValue);
} else if (fieldValue instanceof byte[]) {
slices[column] = Slices.wrappedBuffer((byte[]) fieldValue);
} else {
throw new IllegalStateException(
"unsupported string field type: " + fieldValue.getClass().getName());
}
nulls[column] = false;
}
}
private void parseDecimalColumn(int column) {
// don't include column number in message because it causes boxing which is expensive here
checkArgument(!isPartitionColumn[column], "Column is a partition key");
loaded[column] = true;
Object fieldData = rowInspector.getStructFieldData(rowData, structFields[column]);
if (fieldData == null) {
nulls[column] = true;
} else {
Object fieldValue =
((PrimitiveObjectInspector) fieldInspectors[column]).getPrimitiveJavaObject(fieldData);
checkState(fieldValue != null, "fieldValue should not be null");
HiveDecimal decimal = (HiveDecimal) fieldValue;
DecimalType columnType = (DecimalType) types[column];
BigInteger unscaledDecimal =
rescale(decimal.unscaledValue(), decimal.scale(), columnType.getScale());
if (columnType.isShort()) {
longs[column] = unscaledDecimal.longValue();
} else {
slices[column] = Decimals.encodeUnscaledValue(unscaledDecimal);
}
nulls[column] = false;
}
}
private void parseStringColumn(int column) {
// don't include column number in message because it causes boxing which is expensive here
checkArgument(!isPartitionColumn[column], "Column is a partition key");
loaded[column] = true;
Object fieldData = rowInspector.getStructFieldData(rowData, structFields[column]);
if (fieldData == null) {
nulls[column] = true;
} else {
Object fieldValue =
((PrimitiveObjectInspector) fieldInspectors[column]).getPrimitiveJavaObject(fieldData);
checkState(fieldValue != null, "fieldValue should not be null");
Slice value;
if (fieldValue instanceof String) {
value = Slices.utf8Slice((String) fieldValue);
} else if (fieldValue instanceof byte[]) {
value = Slices.wrappedBuffer((byte[]) fieldValue);
} else if (fieldValue instanceof HiveVarchar) {
value = Slices.utf8Slice(((HiveVarchar) fieldValue).getValue());
} else {
throw new IllegalStateException(
"unsupported string field type: " + fieldValue.getClass().getName());
}
Type type = types[column];
if (isVarcharType(type)) {
value = truncateToLength(value, type);
}
slices[column] = value;
nulls[column] = false;
}
}
private void initOperatorContext(List<Operator<? extends OperatorDesc>> children)
throws HiveException {
for (Map<Operator<?>, MapOpCtx> contexts : opCtxMap.values()) {
for (MapOpCtx context : contexts.values()) {
if (!children.contains(context.op)) {
continue;
}
StructObjectInspector prev = childrenOpToOI.put(context.op, context.rowObjectInspector);
if (prev != null && !prev.equals(context.rowObjectInspector)) {
throw new HiveException("Conflict on row inspector for " + context.alias);
}
if (isLogDebugEnabled) {
LOG.debug("dump " + context.op + " " + context.rowObjectInspector.getTypeName());
}
}
}
}
/**
* Builds the assigners from an object inspector and from a list of columns.
*
* @param outputBatch The batch to which the assigners are bound
* @param outputOI The row object inspector
* @param columnMap Vector column map
* @param outputColumnNames Column names, used both to find the vector columns and the
* @return
* @throws HiveException
*/
public static VectorColumnAssign[] buildAssigners(
VectorizedRowBatch outputBatch,
ObjectInspector outputOI,
Map<String, Integer> columnMap,
List<String> outputColumnNames)
throws HiveException {
StructObjectInspector soi = (StructObjectInspector) outputOI;
VectorColumnAssign[] vcas = new VectorColumnAssign[outputColumnNames.size()];
for (int i = 0; i < outputColumnNames.size(); ++i) {
String columnName = outputColumnNames.get(i);
Integer columnIndex = columnMap.get(columnName);
StructField columnRef = soi.getStructFieldRef(columnName);
ObjectInspector valueOI = columnRef.getFieldObjectInspector();
vcas[i] = buildObjectAssign(outputBatch, columnIndex, valueOI);
}
return vcas;
}
@Override
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
outputByteBuffer.reset();
StructObjectInspector soi = (StructObjectInspector) objInspector;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int i = 0; i < columnNames.size(); i++) {
serialize(
outputByteBuffer,
soi.getStructFieldData(obj, fields.get(i)),
fields.get(i).getFieldObjectInspector(),
columnSortOrderIsDesc[i]);
}
serializeBytesWritable.set(outputByteBuffer.getData(), 0, outputByteBuffer.getLength());
return serializeBytesWritable;
}
/** return false though element is null if nullsafe flag is true for that */
public static boolean hasAnyNullObject(List o, StructObjectInspector loi, boolean[] nullSafes) {
List<? extends StructField> fields = loi.getAllStructFieldRefs();
for (int i = 0; i < o.size(); i++) {
if ((nullSafes == null || !nullSafes[i])
&& hasAnyNullObject(o.get(i), fields.get(i).getFieldObjectInspector())) {
return true;
}
}
return false;
}
private void partialReadTest(FileSystem fs, int count, Path file)
throws IOException, SerDeException {
LOG.debug("reading " + count + " records");
long start = System.currentTimeMillis();
java.util.ArrayList<Integer> readCols = new java.util.ArrayList<Integer>();
readCols.add(Integer.valueOf(2));
readCols.add(Integer.valueOf(3));
ColumnProjectionUtils.setReadColumnIDs(conf, readCols);
RCFile.Reader reader = new RCFile.Reader(fs, file, conf);
LongWritable rowID = new LongWritable();
BytesRefArrayWritable cols = new BytesRefArrayWritable();
while (reader.next(rowID)) {
reader.getCurrentRow(cols);
cols.resetValid(8);
Object row = serDe.deserialize(cols);
StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
assertEquals("Field size should be 8", 8, fieldRefs.size());
for (int i : readCols) {
Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
Object standardWritableData =
ObjectInspectorUtils.copyToStandardObject(
fieldData,
fieldRefs.get(i).getFieldObjectInspector(),
ObjectInspectorCopyOption.WRITABLE);
assertEquals("Field " + i, standardWritableData, expectedPartitalFieldsData[i]);
}
assertEquals(
"Class of the serialized object should be BytesRefArrayWritable",
BytesRefArrayWritable.class,
serDe.getSerializedClass());
BytesRefArrayWritable serializedBytes = (BytesRefArrayWritable) serDe.serialize(row, oi);
assertEquals("Serialized data", patialS, serializedBytes);
}
reader.close();
long cost = System.currentTimeMillis() - start;
LOG.debug("reading fully costs:" + cost + " milliseconds");
}
@Override
public synchronized void process(Object row, int tag) throws HiveException {
StructObjectInspector soi = parentObjInspectors[tag];
List<? extends StructField> fields = parentFields[tag];
if (needsTransform[tag]) {
for (int c = 0; c < fields.size(); c++) {
outputRow.set(
c,
columnTypeResolvers[c].convertIfNecessary(
soi.getStructFieldData(row, fields.get(c)),
fields.get(c).getFieldObjectInspector()));
}
forward(outputRow, outputObjInspector);
} else {
forward(row, inputObjInspectors[tag]);
}
}
public KeyValueWriter(SerDe keySerDe, SerDe valSerDe, boolean hasFilterTag)
throws SerDeException {
this.keySerDe = keySerDe;
this.valSerDe = valSerDe;
keySoi = (StructObjectInspector) keySerDe.getObjectInspector();
valSoi = (StructObjectInspector) valSerDe.getObjectInspector();
List<? extends StructField> keyFields = keySoi.getAllStructFieldRefs(),
valFields = valSoi.getAllStructFieldRefs();
keyOis = new ArrayList<ObjectInspector>(keyFields.size());
valOis = new ArrayList<ObjectInspector>(valFields.size());
for (int i = 0; i < keyFields.size(); ++i) {
keyOis.add(keyFields.get(i).getFieldObjectInspector());
}
for (int i = 0; i < valFields.size(); ++i) {
valOis.add(valFields.get(i).getFieldObjectInspector());
}
keyObjs = new Object[keyOis.size()];
valObjs = new Object[valOis.size()];
this.hasFilterTag = hasFilterTag;
}
public int openFile() throws Exception {
m_reader = OrcFile.createReader(m_file_path, OrcFile.readerOptions(m_conf));
m_types = m_reader.getTypes();
m_oi = (StructObjectInspector) m_reader.getObjectInspector();
m_fields = m_oi.getAllStructFieldRefs();
m_rr = m_reader.rows();
return 0;
}
/** Get the list of field names as csv from a StructObjectInspector. */
public static String getFieldNames(StructObjectInspector soi) {
List<? extends StructField> fields = soi.getAllStructFieldRefs();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < fields.size(); i++) {
if (i > 0) {
sb.append(",");
}
sb.append(fields.get(i).getFieldName());
}
return sb.toString();
}
