@Override
public void setKeyValue(Writable key, Writable val) throws SerDeException {
Object keyObj = keySerDe.deserialize(key), valObj = valSerDe.deserialize(val);
List<? extends StructField> keyFields = keySoi.getAllStructFieldRefs(),
valFields = valSoi.getAllStructFieldRefs();
for (int i = 0; i < keyFields.size(); ++i) {
keyObjs[i] = keySoi.getStructFieldData(keyObj, keyFields.get(i));
}
for (int i = 0; i < valFields.size(); ++i) {
valObjs[i] = valSoi.getStructFieldData(valObj, valFields.get(i));
}
}
private void stringifyObject(StringBuilder buffer, Object obj, ObjectInspector inspector)
throws IOException {
if (inspector instanceof StructObjectInspector) {
buffer.append("{ ");
StructObjectInspector soi = (StructObjectInspector) inspector;
boolean isFirst = true;
for (StructField field : soi.getAllStructFieldRefs()) {
if (isFirst) {
isFirst = false;
} else {
buffer.append(", ");
}
buffer.append(field.getFieldName());
buffer.append(": ");
stringifyObject(
buffer, soi.getStructFieldData(obj, field), field.getFieldObjectInspector());
}
buffer.append(" }");
} else if (inspector instanceof PrimitiveObjectInspector) {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
buffer.append(poi.getPrimitiveJavaObject(obj).toString());
} else {
buffer.append("*unknown*");
}
}
protected static StructObjectInspector createSelectListOI(
MatchPath evaluator, PTFInputDef inpDef) {
StructObjectInspector inOI = inpDef.getOutputShape().getOI();
ArrayList<String> inputColumnNames = new ArrayList<String>();
ArrayList<String> selectListNames = new ArrayList<String>();
ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
for (StructField f : inOI.getAllStructFieldRefs()) {
String inputColName = evaluator.inputColumnNamesMap.get(f.getFieldName());
if (inputColName != null) {
inputColumnNames.add(inputColName);
selectListNames.add(f.getFieldName());
fieldOIs.add(f.getFieldObjectInspector());
}
}
StandardListObjectInspector pathAttrOI =
ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, fieldOIs));
ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>();
selectFieldOIs.addAll(fieldOIs);
selectFieldOIs.add(pathAttrOI);
selectListNames.add(MatchPath.PATHATTR_NAME);
return ObjectInspectorFactory.getStandardStructObjectInspector(selectListNames, selectFieldOIs);
}
@Override
public Writable serialize(Object obj, ObjectInspector oi) throws SerDeException {
if (oi.getCategory() != Category.STRUCT) {
throw new VoltSerdeException(
getClass().toString()
+ " can only serialize struct types, but we got: "
+ oi.getTypeName());
}
VoltRecord vr = new VoltRecord(m_voltConf.getTableName());
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> structFields = soi.getAllStructFieldRefs();
List<Object> fieldValues = soi.getStructFieldsDataAsList(obj);
final int size = m_oig.getColumnTypes().size();
for (int i = 0; i < size; ++i) {
ObjectInspector fieldOI = structFields.get(i).getFieldObjectInspector();
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) fieldOI;
Object fieldValue = poi.getPrimitiveJavaObject(fieldValues.get(i));
if (poi.getTypeInfo().equals(TypeInfoFactory.timestampTypeInfo)) {
fieldValue = fieldValue != null ? new Date(((Timestamp) fieldValue).getTime()) : null;
}
vr.add(fieldValue);
}
return vr;
}
private RowSet decodeFromString(List<Object> rows, RowSet rowSet)
throws SQLException, SerDeException {
getSerDe();
StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
Object[] deserializedFields = new Object[fieldRefs.size()];
Object rowObj;
ObjectInspector fieldOI;
int protocol = getProtocolVersion().getValue();
for (Object rowString : rows) {
try {
rowObj = serde.deserialize(new BytesWritable(((String) rowString).getBytes("UTF-8")));
} catch (UnsupportedEncodingException e) {
throw new SerDeException(e);
}
for (int i = 0; i < fieldRefs.size(); i++) {
StructField fieldRef = fieldRefs.get(i);
fieldOI = fieldRef.getFieldObjectInspector();
Object fieldData = soi.getStructFieldData(rowObj, fieldRef);
deserializedFields[i] = SerDeUtils.toThriftPayload(fieldData, fieldOI, protocol);
}
rowSet.addRow(deserializedFields);
}
return rowSet;
}
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
if (objInspector.getCategory() != Category.STRUCT) {
throw new SerDeException(
getClass().toString()
+ " can only serialize struct types, but we got: "
+ objInspector.getTypeName());
}
StructObjectInspector soi = (StructObjectInspector) objInspector;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < fields.size(); i++) {
if (i > 0) sb.append(separator);
Object column = soi.getStructFieldData(obj, fields.get(i));
if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) {
// For primitive object, serialize to plain string
sb.append(column == null ? nullString : column.toString());
} else {
// For complex object, serialize to JSON format
sb.append(SerDeUtils.getJSONString(column, fields.get(i).getFieldObjectInspector()));
}
}
serializeCache.set(sb.toString());
return serializeCache;
}
private ObjectInspector solveOi(ObjectInspector arg) {
switch (arg.getCategory()) {
case PRIMITIVE:
// VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
// UNKNOWN
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
poi.getPrimitiveCategory());
case LIST:
return ObjectInspectorFactory.getStandardListObjectInspector(
solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
case MAP:
return ObjectInspectorFactory.getStandardMapObjectInspector(
solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) arg;
int size = soi.getAllStructFieldRefs().size();
ArrayList<String> fnl = new ArrayList<String>(size);
ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);
for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
fnl.add(sf.getFieldName());
foil.add(solveOi(sf.getFieldObjectInspector()));
}
return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
default:
return arg;
}
}
/**
* Whether comparison is supported for this type. Currently all types that references any map are
* not comparable.
*/
public static boolean compareSupported(ObjectInspector oi) {
switch (oi.getCategory()) {
case PRIMITIVE:
return true;
case LIST:
ListObjectInspector loi = (ListObjectInspector) oi;
return compareSupported(loi.getListElementObjectInspector());
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int f = 0; f < fields.size(); f++) {
if (!compareSupported(fields.get(f).getFieldObjectInspector())) {
return false;
}
}
return true;
case MAP:
return false;
case UNION:
UnionObjectInspector uoi = (UnionObjectInspector) oi;
for (ObjectInspector eoi : uoi.getObjectInspectors()) {
if (!compareSupported(eoi)) {
return false;
}
}
return true;
default:
return false;
}
}
/**
* Serializing means getting every field, and setting the appropriate JSONObject field. Actual
* serialization is done at the end when the whole JSON object is built
*
* @param serializer
* @param obj
* @param structObjectInspector
*/
private JSONObject serializeStruct(
Object obj, StructObjectInspector soi, List<String> columnNames) {
// do nothing for null struct
if (null == obj) {
return null;
}
JSONObject result = new JSONObject();
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int i = 0; i < fields.size(); i++) {
StructField sf = fields.get(i);
Object data = soi.getStructFieldData(obj, sf);
if (null != data) {
try {
// we want to serialize columns with their proper HIVE name,
// not the _col2 kind of name usually generated upstream
result.put(
getSerializedFieldName(columnNames, i, sf),
serializeField(data, sf.getFieldObjectInspector()));
} catch (JSONException ex) {
LOG.warn("Problem serializing", ex);
throw new RuntimeException(ex);
}
}
}
return result;
}
private static void addInputColumnsToList(
ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) {
StructObjectInspector OI = shape.getOI();
for (StructField f : OI.getAllStructFieldRefs()) {
fieldNames.add(f.getFieldName());
fieldOIs.add(f.getFieldObjectInspector());
}
}
/**
* @param oi - Input object inspector
* @param oiSettableProperties - Lookup map to cache the result.(If no caching, pass null)
* @return - true if : (1) oi is an instance of settable<DataType>OI. (2) All the embedded object
* inspectors are instances of settable<DataType>OI. If (1) or (2) is false, return false.
*/
public static boolean hasAllFieldsSettable(
ObjectInspector oi, Map<ObjectInspector, Boolean> oiSettableProperties) {
// If the result is already present in the cache, return it.
if (!(oiSettableProperties == null) && oiSettableProperties.containsKey(oi)) {
return oiSettableProperties.get(oi).booleanValue();
}
// If the top-level object inspector is non-settable return false
if (!(isInstanceOfSettableOI(oi))) {
return setOISettablePropertiesMap(oi, oiSettableProperties, false);
}
Boolean returnValue = true;
switch (oi.getCategory()) {
case PRIMITIVE:
break;
case STRUCT:
StructObjectInspector structOutputOI = (StructObjectInspector) oi;
List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
for (StructField listField : listFields) {
if (!hasAllFieldsSettable(listField.getFieldObjectInspector(), oiSettableProperties)) {
returnValue = false;
break;
}
}
break;
case LIST:
ListObjectInspector listOutputOI = (ListObjectInspector) oi;
returnValue =
hasAllFieldsSettable(
listOutputOI.getListElementObjectInspector(), oiSettableProperties);
break;
case MAP:
MapObjectInspector mapOutputOI = (MapObjectInspector) oi;
returnValue =
hasAllFieldsSettable(mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties)
&& hasAllFieldsSettable(
mapOutputOI.getMapValueObjectInspector(), oiSettableProperties);
break;
case UNION:
UnionObjectInspector unionOutputOI = (UnionObjectInspector) oi;
List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
for (ObjectInspector listField : unionListFields) {
if (!hasAllFieldsSettable(listField, oiSettableProperties)) {
returnValue = false;
break;
}
}
break;
default:
throw new RuntimeException(
"Hive internal error inside hasAllFieldsSettable : "
+ oi.getTypeName()
+ " not supported yet.");
}
return setOISettablePropertiesMap(oi, oiSettableProperties, returnValue);
}
/** return false though element is null if nullsafe flag is true for that */
public static boolean hasAnyNullObject(List o, StructObjectInspector loi, boolean[] nullSafes) {
List<? extends StructField> fields = loi.getAllStructFieldRefs();
for (int i = 0; i < o.size(); i++) {
if ((nullSafes == null || !nullSafes[i])
&& hasAnyNullObject(o.get(i), fields.get(i).getFieldObjectInspector())) {
return true;
}
}
return false;
}
@Override
public Writable serialize(final Object obj, final ObjectInspector inspector)
throws SerDeException {
final StructObjectInspector structInspector = (StructObjectInspector) inspector;
final List<? extends StructField> fields = structInspector.getAllStructFieldRefs();
if (fields.size() != columnNames.size()) {
throw new SerDeException(
String.format("Required %d columns, received %d.", columnNames.size(), fields.size()));
}
cachedWritable.clear();
for (int c = 0; c < fieldCount; c++) {
StructField structField = fields.get(c);
LOG.debug("fieldId=" + c + ",structField=" + structField.toString());
if (structField != null) {
final Object field = structInspector.getStructFieldData(obj, fields.get(c));
final AbstractPrimitiveObjectInspector fieldOI =
(AbstractPrimitiveObjectInspector) fields.get(c).getFieldObjectInspector();
Writable value = (Writable) fieldOI.getPrimitiveWritableObject(field);
if (value == null) {
continue;
}
LOG.debug("fieldCount=" + fieldCount + ",value=" + value.toString());
if (value instanceof IntWritable) {
cachedWritable.put(new Text(columnNames.get(c)), value);
} else if (value instanceof Text) {
cachedWritable.put(new Text(columnNames.get(c)), ((Text) value));
} else if (value instanceof LongWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((LongWritable) value));
} else if (value instanceof DoubleWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((DoubleWritable) value));
} else if (value instanceof FloatWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((FloatWritable) value));
} else if (value instanceof BooleanWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((BooleanWritable) value));
} else if (value instanceof ByteWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((ByteWritable) value));
} else if (value instanceof BytesWritable) {
cachedWritable.put(new Text(columnNames.get(c)), ((BytesWritable) value));
} else {
LOG.warn("fieldCount=" + fieldCount + ",type=" + value.getClass().getName());
}
}
}
return cachedWritable;
}
public int openFile() throws Exception {
m_reader = OrcFile.createReader(m_file_path, OrcFile.readerOptions(m_conf));
m_types = m_reader.getTypes();
m_oi = (StructObjectInspector) m_reader.getObjectInspector();
m_fields = m_oi.getAllStructFieldRefs();
m_rr = m_reader.rows();
return 0;
}
/** Get the list of field names as csv from a StructObjectInspector. */
public static String getFieldNames(StructObjectInspector soi) {
List<? extends StructField> fields = soi.getAllStructFieldRefs();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < fields.size(); i++) {
if (i > 0) {
sb.append(",");
}
sb.append(fields.get(i).getFieldName());
}
return sb.toString();
}
public KeyValueWriter(SerDe keySerDe, SerDe valSerDe, boolean hasFilterTag)
throws SerDeException {
this.keySerDe = keySerDe;
this.valSerDe = valSerDe;
keySoi = (StructObjectInspector) keySerDe.getObjectInspector();
valSoi = (StructObjectInspector) valSerDe.getObjectInspector();
List<? extends StructField> keyFields = keySoi.getAllStructFieldRefs(),
valFields = valSoi.getAllStructFieldRefs();
keyOis = new ArrayList<ObjectInspector>(keyFields.size());
valOis = new ArrayList<ObjectInspector>(valFields.size());
for (int i = 0; i < keyFields.size(); ++i) {
keyOis.add(keyFields.get(i).getFieldObjectInspector());
}
for (int i = 0; i < valFields.size(); ++i) {
valOis.add(valFields.get(i).getFieldObjectInspector());
}
keyObjs = new Object[keyOis.size()];
valObjs = new Object[valOis.size()];
this.hasFilterTag = hasFilterTag;
}
private ArrayWritable createStruct(final Object obj, final StructObjectInspector inspector)
throws SerDeException {
final List<? extends StructField> fields = inspector.getAllStructFieldRefs();
final Writable[] arr = new Writable[fields.size()];
for (int i = 0; i < fields.size(); i++) {
final StructField field = fields.get(i);
final Object subObj = inspector.getStructFieldData(obj, field);
final ObjectInspector subInspector = field.getFieldObjectInspector();
arr[i] = createObject(subObj, subInspector);
}
return new ArrayWritable(Writable.class, arr);
}
/* Initialize the UDAF on the reduce side (or the map side in some cases). */
private void initReduceSide(StructObjectInspector inputStructOI) throws HiveException {
List<? extends StructField> fields = inputStructOI.getAllStructFieldRefs();
int length = fields.size();
inputOIs = new ObjectInspector[length];
outputOIs = new ObjectInspector[length];
for (int i = 0; i < length; i++) {
StructField field = fields.get(i);
inputOIs[i] = field.getFieldObjectInspector();
outputOIs[i] = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i]);
}
structOI = ObjectInspectorUtils.getStandardObjectInspector(inputStructOI);
}
/**
* Copy fields in the input row to the output array of standard objects.
*
* @param result output list of standard objects.
* @param row input row.
* @param soi Object inspector for the to-be-copied columns.
* @param objectInspectorOption
*/
public static void copyToStandardObject(
List<Object> result,
Object row,
StructObjectInspector soi,
ObjectInspectorCopyOption objectInspectorOption) {
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (StructField f : fields) {
result.add(
copyToStandardObject(
soi.getStructFieldData(row, f), f.getFieldObjectInspector(), objectInspectorOption));
}
}
/*
* from the prunedCols list filter out columns that refer to WindowFns or WindowExprs
* the returned list is set as the prunedList needed by the PTFOp.
*/
private ArrayList<String> prunedInputList(
List<String> prunedCols, WindowTableFunctionDef tDef) {
ArrayList<String> prunedInputCols = new ArrayList<String>();
StructObjectInspector OI = tDef.getInput().getOutputShape().getOI();
for (StructField f : OI.getAllStructFieldRefs()) {
String fName = f.getFieldName();
if (prunedCols.contains(fName)) {
prunedInputCols.add(fName);
}
}
return prunedInputCols;
}
public static String getStandardStructTypeName(StructObjectInspector soi) {
StringBuilder sb = new StringBuilder();
sb.append("struct<");
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int i = 0; i < fields.size(); i++) {
if (i > 0) {
sb.append(",");
}
sb.append(fields.get(i).getFieldName());
sb.append(":");
sb.append(fields.get(i).getFieldObjectInspector().getTypeName());
}
sb.append(">");
return sb.toString();
}
@Override
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
outputByteBuffer.reset();
StructObjectInspector soi = (StructObjectInspector) objInspector;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int i = 0; i < columnNames.size(); i++) {
serialize(
outputByteBuffer,
soi.getStructFieldData(obj, fields.get(i)),
fields.get(i).getFieldObjectInspector(),
columnSortOrderIsDesc[i]);
}
serializeBytesWritable.set(outputByteBuffer.getData(), 0, outputByteBuffer.getLength());
return serializeBytesWritable;
}
private void partialReadTest(FileSystem fs, int count, Path file)
throws IOException, SerDeException {
LOG.debug("reading " + count + " records");
long start = System.currentTimeMillis();
java.util.ArrayList<Integer> readCols = new java.util.ArrayList<Integer>();
readCols.add(Integer.valueOf(2));
readCols.add(Integer.valueOf(3));
ColumnProjectionUtils.setReadColumnIDs(conf, readCols);
RCFile.Reader reader = new RCFile.Reader(fs, file, conf);
LongWritable rowID = new LongWritable();
BytesRefArrayWritable cols = new BytesRefArrayWritable();
while (reader.next(rowID)) {
reader.getCurrentRow(cols);
cols.resetValid(8);
Object row = serDe.deserialize(cols);
StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
assertEquals("Field size should be 8", 8, fieldRefs.size());
for (int i : readCols) {
Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
Object standardWritableData =
ObjectInspectorUtils.copyToStandardObject(
fieldData,
fieldRefs.get(i).getFieldObjectInspector(),
ObjectInspectorCopyOption.WRITABLE);
assertEquals("Field " + i, standardWritableData, expectedPartitalFieldsData[i]);
}
assertEquals(
"Class of the serialized object should be BytesRefArrayWritable",
BytesRefArrayWritable.class,
serDe.getSerializedClass());
BytesRefArrayWritable serializedBytes = (BytesRefArrayWritable) serDe.serialize(row, oi);
assertEquals("Serialized data", patialS, serializedBytes);
}
reader.close();
long cost = System.currentTimeMillis() - start;
LOG.debug("reading fully costs:" + cost + " milliseconds");
}
public void fullyReadTest(FileSystem fs, int count, Path file)
throws IOException, SerDeException {
LOG.debug("reading " + count + " records");
long start = System.currentTimeMillis();
ColumnProjectionUtils.setFullyReadColumns(conf);
RCFile.Reader reader = new RCFile.Reader(fs, file, conf);
LongWritable rowID = new LongWritable();
int actualRead = 0;
BytesRefArrayWritable cols = new BytesRefArrayWritable();
while (reader.next(rowID)) {
reader.getCurrentRow(cols);
cols.resetValid(8);
Object row = serDe.deserialize(cols);
StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
assertEquals("Field size should be 8", 8, fieldRefs.size());
for (int i = 0; i < fieldRefs.size(); i++) {
Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
Object standardWritableData =
ObjectInspectorUtils.copyToStandardObject(
fieldData,
fieldRefs.get(i).getFieldObjectInspector(),
ObjectInspectorCopyOption.WRITABLE);
assertEquals("Field " + i, standardWritableData, expectedFieldsData[i]);
}
// Serialize
assertEquals(
"Class of the serialized object should be BytesRefArrayWritable",
BytesRefArrayWritable.class,
serDe.getSerializedClass());
BytesRefArrayWritable serializedText = (BytesRefArrayWritable) serDe.serialize(row, oi);
assertEquals("Serialized data", s, serializedText);
actualRead++;
}
reader.close();
assertEquals("Expect " + count + " rows, actual read " + actualRead, actualRead, count);
long cost = System.currentTimeMillis() - start;
LOG.debug("reading fully costs:" + cost + " milliseconds");
}
@Override
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
final StructObjectInspector outputRowOI = (StructObjectInspector) objInspector;
final List<? extends StructField> outputFieldRefs = outputRowOI.getAllStructFieldRefs();
if (outputFieldRefs.size() != numCols) {
throw new SerDeException(
"Cannot serialize the object because there are "
+ outputFieldRefs.size()
+ " fields but the table has "
+ numCols
+ " columns.");
}
// Get all data out.
for (int c = 0; c < numCols; c++) {
final Object field = outputRowOI.getStructFieldData(obj, outputFieldRefs.get(c));
final ObjectInspector fieldOI = outputFieldRefs.get(c).getFieldObjectInspector();
// The data must be of type String
final StringObjectInspector fieldStringOI = (StringObjectInspector) fieldOI;
// Convert the field to Java class String, because objects of String
// type
// can be stored in String, Text, or some other classes.
outputFields[c] = fieldStringOI.getPrimitiveJavaObject(field);
}
final StringWriter writer = new StringWriter();
final CSVWriter csv = newWriter(writer, separatorChar, quoteChar, escapeChar);
try {
csv.writeNext(outputFields);
csv.close();
return new Text(writer.toString());
} catch (final IOException ioe) {
throw new SerDeException(ioe);
}
}
/**
* If we received data with tags from ReduceSinkOperators, no keys will match. This should not
* happen, but is important enough that we want to find out and work around it if some optimized
* change causes RSO to pass on tags.
*/
private void sanityCheckKeyForTag() throws SerDeException {
if (hasTag != null) return;
BinaryComparable b = (BinaryComparable) key;
Object o = keySerDe.deserialize(key);
StructObjectInspector soi = (StructObjectInspector) keySerDe.getObjectInspector();
List<? extends StructField> fields = soi.getAllStructFieldRefs();
Object[] data = new Object[fields.size()];
List<ObjectInspector> fois = new ArrayList<ObjectInspector>(fields.size());
for (int i = 0; i < fields.size(); i++) {
data[i] = soi.getStructFieldData(o, fields.get(i));
fois.add(fields.get(i).getFieldObjectInspector());
}
Output output = new Output();
BinarySortableSerDe.serializeStruct(output, data, fois, new boolean[fields.size()]);
hasTag = (output.getLength() != b.getLength());
if (hasTag) {
LOG.error("Tag found in keys and will be removed. This should not happen.");
if (output.getLength() != (b.getLength() - 1)) {
throw new SerDeException(
"Unexpected tag: " + b.getLength() + " reserialized to " + output.getLength());
}
}
}
/**
* Copy specified fields in the input row to the output array of standard objects.
*
* @param result output list of standard objects.
* @param row input row.
* @param startCol starting column number from the input row.
* @param numCols number of columns to copy.
* @param soi Object inspector for the to-be-copied columns.
*/
public static void partialCopyToStandardObject(
List<Object> result,
Object row,
int startCol,
int numCols,
StructObjectInspector soi,
ObjectInspectorCopyOption objectInspectorOption) {
List<? extends StructField> fields = soi.getAllStructFieldRefs();
int i = 0, j = 0;
for (StructField f : fields) {
if (i++ >= startCol) {
result.add(
copyToStandardObject(
soi.getStructFieldData(row, f),
f.getFieldObjectInspector(),
objectInspectorOption));
if (++j == numCols) {
break;
}
}
}
}
@Override
public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
assertState(OperationState.FINISHED);
ArrayList<String> rows = new ArrayList<String>();
driver.setMaxRows((int) maxRows);
try {
driver.getResults(rows);
getSerDe();
StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
RowSet rowSet = new RowSet();
Object[] deserializedFields = new Object[fieldRefs.size()];
Object rowObj;
ObjectInspector fieldOI;
for (String rowString : rows) {
rowObj = serde.deserialize(new BytesWritable(rowString.getBytes()));
for (int i = 0; i < fieldRefs.size(); i++) {
StructField fieldRef = fieldRefs.get(i);
fieldOI = fieldRef.getFieldObjectInspector();
deserializedFields[i] =
convertLazyToJava(soi.getStructFieldData(rowObj, fieldRef), fieldOI);
}
rowSet.addRow(resultSchema, deserializedFields);
}
return rowSet;
} catch (IOException e) {
throw new HiveSQLException(e);
} catch (CommandNeedRetryException e) {
throw new HiveSQLException(e);
} catch (Exception e) {
throw new HiveSQLException(e);
}
}
public static void copyStructToArray(
Object o,
ObjectInspector oi,
ObjectInspectorCopyOption objectInspectorOption,
Object[] dest,
int offset)
throws SerDeException {
if (o == null) {
return;
}
if (oi.getCategory() != Category.STRUCT) {
throw new SerDeException("Unexpected category " + oi.getCategory());
}
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
for (int i = 0; i < fields.size(); ++i) {
StructField f = fields.get(i);
dest[offset + i] =
copyToStandardObject(
soi.getStructFieldData(o, f), f.getFieldObjectInspector(), objectInspectorOption);
}
}
private static Block serializeStruct(
Type type, BlockBuilder builder, Object object, StructObjectInspector inspector) {
if (object == null) {
requireNonNull(builder, "parent builder is null").appendNull();
return null;
}
List<Type> typeParameters = type.getTypeParameters();
List<? extends StructField> allStructFieldRefs = inspector.getAllStructFieldRefs();
checkArgument(typeParameters.size() == allStructFieldRefs.size());
BlockBuilder currentBuilder;
if (builder != null) {
currentBuilder = builder.beginBlockEntry();
} else {
currentBuilder =
new InterleavedBlockBuilder(
typeParameters, new BlockBuilderStatus(), typeParameters.size());
}
for (int i = 0; i < typeParameters.size(); i++) {
StructField field = allStructFieldRefs.get(i);
serializeObject(
typeParameters.get(i),
currentBuilder,
inspector.getStructFieldData(object, field),
field.getFieldObjectInspector());
}
if (builder != null) {
builder.closeEntry();
return null;
} else {
Block resultBlock = currentBuilder.build();
return resultBlock;
}
}