@Override public void setKeyValue(Writable key, Writable val) throws SerDeException { Object keyObj = keySerDe.deserialize(key), valObj = valSerDe.deserialize(val); List<? extends StructField> keyFields = keySoi.getAllStructFieldRefs(), valFields = valSoi.getAllStructFieldRefs(); for (int i = 0; i < keyFields.size(); ++i) { keyObjs[i] = keySoi.getStructFieldData(keyObj, keyFields.get(i)); } for (int i = 0; i < valFields.size(); ++i) { valObjs[i] = valSoi.getStructFieldData(valObj, valFields.get(i)); } }
private void stringifyObject(StringBuilder buffer, Object obj, ObjectInspector inspector) throws IOException { if (inspector instanceof StructObjectInspector) { buffer.append("{ "); StructObjectInspector soi = (StructObjectInspector) inspector; boolean isFirst = true; for (StructField field : soi.getAllStructFieldRefs()) { if (isFirst) { isFirst = false; } else { buffer.append(", "); } buffer.append(field.getFieldName()); buffer.append(": "); stringifyObject( buffer, soi.getStructFieldData(obj, field), field.getFieldObjectInspector()); } buffer.append(" }"); } else if (inspector instanceof PrimitiveObjectInspector) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector; buffer.append(poi.getPrimitiveJavaObject(obj).toString()); } else { buffer.append("*unknown*"); } }
protected static StructObjectInspector createSelectListOI( MatchPath evaluator, PTFInputDef inpDef) { StructObjectInspector inOI = inpDef.getOutputShape().getOI(); ArrayList<String> inputColumnNames = new ArrayList<String>(); ArrayList<String> selectListNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); for (StructField f : inOI.getAllStructFieldRefs()) { String inputColName = evaluator.inputColumnNamesMap.get(f.getFieldName()); if (inputColName != null) { inputColumnNames.add(inputColName); selectListNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } } StandardListObjectInspector pathAttrOI = ObjectInspectorFactory.getStandardListObjectInspector( ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, fieldOIs)); ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>(); selectFieldOIs.addAll(fieldOIs); selectFieldOIs.add(pathAttrOI); selectListNames.add(MatchPath.PATHATTR_NAME); return ObjectInspectorFactory.getStandardStructObjectInspector(selectListNames, selectFieldOIs); }
@Override public Writable serialize(Object obj, ObjectInspector oi) throws SerDeException { if (oi.getCategory() != Category.STRUCT) { throw new VoltSerdeException( getClass().toString() + " can only serialize struct types, but we got: " + oi.getTypeName()); } VoltRecord vr = new VoltRecord(m_voltConf.getTableName()); StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> structFields = soi.getAllStructFieldRefs(); List<Object> fieldValues = soi.getStructFieldsDataAsList(obj); final int size = m_oig.getColumnTypes().size(); for (int i = 0; i < size; ++i) { ObjectInspector fieldOI = structFields.get(i).getFieldObjectInspector(); PrimitiveObjectInspector poi = (PrimitiveObjectInspector) fieldOI; Object fieldValue = poi.getPrimitiveJavaObject(fieldValues.get(i)); if (poi.getTypeInfo().equals(TypeInfoFactory.timestampTypeInfo)) { fieldValue = fieldValue != null ? new Date(((Timestamp) fieldValue).getTime()) : null; } vr.add(fieldValue); } return vr; }
private RowSet decodeFromString(List<Object> rows, RowSet rowSet) throws SQLException, SerDeException { getSerDe(); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs(); Object[] deserializedFields = new Object[fieldRefs.size()]; Object rowObj; ObjectInspector fieldOI; int protocol = getProtocolVersion().getValue(); for (Object rowString : rows) { try { rowObj = serde.deserialize(new BytesWritable(((String) rowString).getBytes("UTF-8"))); } catch (UnsupportedEncodingException e) { throw new SerDeException(e); } for (int i = 0; i < fieldRefs.size(); i++) { StructField fieldRef = fieldRefs.get(i); fieldOI = fieldRef.getFieldObjectInspector(); Object fieldData = soi.getStructFieldData(rowObj, fieldRef); deserializedFields[i] = SerDeUtils.toThriftPayload(fieldData, fieldOI, protocol); } rowSet.addRow(deserializedFields); } return rowSet; }
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { if (objInspector.getCategory() != Category.STRUCT) { throw new SerDeException( getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); } StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < fields.size(); i++) { if (i > 0) sb.append(separator); Object column = soi.getStructFieldData(obj, fields.get(i)); if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) { // For primitive object, serialize to plain string sb.append(column == null ? nullString : column.toString()); } else { // For complex object, serialize to JSON format sb.append(SerDeUtils.getJSONString(column, fields.get(i).getFieldObjectInspector())); } } serializeCache.set(sb.toString()); return serializeCache; }
private ObjectInspector solveOi(ObjectInspector arg) { switch (arg.getCategory()) { case PRIMITIVE: // VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL, // UNKNOWN PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg; return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( poi.getPrimitiveCategory()); case LIST: return ObjectInspectorFactory.getStandardListObjectInspector( solveOi(((ListObjectInspector) arg).getListElementObjectInspector())); case MAP: return ObjectInspectorFactory.getStandardMapObjectInspector( solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()), solveOi(((MapObjectInspector) arg).getMapValueObjectInspector())); case STRUCT: StructObjectInspector soi = (StructObjectInspector) arg; int size = soi.getAllStructFieldRefs().size(); ArrayList<String> fnl = new ArrayList<String>(size); ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size); for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) { fnl.add(sf.getFieldName()); foil.add(solveOi(sf.getFieldObjectInspector())); } return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil); default: return arg; } }
/** * Whether comparison is supported for this type. Currently all types that references any map are * not comparable. */ public static boolean compareSupported(ObjectInspector oi) { switch (oi.getCategory()) { case PRIMITIVE: return true; case LIST: ListObjectInspector loi = (ListObjectInspector) oi; return compareSupported(loi.getListElementObjectInspector()); case STRUCT: StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int f = 0; f < fields.size(); f++) { if (!compareSupported(fields.get(f).getFieldObjectInspector())) { return false; } } return true; case MAP: return false; case UNION: UnionObjectInspector uoi = (UnionObjectInspector) oi; for (ObjectInspector eoi : uoi.getObjectInspectors()) { if (!compareSupported(eoi)) { return false; } } return true; default: return false; } }
/** * Serializing means getting every field, and setting the appropriate JSONObject field. Actual * serialization is done at the end when the whole JSON object is built * * @param serializer * @param obj * @param structObjectInspector */ private JSONObject serializeStruct( Object obj, StructObjectInspector soi, List<String> columnNames) { // do nothing for null struct if (null == obj) { return null; } JSONObject result = new JSONObject(); List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { StructField sf = fields.get(i); Object data = soi.getStructFieldData(obj, sf); if (null != data) { try { // we want to serialize columns with their proper HIVE name, // not the _col2 kind of name usually generated upstream result.put( getSerializedFieldName(columnNames, i, sf), serializeField(data, sf.getFieldObjectInspector())); } catch (JSONException ex) { LOG.warn("Problem serializing", ex); throw new RuntimeException(ex); } } } return result; }
private static void addInputColumnsToList( ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
/** * @param oi - Input object inspector * @param oiSettableProperties - Lookup map to cache the result.(If no caching, pass null) * @return - true if : (1) oi is an instance of settable<DataType>OI. (2) All the embedded object * inspectors are instances of settable<DataType>OI. If (1) or (2) is false, return false. */ public static boolean hasAllFieldsSettable( ObjectInspector oi, Map<ObjectInspector, Boolean> oiSettableProperties) { // If the result is already present in the cache, return it. if (!(oiSettableProperties == null) && oiSettableProperties.containsKey(oi)) { return oiSettableProperties.get(oi).booleanValue(); } // If the top-level object inspector is non-settable return false if (!(isInstanceOfSettableOI(oi))) { return setOISettablePropertiesMap(oi, oiSettableProperties, false); } Boolean returnValue = true; switch (oi.getCategory()) { case PRIMITIVE: break; case STRUCT: StructObjectInspector structOutputOI = (StructObjectInspector) oi; List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs(); for (StructField listField : listFields) { if (!hasAllFieldsSettable(listField.getFieldObjectInspector(), oiSettableProperties)) { returnValue = false; break; } } break; case LIST: ListObjectInspector listOutputOI = (ListObjectInspector) oi; returnValue = hasAllFieldsSettable( listOutputOI.getListElementObjectInspector(), oiSettableProperties); break; case MAP: MapObjectInspector mapOutputOI = (MapObjectInspector) oi; returnValue = hasAllFieldsSettable(mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties) && hasAllFieldsSettable( mapOutputOI.getMapValueObjectInspector(), oiSettableProperties); break; case UNION: UnionObjectInspector unionOutputOI = (UnionObjectInspector) oi; List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors(); for (ObjectInspector listField : unionListFields) { if (!hasAllFieldsSettable(listField, oiSettableProperties)) { returnValue = false; break; } } break; default: throw new RuntimeException( "Hive internal error inside hasAllFieldsSettable : " + oi.getTypeName() + " not supported yet."); } return setOISettablePropertiesMap(oi, oiSettableProperties, returnValue); }
/** return false though element is null if nullsafe flag is true for that */ public static boolean hasAnyNullObject(List o, StructObjectInspector loi, boolean[] nullSafes) { List<? extends StructField> fields = loi.getAllStructFieldRefs(); for (int i = 0; i < o.size(); i++) { if ((nullSafes == null || !nullSafes[i]) && hasAnyNullObject(o.get(i), fields.get(i).getFieldObjectInspector())) { return true; } } return false; }
@Override public Writable serialize(final Object obj, final ObjectInspector inspector) throws SerDeException { final StructObjectInspector structInspector = (StructObjectInspector) inspector; final List<? extends StructField> fields = structInspector.getAllStructFieldRefs(); if (fields.size() != columnNames.size()) { throw new SerDeException( String.format("Required %d columns, received %d.", columnNames.size(), fields.size())); } cachedWritable.clear(); for (int c = 0; c < fieldCount; c++) { StructField structField = fields.get(c); LOG.debug("fieldId=" + c + ",structField=" + structField.toString()); if (structField != null) { final Object field = structInspector.getStructFieldData(obj, fields.get(c)); final AbstractPrimitiveObjectInspector fieldOI = (AbstractPrimitiveObjectInspector) fields.get(c).getFieldObjectInspector(); Writable value = (Writable) fieldOI.getPrimitiveWritableObject(field); if (value == null) { continue; } LOG.debug("fieldCount=" + fieldCount + ",value=" + value.toString()); if (value instanceof IntWritable) { cachedWritable.put(new Text(columnNames.get(c)), value); } else if (value instanceof Text) { cachedWritable.put(new Text(columnNames.get(c)), ((Text) value)); } else if (value instanceof LongWritable) { cachedWritable.put(new Text(columnNames.get(c)), ((LongWritable) value)); } else if (value instanceof DoubleWritable) { cachedWritable.put(new Text(columnNames.get(c)), ((DoubleWritable) value)); } else if (value instanceof FloatWritable) { cachedWritable.put(new Text(columnNames.get(c)), ((FloatWritable) value)); } else if (value instanceof BooleanWritable) { cachedWritable.put(new Text(columnNames.get(c)), ((BooleanWritable) value)); } else if (value instanceof ByteWritable) { cachedWritable.put(new Text(columnNames.get(c)), ((ByteWritable) value)); } else if (value instanceof BytesWritable) { cachedWritable.put(new Text(columnNames.get(c)), ((BytesWritable) value)); } else { LOG.warn("fieldCount=" + fieldCount + ",type=" + value.getClass().getName()); } } } return cachedWritable; }
public int openFile() throws Exception { m_reader = OrcFile.createReader(m_file_path, OrcFile.readerOptions(m_conf)); m_types = m_reader.getTypes(); m_oi = (StructObjectInspector) m_reader.getObjectInspector(); m_fields = m_oi.getAllStructFieldRefs(); m_rr = m_reader.rows(); return 0; }
/** Get the list of field names as csv from a StructObjectInspector. */ public static String getFieldNames(StructObjectInspector soi) { List<? extends StructField> fields = soi.getAllStructFieldRefs(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < fields.size(); i++) { if (i > 0) { sb.append(","); } sb.append(fields.get(i).getFieldName()); } return sb.toString(); }
public KeyValueWriter(SerDe keySerDe, SerDe valSerDe, boolean hasFilterTag) throws SerDeException { this.keySerDe = keySerDe; this.valSerDe = valSerDe; keySoi = (StructObjectInspector) keySerDe.getObjectInspector(); valSoi = (StructObjectInspector) valSerDe.getObjectInspector(); List<? extends StructField> keyFields = keySoi.getAllStructFieldRefs(), valFields = valSoi.getAllStructFieldRefs(); keyOis = new ArrayList<ObjectInspector>(keyFields.size()); valOis = new ArrayList<ObjectInspector>(valFields.size()); for (int i = 0; i < keyFields.size(); ++i) { keyOis.add(keyFields.get(i).getFieldObjectInspector()); } for (int i = 0; i < valFields.size(); ++i) { valOis.add(valFields.get(i).getFieldObjectInspector()); } keyObjs = new Object[keyOis.size()]; valObjs = new Object[valOis.size()]; this.hasFilterTag = hasFilterTag; }
private ArrayWritable createStruct(final Object obj, final StructObjectInspector inspector) throws SerDeException { final List<? extends StructField> fields = inspector.getAllStructFieldRefs(); final Writable[] arr = new Writable[fields.size()]; for (int i = 0; i < fields.size(); i++) { final StructField field = fields.get(i); final Object subObj = inspector.getStructFieldData(obj, field); final ObjectInspector subInspector = field.getFieldObjectInspector(); arr[i] = createObject(subObj, subInspector); } return new ArrayWritable(Writable.class, arr); }
/* Initialize the UDAF on the reduce side (or the map side in some cases). */ private void initReduceSide(StructObjectInspector inputStructOI) throws HiveException { List<? extends StructField> fields = inputStructOI.getAllStructFieldRefs(); int length = fields.size(); inputOIs = new ObjectInspector[length]; outputOIs = new ObjectInspector[length]; for (int i = 0; i < length; i++) { StructField field = fields.get(i); inputOIs[i] = field.getFieldObjectInspector(); outputOIs[i] = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i]); } structOI = ObjectInspectorUtils.getStandardObjectInspector(inputStructOI); }
/** * Copy fields in the input row to the output array of standard objects. * * @param result output list of standard objects. * @param row input row. * @param soi Object inspector for the to-be-copied columns. * @param objectInspectorOption */ public static void copyToStandardObject( List<Object> result, Object row, StructObjectInspector soi, ObjectInspectorCopyOption objectInspectorOption) { List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (StructField f : fields) { result.add( copyToStandardObject( soi.getStructFieldData(row, f), f.getFieldObjectInspector(), objectInspectorOption)); } }
/* * from the prunedCols list filter out columns that refer to WindowFns or WindowExprs * the returned list is set as the prunedList needed by the PTFOp. */ private ArrayList<String> prunedInputList( List<String> prunedCols, WindowTableFunctionDef tDef) { ArrayList<String> prunedInputCols = new ArrayList<String>(); StructObjectInspector OI = tDef.getInput().getOutputShape().getOI(); for (StructField f : OI.getAllStructFieldRefs()) { String fName = f.getFieldName(); if (prunedCols.contains(fName)) { prunedInputCols.add(fName); } } return prunedInputCols; }
public static String getStandardStructTypeName(StructObjectInspector soi) { StringBuilder sb = new StringBuilder(); sb.append("struct<"); List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { if (i > 0) { sb.append(","); } sb.append(fields.get(i).getFieldName()); sb.append(":"); sb.append(fields.get(i).getFieldObjectInspector().getTypeName()); } sb.append(">"); return sb.toString(); }
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { outputByteBuffer.reset(); StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < columnNames.size(); i++) { serialize( outputByteBuffer, soi.getStructFieldData(obj, fields.get(i)), fields.get(i).getFieldObjectInspector(), columnSortOrderIsDesc[i]); } serializeBytesWritable.set(outputByteBuffer.getData(), 0, outputByteBuffer.getLength()); return serializeBytesWritable; }
private void partialReadTest(FileSystem fs, int count, Path file) throws IOException, SerDeException { LOG.debug("reading " + count + " records"); long start = System.currentTimeMillis(); java.util.ArrayList<Integer> readCols = new java.util.ArrayList<Integer>(); readCols.add(Integer.valueOf(2)); readCols.add(Integer.valueOf(3)); ColumnProjectionUtils.setReadColumnIDs(conf, readCols); RCFile.Reader reader = new RCFile.Reader(fs, file, conf); LongWritable rowID = new LongWritable(); BytesRefArrayWritable cols = new BytesRefArrayWritable(); while (reader.next(rowID)) { reader.getCurrentRow(cols); cols.resetValid(8); Object row = serDe.deserialize(cols); StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector(); List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs(); assertEquals("Field size should be 8", 8, fieldRefs.size()); for (int i : readCols) { Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); Object standardWritableData = ObjectInspectorUtils.copyToStandardObject( fieldData, fieldRefs.get(i).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE); assertEquals("Field " + i, standardWritableData, expectedPartitalFieldsData[i]); } assertEquals( "Class of the serialized object should be BytesRefArrayWritable", BytesRefArrayWritable.class, serDe.getSerializedClass()); BytesRefArrayWritable serializedBytes = (BytesRefArrayWritable) serDe.serialize(row, oi); assertEquals("Serialized data", patialS, serializedBytes); } reader.close(); long cost = System.currentTimeMillis() - start; LOG.debug("reading fully costs:" + cost + " milliseconds"); }
public void fullyReadTest(FileSystem fs, int count, Path file) throws IOException, SerDeException { LOG.debug("reading " + count + " records"); long start = System.currentTimeMillis(); ColumnProjectionUtils.setFullyReadColumns(conf); RCFile.Reader reader = new RCFile.Reader(fs, file, conf); LongWritable rowID = new LongWritable(); int actualRead = 0; BytesRefArrayWritable cols = new BytesRefArrayWritable(); while (reader.next(rowID)) { reader.getCurrentRow(cols); cols.resetValid(8); Object row = serDe.deserialize(cols); StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector(); List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs(); assertEquals("Field size should be 8", 8, fieldRefs.size()); for (int i = 0; i < fieldRefs.size(); i++) { Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); Object standardWritableData = ObjectInspectorUtils.copyToStandardObject( fieldData, fieldRefs.get(i).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE); assertEquals("Field " + i, standardWritableData, expectedFieldsData[i]); } // Serialize assertEquals( "Class of the serialized object should be BytesRefArrayWritable", BytesRefArrayWritable.class, serDe.getSerializedClass()); BytesRefArrayWritable serializedText = (BytesRefArrayWritable) serDe.serialize(row, oi); assertEquals("Serialized data", s, serializedText); actualRead++; } reader.close(); assertEquals("Expect " + count + " rows, actual read " + actualRead, actualRead, count); long cost = System.currentTimeMillis() - start; LOG.debug("reading fully costs:" + cost + " milliseconds"); }
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { final StructObjectInspector outputRowOI = (StructObjectInspector) objInspector; final List<? extends StructField> outputFieldRefs = outputRowOI.getAllStructFieldRefs(); if (outputFieldRefs.size() != numCols) { throw new SerDeException( "Cannot serialize the object because there are " + outputFieldRefs.size() + " fields but the table has " + numCols + " columns."); } // Get all data out. for (int c = 0; c < numCols; c++) { final Object field = outputRowOI.getStructFieldData(obj, outputFieldRefs.get(c)); final ObjectInspector fieldOI = outputFieldRefs.get(c).getFieldObjectInspector(); // The data must be of type String final StringObjectInspector fieldStringOI = (StringObjectInspector) fieldOI; // Convert the field to Java class String, because objects of String // type // can be stored in String, Text, or some other classes. outputFields[c] = fieldStringOI.getPrimitiveJavaObject(field); } final StringWriter writer = new StringWriter(); final CSVWriter csv = newWriter(writer, separatorChar, quoteChar, escapeChar); try { csv.writeNext(outputFields); csv.close(); return new Text(writer.toString()); } catch (final IOException ioe) { throw new SerDeException(ioe); } }
/** * If we received data with tags from ReduceSinkOperators, no keys will match. This should not * happen, but is important enough that we want to find out and work around it if some optimized * change causes RSO to pass on tags. */ private void sanityCheckKeyForTag() throws SerDeException { if (hasTag != null) return; BinaryComparable b = (BinaryComparable) key; Object o = keySerDe.deserialize(key); StructObjectInspector soi = (StructObjectInspector) keySerDe.getObjectInspector(); List<? extends StructField> fields = soi.getAllStructFieldRefs(); Object[] data = new Object[fields.size()]; List<ObjectInspector> fois = new ArrayList<ObjectInspector>(fields.size()); for (int i = 0; i < fields.size(); i++) { data[i] = soi.getStructFieldData(o, fields.get(i)); fois.add(fields.get(i).getFieldObjectInspector()); } Output output = new Output(); BinarySortableSerDe.serializeStruct(output, data, fois, new boolean[fields.size()]); hasTag = (output.getLength() != b.getLength()); if (hasTag) { LOG.error("Tag found in keys and will be removed. This should not happen."); if (output.getLength() != (b.getLength() - 1)) { throw new SerDeException( "Unexpected tag: " + b.getLength() + " reserialized to " + output.getLength()); } } }
/** * Copy specified fields in the input row to the output array of standard objects. * * @param result output list of standard objects. * @param row input row. * @param startCol starting column number from the input row. * @param numCols number of columns to copy. * @param soi Object inspector for the to-be-copied columns. */ public static void partialCopyToStandardObject( List<Object> result, Object row, int startCol, int numCols, StructObjectInspector soi, ObjectInspectorCopyOption objectInspectorOption) { List<? extends StructField> fields = soi.getAllStructFieldRefs(); int i = 0, j = 0; for (StructField f : fields) { if (i++ >= startCol) { result.add( copyToStandardObject( soi.getStructFieldData(row, f), f.getFieldObjectInspector(), objectInspectorOption)); if (++j == numCols) { break; } } } }
@Override public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException { assertState(OperationState.FINISHED); ArrayList<String> rows = new ArrayList<String>(); driver.setMaxRows((int) maxRows); try { driver.getResults(rows); getSerDe(); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs(); RowSet rowSet = new RowSet(); Object[] deserializedFields = new Object[fieldRefs.size()]; Object rowObj; ObjectInspector fieldOI; for (String rowString : rows) { rowObj = serde.deserialize(new BytesWritable(rowString.getBytes())); for (int i = 0; i < fieldRefs.size(); i++) { StructField fieldRef = fieldRefs.get(i); fieldOI = fieldRef.getFieldObjectInspector(); deserializedFields[i] = convertLazyToJava(soi.getStructFieldData(rowObj, fieldRef), fieldOI); } rowSet.addRow(resultSchema, deserializedFields); } return rowSet; } catch (IOException e) { throw new HiveSQLException(e); } catch (CommandNeedRetryException e) { throw new HiveSQLException(e); } catch (Exception e) { throw new HiveSQLException(e); } }
public static void copyStructToArray( Object o, ObjectInspector oi, ObjectInspectorCopyOption objectInspectorOption, Object[] dest, int offset) throws SerDeException { if (o == null) { return; } if (oi.getCategory() != Category.STRUCT) { throw new SerDeException("Unexpected category " + oi.getCategory()); } StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); ++i) { StructField f = fields.get(i); dest[offset + i] = copyToStandardObject( soi.getStructFieldData(o, f), f.getFieldObjectInspector(), objectInspectorOption); } }
private static Block serializeStruct( Type type, BlockBuilder builder, Object object, StructObjectInspector inspector) { if (object == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); List<? extends StructField> allStructFieldRefs = inspector.getAllStructFieldRefs(); checkArgument(typeParameters.size() == allStructFieldRefs.size()); BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); } else { currentBuilder = new InterleavedBlockBuilder( typeParameters, new BlockBuilderStatus(), typeParameters.size()); } for (int i = 0; i < typeParameters.size(); i++) { StructField field = allStructFieldRefs.get(i); serializeObject( typeParameters.get(i), currentBuilder, inspector.getStructFieldData(object, field), field.getFieldObjectInspector()); } if (builder != null) { builder.closeEntry(); return null; } else { Block resultBlock = currentBuilder.build(); return resultBlock; } }