@Override
 public void setKeyValue(Writable key, Writable val) throws SerDeException {
   Object keyObj = keySerDe.deserialize(key), valObj = valSerDe.deserialize(val);
   List<? extends StructField> keyFields = keySoi.getAllStructFieldRefs(),
       valFields = valSoi.getAllStructFieldRefs();
   for (int i = 0; i < keyFields.size(); ++i) {
     keyObjs[i] = keySoi.getStructFieldData(keyObj, keyFields.get(i));
   }
   for (int i = 0; i < valFields.size(); ++i) {
     valObjs[i] = valSoi.getStructFieldData(valObj, valFields.get(i));
   }
 }
Esempio n. 2
0
 private void stringifyObject(StringBuilder buffer, Object obj, ObjectInspector inspector)
     throws IOException {
   if (inspector instanceof StructObjectInspector) {
     buffer.append("{ ");
     StructObjectInspector soi = (StructObjectInspector) inspector;
     boolean isFirst = true;
     for (StructField field : soi.getAllStructFieldRefs()) {
       if (isFirst) {
         isFirst = false;
       } else {
         buffer.append(", ");
       }
       buffer.append(field.getFieldName());
       buffer.append(": ");
       stringifyObject(
           buffer, soi.getStructFieldData(obj, field), field.getFieldObjectInspector());
     }
     buffer.append(" }");
   } else if (inspector instanceof PrimitiveObjectInspector) {
     PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
     buffer.append(poi.getPrimitiveJavaObject(obj).toString());
   } else {
     buffer.append("*unknown*");
   }
 }
Esempio n. 3
0
  protected static StructObjectInspector createSelectListOI(
      MatchPath evaluator, PTFInputDef inpDef) {
    StructObjectInspector inOI = inpDef.getOutputShape().getOI();
    ArrayList<String> inputColumnNames = new ArrayList<String>();
    ArrayList<String> selectListNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (StructField f : inOI.getAllStructFieldRefs()) {
      String inputColName = evaluator.inputColumnNamesMap.get(f.getFieldName());
      if (inputColName != null) {
        inputColumnNames.add(inputColName);
        selectListNames.add(f.getFieldName());
        fieldOIs.add(f.getFieldObjectInspector());
      }
    }

    StandardListObjectInspector pathAttrOI =
        ObjectInspectorFactory.getStandardListObjectInspector(
            ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, fieldOIs));

    ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>();
    selectFieldOIs.addAll(fieldOIs);
    selectFieldOIs.add(pathAttrOI);
    selectListNames.add(MatchPath.PATHATTR_NAME);
    return ObjectInspectorFactory.getStandardStructObjectInspector(selectListNames, selectFieldOIs);
  }
  @Override
  public Writable serialize(Object obj, ObjectInspector oi) throws SerDeException {
    if (oi.getCategory() != Category.STRUCT) {
      throw new VoltSerdeException(
          getClass().toString()
              + " can only serialize struct types, but we got: "
              + oi.getTypeName());
    }
    VoltRecord vr = new VoltRecord(m_voltConf.getTableName());
    StructObjectInspector soi = (StructObjectInspector) oi;
    List<? extends StructField> structFields = soi.getAllStructFieldRefs();
    List<Object> fieldValues = soi.getStructFieldsDataAsList(obj);

    final int size = m_oig.getColumnTypes().size();

    for (int i = 0; i < size; ++i) {
      ObjectInspector fieldOI = structFields.get(i).getFieldObjectInspector();
      PrimitiveObjectInspector poi = (PrimitiveObjectInspector) fieldOI;

      Object fieldValue = poi.getPrimitiveJavaObject(fieldValues.get(i));
      if (poi.getTypeInfo().equals(TypeInfoFactory.timestampTypeInfo)) {
        fieldValue = fieldValue != null ? new Date(((Timestamp) fieldValue).getTime()) : null;
      }
      vr.add(fieldValue);
    }

    return vr;
  }
Esempio n. 5
0
  private RowSet decodeFromString(List<Object> rows, RowSet rowSet)
      throws SQLException, SerDeException {
    getSerDe();
    StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();

    Object[] deserializedFields = new Object[fieldRefs.size()];
    Object rowObj;
    ObjectInspector fieldOI;

    int protocol = getProtocolVersion().getValue();
    for (Object rowString : rows) {
      try {
        rowObj = serde.deserialize(new BytesWritable(((String) rowString).getBytes("UTF-8")));
      } catch (UnsupportedEncodingException e) {
        throw new SerDeException(e);
      }
      for (int i = 0; i < fieldRefs.size(); i++) {
        StructField fieldRef = fieldRefs.get(i);
        fieldOI = fieldRef.getFieldObjectInspector();
        Object fieldData = soi.getStructFieldData(rowObj, fieldRef);
        deserializedFields[i] = SerDeUtils.toThriftPayload(fieldData, fieldOI, protocol);
      }
      rowSet.addRow(deserializedFields);
    }
    return rowSet;
  }
Esempio n. 6
0
  public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {

    if (objInspector.getCategory() != Category.STRUCT) {
      throw new SerDeException(
          getClass().toString()
              + " can only serialize struct types, but we got: "
              + objInspector.getTypeName());
    }
    StructObjectInspector soi = (StructObjectInspector) objInspector;
    List<? extends StructField> fields = soi.getAllStructFieldRefs();

    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < fields.size(); i++) {
      if (i > 0) sb.append(separator);
      Object column = soi.getStructFieldData(obj, fields.get(i));
      if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) {
        // For primitive object, serialize to plain string
        sb.append(column == null ? nullString : column.toString());
      } else {
        // For complex object, serialize to JSON format
        sb.append(SerDeUtils.getJSONString(column, fields.get(i).getFieldObjectInspector()));
      }
    }
    serializeCache.set(sb.toString());
    return serializeCache;
  }
  private ObjectInspector solveOi(ObjectInspector arg) {

    switch (arg.getCategory()) {
      case PRIMITIVE:

        // VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL,
        // UNKNOWN
        PrimitiveObjectInspector poi = (PrimitiveObjectInspector) arg;
        return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
            poi.getPrimitiveCategory());
      case LIST:
        return ObjectInspectorFactory.getStandardListObjectInspector(
            solveOi(((ListObjectInspector) arg).getListElementObjectInspector()));
      case MAP:
        return ObjectInspectorFactory.getStandardMapObjectInspector(
            solveOi(((MapObjectInspector) arg).getMapKeyObjectInspector()),
            solveOi(((MapObjectInspector) arg).getMapValueObjectInspector()));
      case STRUCT:
        StructObjectInspector soi = (StructObjectInspector) arg;
        int size = soi.getAllStructFieldRefs().size();
        ArrayList<String> fnl = new ArrayList<String>(size);
        ArrayList<ObjectInspector> foil = new ArrayList<ObjectInspector>(size);

        for (StructField sf : ((StructObjectInspector) arg).getAllStructFieldRefs()) {
          fnl.add(sf.getFieldName());
          foil.add(solveOi(sf.getFieldObjectInspector()));
        }

        return JsonStructObjectInspector.getJsonStructObjectInspector(fnl, foil);
      default:
        return arg;
    }
  }
 /**
  * Whether comparison is supported for this type. Currently all types that references any map are
  * not comparable.
  */
 public static boolean compareSupported(ObjectInspector oi) {
   switch (oi.getCategory()) {
     case PRIMITIVE:
       return true;
     case LIST:
       ListObjectInspector loi = (ListObjectInspector) oi;
       return compareSupported(loi.getListElementObjectInspector());
     case STRUCT:
       StructObjectInspector soi = (StructObjectInspector) oi;
       List<? extends StructField> fields = soi.getAllStructFieldRefs();
       for (int f = 0; f < fields.size(); f++) {
         if (!compareSupported(fields.get(f).getFieldObjectInspector())) {
           return false;
         }
       }
       return true;
     case MAP:
       return false;
     case UNION:
       UnionObjectInspector uoi = (UnionObjectInspector) oi;
       for (ObjectInspector eoi : uoi.getObjectInspectors()) {
         if (!compareSupported(eoi)) {
           return false;
         }
       }
       return true;
     default:
       return false;
   }
 }
  /**
   * Serializing means getting every field, and setting the appropriate JSONObject field. Actual
   * serialization is done at the end when the whole JSON object is built
   *
   * @param serializer
   * @param obj
   * @param structObjectInspector
   */
  private JSONObject serializeStruct(
      Object obj, StructObjectInspector soi, List<String> columnNames) {
    // do nothing for null struct
    if (null == obj) {
      return null;
    }

    JSONObject result = new JSONObject();

    List<? extends StructField> fields = soi.getAllStructFieldRefs();

    for (int i = 0; i < fields.size(); i++) {
      StructField sf = fields.get(i);
      Object data = soi.getStructFieldData(obj, sf);

      if (null != data) {
        try {
          // we want to serialize columns with their proper HIVE name,
          // not the _col2 kind of name usually generated upstream
          result.put(
              getSerializedFieldName(columnNames, i, sf),
              serializeField(data, sf.getFieldObjectInspector()));

        } catch (JSONException ex) {
          LOG.warn("Problem serializing", ex);
          throw new RuntimeException(ex);
        }
      }
    }
    return result;
  }
Esempio n. 10
0
 private static void addInputColumnsToList(
     ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) {
   StructObjectInspector OI = shape.getOI();
   for (StructField f : OI.getAllStructFieldRefs()) {
     fieldNames.add(f.getFieldName());
     fieldOIs.add(f.getFieldObjectInspector());
   }
 }
  /**
   * @param oi - Input object inspector
   * @param oiSettableProperties - Lookup map to cache the result.(If no caching, pass null)
   * @return - true if : (1) oi is an instance of settable<DataType>OI. (2) All the embedded object
   *     inspectors are instances of settable<DataType>OI. If (1) or (2) is false, return false.
   */
  public static boolean hasAllFieldsSettable(
      ObjectInspector oi, Map<ObjectInspector, Boolean> oiSettableProperties) {
    // If the result is already present in the cache, return it.
    if (!(oiSettableProperties == null) && oiSettableProperties.containsKey(oi)) {
      return oiSettableProperties.get(oi).booleanValue();
    }
    // If the top-level object inspector is non-settable return false
    if (!(isInstanceOfSettableOI(oi))) {
      return setOISettablePropertiesMap(oi, oiSettableProperties, false);
    }

    Boolean returnValue = true;

    switch (oi.getCategory()) {
      case PRIMITIVE:
        break;
      case STRUCT:
        StructObjectInspector structOutputOI = (StructObjectInspector) oi;
        List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
        for (StructField listField : listFields) {
          if (!hasAllFieldsSettable(listField.getFieldObjectInspector(), oiSettableProperties)) {
            returnValue = false;
            break;
          }
        }
        break;
      case LIST:
        ListObjectInspector listOutputOI = (ListObjectInspector) oi;
        returnValue =
            hasAllFieldsSettable(
                listOutputOI.getListElementObjectInspector(), oiSettableProperties);
        break;
      case MAP:
        MapObjectInspector mapOutputOI = (MapObjectInspector) oi;
        returnValue =
            hasAllFieldsSettable(mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties)
                && hasAllFieldsSettable(
                    mapOutputOI.getMapValueObjectInspector(), oiSettableProperties);
        break;
      case UNION:
        UnionObjectInspector unionOutputOI = (UnionObjectInspector) oi;
        List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
        for (ObjectInspector listField : unionListFields) {
          if (!hasAllFieldsSettable(listField, oiSettableProperties)) {
            returnValue = false;
            break;
          }
        }
        break;
      default:
        throw new RuntimeException(
            "Hive internal error inside hasAllFieldsSettable : "
                + oi.getTypeName()
                + " not supported yet.");
    }
    return setOISettablePropertiesMap(oi, oiSettableProperties, returnValue);
  }
Esempio n. 12
0
 /** return false though element is null if nullsafe flag is true for that */
 public static boolean hasAnyNullObject(List o, StructObjectInspector loi, boolean[] nullSafes) {
   List<? extends StructField> fields = loi.getAllStructFieldRefs();
   for (int i = 0; i < o.size(); i++) {
     if ((nullSafes == null || !nullSafes[i])
         && hasAnyNullObject(o.get(i), fields.get(i).getFieldObjectInspector())) {
       return true;
     }
   }
   return false;
 }
Esempio n. 13
0
  @Override
  public Writable serialize(final Object obj, final ObjectInspector inspector)
      throws SerDeException {

    final StructObjectInspector structInspector = (StructObjectInspector) inspector;
    final List<? extends StructField> fields = structInspector.getAllStructFieldRefs();
    if (fields.size() != columnNames.size()) {
      throw new SerDeException(
          String.format("Required %d columns, received %d.", columnNames.size(), fields.size()));
    }

    cachedWritable.clear();
    for (int c = 0; c < fieldCount; c++) {
      StructField structField = fields.get(c);

      LOG.debug("fieldId=" + c + ",structField=" + structField.toString());

      if (structField != null) {
        final Object field = structInspector.getStructFieldData(obj, fields.get(c));

        final AbstractPrimitiveObjectInspector fieldOI =
            (AbstractPrimitiveObjectInspector) fields.get(c).getFieldObjectInspector();

        Writable value = (Writable) fieldOI.getPrimitiveWritableObject(field);

        if (value == null) {
          continue;
        }

        LOG.debug("fieldCount=" + fieldCount + ",value=" + value.toString());
        if (value instanceof IntWritable) {
          cachedWritable.put(new Text(columnNames.get(c)), value);
        } else if (value instanceof Text) {
          cachedWritable.put(new Text(columnNames.get(c)), ((Text) value));
        } else if (value instanceof LongWritable) {
          cachedWritable.put(new Text(columnNames.get(c)), ((LongWritable) value));
        } else if (value instanceof DoubleWritable) {
          cachedWritable.put(new Text(columnNames.get(c)), ((DoubleWritable) value));
        } else if (value instanceof FloatWritable) {
          cachedWritable.put(new Text(columnNames.get(c)), ((FloatWritable) value));
        } else if (value instanceof BooleanWritable) {
          cachedWritable.put(new Text(columnNames.get(c)), ((BooleanWritable) value));
        } else if (value instanceof ByteWritable) {
          cachedWritable.put(new Text(columnNames.get(c)), ((ByteWritable) value));
        } else if (value instanceof BytesWritable) {
          cachedWritable.put(new Text(columnNames.get(c)), ((BytesWritable) value));
        } else {
          LOG.warn("fieldCount=" + fieldCount + ",type=" + value.getClass().getName());
        }
      }
    }

    return cachedWritable;
  }
Esempio n. 14
0
  public int openFile() throws Exception {

    m_reader = OrcFile.createReader(m_file_path, OrcFile.readerOptions(m_conf));
    m_types = m_reader.getTypes();
    m_oi = (StructObjectInspector) m_reader.getObjectInspector();
    m_fields = m_oi.getAllStructFieldRefs();

    m_rr = m_reader.rows();

    return 0;
  }
 /** Get the list of field names as csv from a StructObjectInspector. */
 public static String getFieldNames(StructObjectInspector soi) {
   List<? extends StructField> fields = soi.getAllStructFieldRefs();
   StringBuilder sb = new StringBuilder();
   for (int i = 0; i < fields.size(); i++) {
     if (i > 0) {
       sb.append(",");
     }
     sb.append(fields.get(i).getFieldName());
   }
   return sb.toString();
 }
Esempio n. 16
0
 public KeyValueWriter(SerDe keySerDe, SerDe valSerDe, boolean hasFilterTag)
     throws SerDeException {
   this.keySerDe = keySerDe;
   this.valSerDe = valSerDe;
   keySoi = (StructObjectInspector) keySerDe.getObjectInspector();
   valSoi = (StructObjectInspector) valSerDe.getObjectInspector();
   List<? extends StructField> keyFields = keySoi.getAllStructFieldRefs(),
       valFields = valSoi.getAllStructFieldRefs();
   keyOis = new ArrayList<ObjectInspector>(keyFields.size());
   valOis = new ArrayList<ObjectInspector>(valFields.size());
   for (int i = 0; i < keyFields.size(); ++i) {
     keyOis.add(keyFields.get(i).getFieldObjectInspector());
   }
   for (int i = 0; i < valFields.size(); ++i) {
     valOis.add(valFields.get(i).getFieldObjectInspector());
   }
   keyObjs = new Object[keyOis.size()];
   valObjs = new Object[valOis.size()];
   this.hasFilterTag = hasFilterTag;
 }
 private ArrayWritable createStruct(final Object obj, final StructObjectInspector inspector)
     throws SerDeException {
   final List<? extends StructField> fields = inspector.getAllStructFieldRefs();
   final Writable[] arr = new Writable[fields.size()];
   for (int i = 0; i < fields.size(); i++) {
     final StructField field = fields.get(i);
     final Object subObj = inspector.getStructFieldData(obj, field);
     final ObjectInspector subInspector = field.getFieldObjectInspector();
     arr[i] = createObject(subObj, subInspector);
   }
   return new ArrayWritable(Writable.class, arr);
 }
 /* Initialize the UDAF on the reduce side (or the map side in some cases). */
 private void initReduceSide(StructObjectInspector inputStructOI) throws HiveException {
   List<? extends StructField> fields = inputStructOI.getAllStructFieldRefs();
   int length = fields.size();
   inputOIs = new ObjectInspector[length];
   outputOIs = new ObjectInspector[length];
   for (int i = 0; i < length; i++) {
     StructField field = fields.get(i);
     inputOIs[i] = field.getFieldObjectInspector();
     outputOIs[i] = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i]);
   }
   structOI = ObjectInspectorUtils.getStandardObjectInspector(inputStructOI);
 }
 /**
  * Copy fields in the input row to the output array of standard objects.
  *
  * @param result output list of standard objects.
  * @param row input row.
  * @param soi Object inspector for the to-be-copied columns.
  * @param objectInspectorOption
  */
 public static void copyToStandardObject(
     List<Object> result,
     Object row,
     StructObjectInspector soi,
     ObjectInspectorCopyOption objectInspectorOption) {
   List<? extends StructField> fields = soi.getAllStructFieldRefs();
   for (StructField f : fields) {
     result.add(
         copyToStandardObject(
             soi.getStructFieldData(row, f), f.getFieldObjectInspector(), objectInspectorOption));
   }
 }
    /*
     * from the prunedCols list filter out columns that refer to WindowFns or WindowExprs
     * the returned list is set as the prunedList needed by the PTFOp.
     */
    private ArrayList<String> prunedInputList(
        List<String> prunedCols, WindowTableFunctionDef tDef) {
      ArrayList<String> prunedInputCols = new ArrayList<String>();

      StructObjectInspector OI = tDef.getInput().getOutputShape().getOI();
      for (StructField f : OI.getAllStructFieldRefs()) {
        String fName = f.getFieldName();
        if (prunedCols.contains(fName)) {
          prunedInputCols.add(fName);
        }
      }

      return prunedInputCols;
    }
 public static String getStandardStructTypeName(StructObjectInspector soi) {
   StringBuilder sb = new StringBuilder();
   sb.append("struct<");
   List<? extends StructField> fields = soi.getAllStructFieldRefs();
   for (int i = 0; i < fields.size(); i++) {
     if (i > 0) {
       sb.append(",");
     }
     sb.append(fields.get(i).getFieldName());
     sb.append(":");
     sb.append(fields.get(i).getFieldObjectInspector().getTypeName());
   }
   sb.append(">");
   return sb.toString();
 }
  @Override
  public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
    outputByteBuffer.reset();
    StructObjectInspector soi = (StructObjectInspector) objInspector;
    List<? extends StructField> fields = soi.getAllStructFieldRefs();

    for (int i = 0; i < columnNames.size(); i++) {
      serialize(
          outputByteBuffer,
          soi.getStructFieldData(obj, fields.get(i)),
          fields.get(i).getFieldObjectInspector(),
          columnSortOrderIsDesc[i]);
    }

    serializeBytesWritable.set(outputByteBuffer.getData(), 0, outputByteBuffer.getLength());
    return serializeBytesWritable;
  }
Esempio n. 23
0
  private void partialReadTest(FileSystem fs, int count, Path file)
      throws IOException, SerDeException {
    LOG.debug("reading " + count + " records");
    long start = System.currentTimeMillis();
    java.util.ArrayList<Integer> readCols = new java.util.ArrayList<Integer>();
    readCols.add(Integer.valueOf(2));
    readCols.add(Integer.valueOf(3));
    ColumnProjectionUtils.setReadColumnIDs(conf, readCols);
    RCFile.Reader reader = new RCFile.Reader(fs, file, conf);

    LongWritable rowID = new LongWritable();
    BytesRefArrayWritable cols = new BytesRefArrayWritable();

    while (reader.next(rowID)) {
      reader.getCurrentRow(cols);
      cols.resetValid(8);
      Object row = serDe.deserialize(cols);

      StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
      List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
      assertEquals("Field size should be 8", 8, fieldRefs.size());

      for (int i : readCols) {
        Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
        Object standardWritableData =
            ObjectInspectorUtils.copyToStandardObject(
                fieldData,
                fieldRefs.get(i).getFieldObjectInspector(),
                ObjectInspectorCopyOption.WRITABLE);
        assertEquals("Field " + i, standardWritableData, expectedPartitalFieldsData[i]);
      }

      assertEquals(
          "Class of the serialized object should be BytesRefArrayWritable",
          BytesRefArrayWritable.class,
          serDe.getSerializedClass());
      BytesRefArrayWritable serializedBytes = (BytesRefArrayWritable) serDe.serialize(row, oi);
      assertEquals("Serialized data", patialS, serializedBytes);
    }
    reader.close();
    long cost = System.currentTimeMillis() - start;
    LOG.debug("reading fully costs:" + cost + " milliseconds");
  }
Esempio n. 24
0
  public void fullyReadTest(FileSystem fs, int count, Path file)
      throws IOException, SerDeException {
    LOG.debug("reading " + count + " records");
    long start = System.currentTimeMillis();
    ColumnProjectionUtils.setFullyReadColumns(conf);
    RCFile.Reader reader = new RCFile.Reader(fs, file, conf);

    LongWritable rowID = new LongWritable();
    int actualRead = 0;
    BytesRefArrayWritable cols = new BytesRefArrayWritable();
    while (reader.next(rowID)) {
      reader.getCurrentRow(cols);
      cols.resetValid(8);
      Object row = serDe.deserialize(cols);

      StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
      List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
      assertEquals("Field size should be 8", 8, fieldRefs.size());
      for (int i = 0; i < fieldRefs.size(); i++) {
        Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
        Object standardWritableData =
            ObjectInspectorUtils.copyToStandardObject(
                fieldData,
                fieldRefs.get(i).getFieldObjectInspector(),
                ObjectInspectorCopyOption.WRITABLE);
        assertEquals("Field " + i, standardWritableData, expectedFieldsData[i]);
      }
      // Serialize
      assertEquals(
          "Class of the serialized object should be BytesRefArrayWritable",
          BytesRefArrayWritable.class,
          serDe.getSerializedClass());
      BytesRefArrayWritable serializedText = (BytesRefArrayWritable) serDe.serialize(row, oi);
      assertEquals("Serialized data", s, serializedText);
      actualRead++;
    }
    reader.close();
    assertEquals("Expect " + count + " rows, actual read " + actualRead, actualRead, count);
    long cost = System.currentTimeMillis() - start;
    LOG.debug("reading fully costs:" + cost + " milliseconds");
  }
  @Override
  public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
    final StructObjectInspector outputRowOI = (StructObjectInspector) objInspector;
    final List<? extends StructField> outputFieldRefs = outputRowOI.getAllStructFieldRefs();

    if (outputFieldRefs.size() != numCols) {
      throw new SerDeException(
          "Cannot serialize the object because there are "
              + outputFieldRefs.size()
              + " fields but the table has "
              + numCols
              + " columns.");
    }

    // Get all data out.
    for (int c = 0; c < numCols; c++) {
      final Object field = outputRowOI.getStructFieldData(obj, outputFieldRefs.get(c));
      final ObjectInspector fieldOI = outputFieldRefs.get(c).getFieldObjectInspector();

      // The data must be of type String
      final StringObjectInspector fieldStringOI = (StringObjectInspector) fieldOI;

      // Convert the field to Java class String, because objects of String
      // type
      // can be stored in String, Text, or some other classes.
      outputFields[c] = fieldStringOI.getPrimitiveJavaObject(field);
    }

    final StringWriter writer = new StringWriter();
    final CSVWriter csv = newWriter(writer, separatorChar, quoteChar, escapeChar);

    try {
      csv.writeNext(outputFields);
      csv.close();

      return new Text(writer.toString());
    } catch (final IOException ioe) {
      throw new SerDeException(ioe);
    }
  }
Esempio n. 26
0
 /**
  * If we received data with tags from ReduceSinkOperators, no keys will match. This should not
  * happen, but is important enough that we want to find out and work around it if some optimized
  * change causes RSO to pass on tags.
  */
 private void sanityCheckKeyForTag() throws SerDeException {
   if (hasTag != null) return;
   BinaryComparable b = (BinaryComparable) key;
   Object o = keySerDe.deserialize(key);
   StructObjectInspector soi = (StructObjectInspector) keySerDe.getObjectInspector();
   List<? extends StructField> fields = soi.getAllStructFieldRefs();
   Object[] data = new Object[fields.size()];
   List<ObjectInspector> fois = new ArrayList<ObjectInspector>(fields.size());
   for (int i = 0; i < fields.size(); i++) {
     data[i] = soi.getStructFieldData(o, fields.get(i));
     fois.add(fields.get(i).getFieldObjectInspector());
   }
   Output output = new Output();
   BinarySortableSerDe.serializeStruct(output, data, fois, new boolean[fields.size()]);
   hasTag = (output.getLength() != b.getLength());
   if (hasTag) {
     LOG.error("Tag found in keys and will be removed. This should not happen.");
     if (output.getLength() != (b.getLength() - 1)) {
       throw new SerDeException(
           "Unexpected tag: " + b.getLength() + " reserialized to " + output.getLength());
     }
   }
 }
  /**
   * Copy specified fields in the input row to the output array of standard objects.
   *
   * @param result output list of standard objects.
   * @param row input row.
   * @param startCol starting column number from the input row.
   * @param numCols number of columns to copy.
   * @param soi Object inspector for the to-be-copied columns.
   */
  public static void partialCopyToStandardObject(
      List<Object> result,
      Object row,
      int startCol,
      int numCols,
      StructObjectInspector soi,
      ObjectInspectorCopyOption objectInspectorOption) {

    List<? extends StructField> fields = soi.getAllStructFieldRefs();
    int i = 0, j = 0;
    for (StructField f : fields) {
      if (i++ >= startCol) {
        result.add(
            copyToStandardObject(
                soi.getStructFieldData(row, f),
                f.getFieldObjectInspector(),
                objectInspectorOption));
        if (++j == numCols) {
          break;
        }
      }
    }
  }
Esempio n. 28
0
  @Override
  public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
    assertState(OperationState.FINISHED);
    ArrayList<String> rows = new ArrayList<String>();
    driver.setMaxRows((int) maxRows);

    try {
      driver.getResults(rows);

      getSerDe();
      StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
      List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
      RowSet rowSet = new RowSet();

      Object[] deserializedFields = new Object[fieldRefs.size()];
      Object rowObj;
      ObjectInspector fieldOI;

      for (String rowString : rows) {
        rowObj = serde.deserialize(new BytesWritable(rowString.getBytes()));
        for (int i = 0; i < fieldRefs.size(); i++) {
          StructField fieldRef = fieldRefs.get(i);
          fieldOI = fieldRef.getFieldObjectInspector();
          deserializedFields[i] =
              convertLazyToJava(soi.getStructFieldData(rowObj, fieldRef), fieldOI);
        }
        rowSet.addRow(resultSchema, deserializedFields);
      }
      return rowSet;
    } catch (IOException e) {
      throw new HiveSQLException(e);
    } catch (CommandNeedRetryException e) {
      throw new HiveSQLException(e);
    } catch (Exception e) {
      throw new HiveSQLException(e);
    }
  }
  public static void copyStructToArray(
      Object o,
      ObjectInspector oi,
      ObjectInspectorCopyOption objectInspectorOption,
      Object[] dest,
      int offset)
      throws SerDeException {
    if (o == null) {
      return;
    }

    if (oi.getCategory() != Category.STRUCT) {
      throw new SerDeException("Unexpected category " + oi.getCategory());
    }

    StructObjectInspector soi = (StructObjectInspector) oi;
    List<? extends StructField> fields = soi.getAllStructFieldRefs();
    for (int i = 0; i < fields.size(); ++i) {
      StructField f = fields.get(i);
      dest[offset + i] =
          copyToStandardObject(
              soi.getStructFieldData(o, f), f.getFieldObjectInspector(), objectInspectorOption);
    }
  }
Esempio n. 30
0
  private static Block serializeStruct(
      Type type, BlockBuilder builder, Object object, StructObjectInspector inspector) {
    if (object == null) {
      requireNonNull(builder, "parent builder is null").appendNull();
      return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    List<? extends StructField> allStructFieldRefs = inspector.getAllStructFieldRefs();
    checkArgument(typeParameters.size() == allStructFieldRefs.size());
    BlockBuilder currentBuilder;
    if (builder != null) {
      currentBuilder = builder.beginBlockEntry();
    } else {
      currentBuilder =
          new InterleavedBlockBuilder(
              typeParameters, new BlockBuilderStatus(), typeParameters.size());
    }

    for (int i = 0; i < typeParameters.size(); i++) {
      StructField field = allStructFieldRefs.get(i);
      serializeObject(
          typeParameters.get(i),
          currentBuilder,
          inspector.getStructFieldData(object, field),
          field.getFieldObjectInspector());
    }

    if (builder != null) {
      builder.closeEntry();
      return null;
    } else {
      Block resultBlock = currentBuilder.build();
      return resultBlock;
    }
  }