Beispiel #1
0
 public static LazySimpleSerDe getKeyValueLineSerde(
     @Nonnull final PrimitiveObjectInspector keyOI,
     @Nonnull final PrimitiveObjectInspector valueOI)
     throws SerDeException {
   LazySimpleSerDe serde = new LazySimpleSerDe();
   Configuration conf = new Configuration();
   Properties tbl = new Properties();
   tbl.setProperty("columns", "key,value");
   tbl.setProperty("columns.types", keyOI.getTypeName() + "," + valueOI.getTypeName());
   serde.initialize(conf, tbl);
   return serde;
 }
Beispiel #2
0
  public void testRW() throws Exception {

    Configuration conf = new Configuration();

    for (Entry<Properties, HCatRecord> e : getData().entrySet()) {
      Properties tblProps = e.getKey();
      HCatRecord r = e.getValue();

      HCatRecordSerDe hrsd = new HCatRecordSerDe();
      SerDeUtils.initializeSerDe(hrsd, conf, tblProps, null);

      LOG.info("ORIG: {}", r);

      Writable s = hrsd.serialize(r, hrsd.getObjectInspector());
      LOG.info("ONE: {}", s);

      HCatRecord r2 = (HCatRecord) hrsd.deserialize(s);
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, r2));

      // If it went through correctly, then s is also a HCatRecord,
      // and also equal to the above, and a deepcopy, and this holds
      // through for multiple levels more of serialization as well.

      Writable s2 = hrsd.serialize(s, hrsd.getObjectInspector());
      LOG.info("TWO: {}", s2);
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s));
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s2));

      // serialize using another serde, and read out that object repr.
      LazySimpleSerDe testSD = new LazySimpleSerDe();
      SerDeUtils.initializeSerDe(testSD, conf, tblProps, null);

      Writable s3 = testSD.serialize(s, hrsd.getObjectInspector());
      LOG.info("THREE: {}", s3);
      Object o3 = testSD.deserialize(s3);
      Assert.assertFalse(r.getClass().equals(o3.getClass()));

      // then serialize again using hrsd, and compare results
      HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector());
      LOG.info("FOUR: {}", s4);

      // Test LazyHCatRecord init and read
      LazyHCatRecord s5 = new LazyHCatRecord(o3, testSD.getObjectInspector());
      LOG.info("FIVE: {}", s5);

      LazyHCatRecord s6 = new LazyHCatRecord(s4, hrsd.getObjectInspector());
      LOG.info("SIX: {}", s6);
    }
  }
Beispiel #3
0
  public static LazySimpleSerDe getLineSerde(@Nonnull final PrimitiveObjectInspector... OIs)
      throws SerDeException {
    if (OIs.length == 0) {
      throw new IllegalArgumentException("OIs must be specified");
    }
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();

    StringBuilder columnNames = new StringBuilder();
    StringBuilder columnTypes = new StringBuilder();
    for (int i = 0; i < OIs.length; i++) {
      columnNames.append('c').append(i + 1).append(',');
      columnTypes.append(OIs[i].getTypeName()).append(',');
    }
    columnNames.deleteCharAt(columnNames.length() - 1);
    columnTypes.deleteCharAt(columnTypes.length() - 1);

    tbl.setProperty("columns", columnNames.toString());
    tbl.setProperty("columns.types", columnTypes.toString());
    serde.initialize(conf, tbl);
    return serde;
  }
Beispiel #4
0
  @Override
  public void initialize(final Configuration conf, final Properties tbl) throws SerDeException {

    LOG.debug("Entry SdbSerDe::initialize");

    final String columnString = tbl.getProperty(ConfigurationUtil.COLUMN_MAPPING);
    if (StringUtils.isBlank(columnString)) {
      throw new SerDeException("No column mapping found, use " + ConfigurationUtil.COLUMN_MAPPING);
    }
    final String[] columnNamesArray = ConfigurationUtil.getAllColumns(columnString);
    fieldCount = columnNamesArray.length;
    columnNames = new ArrayList<String>(columnNamesArray.length);
    columnNames.addAll(Arrays.asList(columnNamesArray));

    serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, getClass().getName());

    LOG.debug(
        "EscapeChar:"
            + serdeParams.getEscapeChar()
            + "getSeparators:"
            + new String(serdeParams.getSeparators()));

    byte[] sparator = new byte[1];
    sparator[0] = '|';
    objectInspector =
        LazyFactory.createLazyStructInspector(
            serdeParams.getColumnNames(),
            serdeParams.getColumnTypes(),
            sparator,
            serdeParams.getNullSequence(),
            serdeParams.isLastColumnTakesRest(),
            serdeParams.isEscaped(),
            serdeParams.getEscapeChar());

    row = new LazyStruct((LazySimpleStructObjectInspector) objectInspector);

    LOG.debug("Exit SdbSerDe::initialize");
  }