public static LazySimpleSerDe getKeyValueLineSerde(
@Nonnull final PrimitiveObjectInspector keyOI,
@Nonnull final PrimitiveObjectInspector valueOI)
throws SerDeException {
LazySimpleSerDe serde = new LazySimpleSerDe();
Configuration conf = new Configuration();
Properties tbl = new Properties();
tbl.setProperty("columns", "key,value");
tbl.setProperty("columns.types", keyOI.getTypeName() + "," + valueOI.getTypeName());
serde.initialize(conf, tbl);
return serde;
}
public void testRW() throws Exception {
Configuration conf = new Configuration();
for (Entry<Properties, HCatRecord> e : getData().entrySet()) {
Properties tblProps = e.getKey();
HCatRecord r = e.getValue();
HCatRecordSerDe hrsd = new HCatRecordSerDe();
SerDeUtils.initializeSerDe(hrsd, conf, tblProps, null);
LOG.info("ORIG: {}", r);
Writable s = hrsd.serialize(r, hrsd.getObjectInspector());
LOG.info("ONE: {}", s);
HCatRecord r2 = (HCatRecord) hrsd.deserialize(s);
Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, r2));
// If it went through correctly, then s is also a HCatRecord,
// and also equal to the above, and a deepcopy, and this holds
// through for multiple levels more of serialization as well.
Writable s2 = hrsd.serialize(s, hrsd.getObjectInspector());
LOG.info("TWO: {}", s2);
Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s));
Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s2));
// serialize using another serde, and read out that object repr.
LazySimpleSerDe testSD = new LazySimpleSerDe();
SerDeUtils.initializeSerDe(testSD, conf, tblProps, null);
Writable s3 = testSD.serialize(s, hrsd.getObjectInspector());
LOG.info("THREE: {}", s3);
Object o3 = testSD.deserialize(s3);
Assert.assertFalse(r.getClass().equals(o3.getClass()));
// then serialize again using hrsd, and compare results
HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector());
LOG.info("FOUR: {}", s4);
// Test LazyHCatRecord init and read
LazyHCatRecord s5 = new LazyHCatRecord(o3, testSD.getObjectInspector());
LOG.info("FIVE: {}", s5);
LazyHCatRecord s6 = new LazyHCatRecord(s4, hrsd.getObjectInspector());
LOG.info("SIX: {}", s6);
}
}
public static LazySimpleSerDe getLineSerde(@Nonnull final PrimitiveObjectInspector... OIs)
throws SerDeException {
if (OIs.length == 0) {
throw new IllegalArgumentException("OIs must be specified");
}
LazySimpleSerDe serde = new LazySimpleSerDe();
Configuration conf = new Configuration();
Properties tbl = new Properties();
StringBuilder columnNames = new StringBuilder();
StringBuilder columnTypes = new StringBuilder();
for (int i = 0; i < OIs.length; i++) {
columnNames.append('c').append(i + 1).append(',');
columnTypes.append(OIs[i].getTypeName()).append(',');
}
columnNames.deleteCharAt(columnNames.length() - 1);
columnTypes.deleteCharAt(columnTypes.length() - 1);
tbl.setProperty("columns", columnNames.toString());
tbl.setProperty("columns.types", columnTypes.toString());
serde.initialize(conf, tbl);
return serde;
}
@Override
public void initialize(final Configuration conf, final Properties tbl) throws SerDeException {
LOG.debug("Entry SdbSerDe::initialize");
final String columnString = tbl.getProperty(ConfigurationUtil.COLUMN_MAPPING);
if (StringUtils.isBlank(columnString)) {
throw new SerDeException("No column mapping found, use " + ConfigurationUtil.COLUMN_MAPPING);
}
final String[] columnNamesArray = ConfigurationUtil.getAllColumns(columnString);
fieldCount = columnNamesArray.length;
columnNames = new ArrayList<String>(columnNamesArray.length);
columnNames.addAll(Arrays.asList(columnNamesArray));
serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, getClass().getName());
LOG.debug(
"EscapeChar:"
+ serdeParams.getEscapeChar()
+ "getSeparators:"
+ new String(serdeParams.getSeparators()));
byte[] sparator = new byte[1];
sparator[0] = '|';
objectInspector =
LazyFactory.createLazyStructInspector(
serdeParams.getColumnNames(),
serdeParams.getColumnTypes(),
sparator,
serdeParams.getNullSequence(),
serdeParams.isLastColumnTakesRest(),
serdeParams.isEscaped(),
serdeParams.getEscapeChar());
row = new LazyStruct((LazySimpleStructObjectInspector) objectInspector);
LOG.debug("Exit SdbSerDe::initialize");
}