public List<Record> runNewPartitionRecord() throws Exception {
HiveMetastoreTarget hiveTarget = new HiveMetastoreTargetBuilder().build();
TargetRunner runner =
new TargetRunner.Builder(HiveMetastoreTarget.class, hiveTarget)
.setOnRecordError(OnRecordError.TO_ERROR)
.build();
runner.runInit();
Assert.assertEquals("There should be no error records", 0, runner.getErrorRecords().size());
LinkedHashMap<String, String> partitionVals = new LinkedHashMap<String, String>();
partitionVals.put("dt", "2016");
Field newPartitionField =
HiveMetastoreUtil.newPartitionMetadataFieldBuilder(
"default", "tbl", partitionVals, "/user/hive/warehouse/tbl/dt=2016");
Record record = RecordCreator.create();
record.set(newPartitionField);
runner.runWrite(ImmutableList.of(record));
try {
return runner.getEventRecords();
} finally {
runner.runDestroy();
}
}
@SuppressWarnings("unchecked")
public List<Record> runNewTableRecord() throws Exception {
HiveMetastoreTarget hiveTarget = new HiveMetastoreTargetBuilder().build();
TargetRunner runner =
new TargetRunner.Builder(HiveMetastoreTarget.class, hiveTarget)
.setOnRecordError(OnRecordError.STOP_PIPELINE)
.build();
runner.runInit();
LinkedHashMap<String, HiveTypeInfo> columns = new LinkedHashMap<>();
columns.put("name", HiveType.STRING.getSupport().generateHiveTypeInfoFromResultSet("STRING"));
columns.put(
"surname", HiveType.STRING.getSupport().generateHiveTypeInfoFromResultSet("STRING"));
LinkedHashMap<String, HiveTypeInfo> partitions = new LinkedHashMap<>();
partitions.put("dt", HiveType.STRING.getSupport().generateHiveTypeInfoFromResultSet("STRING"));
Field newTableField =
HiveMetastoreUtil.newSchemaMetadataFieldBuilder(
"default",
"tbl",
columns,
partitions,
true,
BaseHiveIT.getDefaultWareHouseDir(),
HiveMetastoreUtil.generateAvroSchema(columns, "tbl"));
Record record = RecordCreator.create();
record.set(newTableField);
Assert.assertTrue(HiveMetastoreUtil.isSchemaChangeRecord(record));
runner.runWrite(ImmutableList.of(record));
assertTableStructure(
"default.tbl",
ImmutablePair.of("tbl.name", Types.VARCHAR),
ImmutablePair.of("tbl.surname", Types.VARCHAR),
ImmutablePair.of("tbl.dt", Types.VARCHAR));
try {
return runner.getEventRecords();
} finally {
runner.runDestroy();
}
}
