private boolean isClonedTable(String orignalTable, String newTable) throws Exception {
assertTableExists(newTable);
TableDesc origTableDesc = client.getTableDesc(orignalTable);
TableDesc newTableDesc = client.getTableDesc(newTable);
if (isClonedSchema(origTableDesc.getSchema(), newTableDesc.getSchema()) == false) {
fail("Schema of input tables do not match");
return false;
}
// Check partition information
PartitionMethodDesc origPartMethod = origTableDesc.getPartitionMethod();
PartitionMethodDesc newPartMethod = newTableDesc.getPartitionMethod();
if (origPartMethod != null) {
if (newPartMethod == null) {
fail("New table does not have partition info");
return false;
}
if (isClonedSchema(origPartMethod.getExpressionSchema(), newPartMethod.getExpressionSchema())
== false) {
fail("Partition columns of input tables do not match");
return false;
}
if (origPartMethod.getPartitionType().equals(newPartMethod.getPartitionType()) == false) {
fail("Partition type of input tables do not match");
return false;
}
}
// Check external flag
if (origTableDesc.isExternal() != newTableDesc.isExternal()) {
fail("External table flag on input tables not equal");
return false;
}
if (origTableDesc.getMeta() != null) {
TableMeta origMeta = origTableDesc.getMeta();
TableMeta newMeta = newTableDesc.getMeta();
if (origMeta.getDataFormat().equals(newMeta.getDataFormat()) == false) {
fail("Store type of input tables not equal");
return false;
}
KeyValueSet origOptions = origMeta.getPropertySet();
KeyValueSet newOptions = newMeta.getPropertySet();
if (origOptions.equals(newOptions) == false) {
fail("Meta options of input tables not equal");
return false;
}
}
return true;
}
@Test
public final void testAddAndDeleteTablePartitionByRange() throws Exception {
Schema schema =
SchemaBuilder.builder()
.add("id", Type.INT4)
.add("name", Type.TEXT)
.add("age", Type.INT4)
.add("score", Type.FLOAT8)
.build();
String tableName =
IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).build();
PartitionMethodDesc partitionDesc =
new PartitionMethodDesc(
DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.RANGE, "id", partSchema);
TableDesc desc =
new TableDesc(
tableName,
schema,
meta,
new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
assertTrue(catalog.existsTable(tableName));
TableDesc retrieved = catalog.getTableDesc(tableName);
assertEquals(retrieved.getName(), tableName);
assertEquals(
retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.RANGE);
assertEquals(
retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");
catalog.dropTable(tableName);
assertFalse(catalog.existsTable(tableName));
}
@Test
public final void testCtasWithOptions() throws Exception {
ResultSet res = executeFile("CtasWithOptions.sql");
res.close();
ResultSet res2 = executeQuery();
resultSetToString(res2);
res2.close();
TableDesc desc =
client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
assertNotNull(desc);
assertTrue("CSV".equalsIgnoreCase(desc.getMeta().getStoreType()));
KeyValueSet options = desc.getMeta().getOptions();
assertNotNull(options);
assertEquals(
StringEscapeUtils.escapeJava("\u0001"), options.get(StorageConstants.TEXT_DELIMITER));
}
@Test
public void testAlterTable() throws Exception {
// CREATE_TABLE
TableDesc tableRenameTestDesc = createMockupTable("default", "mycooltable");
catalog.createTable(tableRenameTestDesc);
// RENAME_TABLE
catalog.alterTable(createMockAlterTableName());
assertTrue(catalog.existsTable("default", "mynewcooltable"));
// RENAME_COLUMN
catalog.alterTable(createMockAlterTableRenameColumn());
TableDesc columnRenameDesc = catalog.getTableDesc("default", "mynewcooltable");
assertTrue(columnRenameDesc.getSchema().containsByName("ren" + FieldName1));
// ADD_COLUMN
catalog.alterTable(createMockAlterTableAddColumn());
TableDesc addColumnDesc = catalog.getTableDesc("default", "mynewcooltable");
assertTrue(addColumnDesc.getSchema().containsByName("mynewcol"));
// SET_PROPERTY
TableDesc setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
KeyValueSet options = new KeyValueSet();
options.set("timezone", "GMT+9"); // Seoul, Korea
setPropertyDesc.setMeta(new TableMeta("TEXT", options));
String prevTimeZone = setPropertyDesc.getMeta().getProperty("timezone");
String newTimeZone = "GMT-7"; // Silicon Valley, California
catalog.alterTable(createMockAlterTableSetProperty(newTimeZone));
setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
assertNotEquals(prevTimeZone, setPropertyDesc.getMeta().getProperty("timezone"));
assertEquals(newTimeZone, setPropertyDesc.getMeta().getProperty("timezone"));
// UNSET_PROPERTY
catalog.alterTable(createMockAlterTableUnsetProperty(Sets.newHashSet("dummy")));
setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
assertTrue(setPropertyDesc.getMeta().getPropertySet().containsKey("timezone"));
assertFalse(setPropertyDesc.getMeta().getPropertySet().containsKey("dummy"));
}
// TODO: This should be added at TAJO-1891
public final void testAddAndDeleteTablePartitionByColumn() throws Exception {
Schema schema =
SchemaBuilder.builder()
.add("id", Type.INT4)
.add("name", Type.TEXT)
.add("age", Type.INT4)
.add("score", Type.FLOAT8)
.build();
String simpleTableName = "addedtable";
String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName);
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).add("name", Type.TEXT).build();
PartitionMethodDesc partitionMethodDesc =
new PartitionMethodDesc(
DEFAULT_DATABASE_NAME,
tableName,
CatalogProtos.PartitionType.COLUMN,
"id,name",
partSchema);
TableDesc desc =
new TableDesc(
tableName,
schema,
meta,
new Path(CommonTestingUtil.getTestDir(), simpleTableName).toUri());
desc.setPartitionMethod(partitionMethodDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
assertTrue(catalog.existsTable(tableName));
TableDesc retrieved = catalog.getTableDesc(tableName);
assertEquals(retrieved.getName(), tableName);
assertEquals(
retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN);
assertEquals(
retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");
testAddPartition(tableName, "id=10/name=aaa");
testAddPartition(tableName, "id=20/name=bbb");
List<CatalogProtos.PartitionDescProto> partitions =
catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
assertNotNull(partitions);
assertEquals(partitions.size(), 2);
assertEquals(partitions.get(0).getNumBytes(), 0L);
testGetPartitionsByAlgebra(DEFAULT_DATABASE_NAME, simpleTableName);
testDropPartition(tableName, "id=10/name=aaa");
testDropPartition(tableName, "id=20/name=bbb");
partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
assertNotNull(partitions);
assertEquals(partitions.size(), 0);
catalog.dropTable(tableName);
assertFalse(catalog.existsTable(tableName));
}
@Override
public final CatalogProtos.TableDescProto getTable(String databaseName, final String tableName)
throws CatalogException {
org.apache.hadoop.hive.ql.metadata.Table table = null;
HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
Path path = null;
String storeType = null;
org.apache.tajo.catalog.Schema schema = null;
KeyValueSet options = null;
TableStats stats = null;
PartitionMethodDesc partitions = null;
//////////////////////////////////
// set tajo table schema.
//////////////////////////////////
try {
// get hive table schema
try {
client = clientPool.getClient();
table = HiveCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName);
path = table.getPath();
} catch (NoSuchObjectException nsoe) {
throw new UndefinedTableException(tableName);
} catch (Exception e) {
throw new TajoInternalError(e);
}
// convert HiveCatalogStore field schema into tajo field schema.
schema = new org.apache.tajo.catalog.Schema();
List<FieldSchema> fieldSchemaList = table.getCols();
boolean isPartitionKey = false;
for (FieldSchema eachField : fieldSchemaList) {
isPartitionKey = false;
if (table.getPartitionKeys() != null) {
for (FieldSchema partitionKey : table.getPartitionKeys()) {
if (partitionKey.getName().equals(eachField.getName())) {
isPartitionKey = true;
}
}
}
if (!isPartitionKey) {
String fieldName =
databaseName
+ CatalogConstants.IDENTIFIER_DELIMITER
+ tableName
+ CatalogConstants.IDENTIFIER_DELIMITER
+ eachField.getName();
TajoDataTypes.Type dataType =
HiveCatalogUtil.getTajoFieldType(eachField.getType().toString());
schema.addColumn(fieldName, dataType);
}
}
// validate field schema.
HiveCatalogUtil.validateSchema(table);
stats = new TableStats();
options = new KeyValueSet();
options.putAll(table.getParameters());
options.remove("EXTERNAL");
Properties properties = table.getMetadata();
if (properties != null) {
// set field delimiter
String fieldDelimiter = "", nullFormat = "";
if (properties.getProperty(serdeConstants.FIELD_DELIM) != null) {
fieldDelimiter = properties.getProperty(serdeConstants.FIELD_DELIM);
} else {
// if hive table used default row format delimiter, Properties doesn't have it.
// So, Tajo must set as follows:
fieldDelimiter = "\u0001";
}
// set null format
if (properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT) != null) {
nullFormat = properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT);
} else {
nullFormat = "\\N";
}
options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT);
// set file output format
String fileOutputformat =
properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT);
storeType = HiveCatalogUtil.getStoreType(fileOutputformat);
if (storeType.equalsIgnoreCase("TEXT")) {
options.set(
StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat));
} else if (storeType.equals("RCFILE")) {
options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) {
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (ColumnarSerDe.class.getName().equals(serde)) {
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
} else if (storeType.equals("SEQUENCEFILE")) {
options.set(
StorageConstants.SEQUENCEFILE_DELIMITER,
StringEscapeUtils.escapeJava(fieldDelimiter));
options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinarySerDe.class.getName().equals(serde)) {
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (LazySimpleSerDe.class.getName().equals(serde)) {
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
}
// set data size
long totalSize = 0;
if (properties.getProperty("totalSize") != null) {
totalSize = Long.parseLong(properties.getProperty("totalSize"));
} else {
try {
FileSystem fs = path.getFileSystem(conf);
if (fs.exists(path)) {
totalSize = fs.getContentSummary(path).getLength();
}
} catch (IOException ioe) {
throw new TajoInternalError(ioe);
}
}
stats.setNumBytes(totalSize);
}
// set partition keys
List<FieldSchema> partitionKeys = table.getPartitionKeys();
if (null != partitionKeys) {
org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema();
StringBuilder sb = new StringBuilder();
if (partitionKeys.size() > 0) {
for (int i = 0; i < partitionKeys.size(); i++) {
FieldSchema fieldSchema = partitionKeys.get(i);
TajoDataTypes.Type dataType =
HiveCatalogUtil.getTajoFieldType(fieldSchema.getType().toString());
String fieldName =
databaseName
+ CatalogConstants.IDENTIFIER_DELIMITER
+ tableName
+ CatalogConstants.IDENTIFIER_DELIMITER
+ fieldSchema.getName();
expressionSchema.addColumn(new Column(fieldName, dataType));
if (i > 0) {
sb.append(",");
}
sb.append(fieldSchema.getName());
}
partitions =
new PartitionMethodDesc(
databaseName, tableName, PartitionType.COLUMN, sb.toString(), expressionSchema);
}
}
} finally {
if (client != null) client.release();
}
TableMeta meta = new TableMeta(storeType, options);
TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri());
if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) {
tableDesc.setExternal(true);
}
if (stats != null) {
tableDesc.setStats(stats);
}
if (partitions != null) {
tableDesc.setPartitionMethod(partitions);
}
return tableDesc.getProto();
}
@Test
public void testColumnKeyValueMapping() throws Exception {
KeyValueSet keyValueSet = new KeyValueSet();
keyValueSet.set(HBaseStorageConstants.META_TABLE_KEY, "test");
keyValueSet.set(HBaseStorageConstants.META_COLUMNS_KEY, ":key,col2:key:,col2:value:#b,col3:");
Schema schema = new Schema();
schema.addColumn("c1", Type.TEXT);
schema.addColumn("c2", Type.TEXT);
schema.addColumn("c3", Type.TEXT);
schema.addColumn("c4", Type.TEXT);
TableMeta tableMeta = new TableMeta("HBASE", keyValueSet);
ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta.getPropertySet());
List<String> cfNames = columnMapping.getColumnFamilyNames();
assertEquals(2, cfNames.size());
assertEquals("col2", cfNames.get(0));
assertEquals("col3", cfNames.get(1));
for (int i = 0; i < columnMapping.getIsBinaryColumns().length; i++) {
if (i == 2) {
assertTrue(columnMapping.getIsBinaryColumns()[i]);
} else {
assertFalse(columnMapping.getIsBinaryColumns()[i]);
}
}
for (int i = 0; i < columnMapping.getIsRowKeyMappings().length; i++) {
if (i == 0) {
assertTrue(columnMapping.getIsRowKeyMappings()[i]);
} else {
assertFalse(columnMapping.getIsRowKeyMappings()[i]);
}
}
String[] expectedColumnNames = {null, null, null, null};
for (int i = 0; i < schema.size(); i++) {
String columnName =
columnMapping.getMappingColumns()[i][1] == null
? null
: new String(columnMapping.getMappingColumns()[i][1]);
assertEquals(expectedColumnNames[i], columnName);
}
for (int i = 0; i < schema.size(); i++) {
if (i == 1) {
assertTrue(columnMapping.getIsColumnKeys()[i]);
} else {
assertFalse(columnMapping.getIsColumnKeys()[i]);
}
}
for (int i = 0; i < schema.size(); i++) {
if (i == 2) {
assertTrue(columnMapping.getIsColumnValues()[i]);
} else {
assertFalse(columnMapping.getIsColumnValues()[i]);
}
}
}