private final void assertPathOfCreatedTable(
final String databaseName,
final String originalTableName,
final String newTableName,
String createTableStmt)
throws Exception {
// create one table
executeString("CREATE DATABASE " + CatalogUtil.denormalizeIdentifier(databaseName)).close();
getClient().existDatabase(CatalogUtil.denormalizeIdentifier(databaseName));
final String oldFQTableName = CatalogUtil.buildFQName(databaseName, originalTableName);
ResultSet res = executeString(createTableStmt);
res.close();
assertTableExists(oldFQTableName);
TableDesc oldTableDesc = client.getTableDesc(oldFQTableName);
// checking the existence of the table directory and validating the path
Path warehouseDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration());
FileSystem fs = warehouseDir.getFileSystem(testingCluster.getConfiguration());
assertTrue(fs.exists(new Path(oldTableDesc.getUri())));
assertEquals(
StorageUtil.concatPath(warehouseDir, databaseName, originalTableName),
new Path(oldTableDesc.getUri()));
// Rename
client.executeQuery(
"ALTER TABLE "
+ CatalogUtil.denormalizeIdentifier(oldFQTableName)
+ " RENAME to "
+ CatalogUtil.denormalizeIdentifier(newTableName));
// checking the existence of the new table directory and validating the path
final String newFQTableName = CatalogUtil.buildFQName(databaseName, newTableName);
TableDesc newTableDesc = client.getTableDesc(newFQTableName);
assertTrue(fs.exists(new Path(newTableDesc.getUri())));
assertEquals(
StorageUtil.concatPath(warehouseDir, databaseName, newTableName),
new Path(newTableDesc.getUri()));
}
@Test
public final void testSelfDescTable2() throws Exception {
executeString("create database d10;").close();
String className = getClass().getSimpleName();
Path currentDatasetPath = new Path(datasetBasePath, className);
Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
String sql =
"create external table d10.schemaless (*) using json with ('compression.codec'='none') location '"
+ filePath.toString()
+ "'";
executeString(sql).close();
assertTableExists("d10.schemaless");
TableDesc desc = getClient().getTableDesc("d10.schemaless");
assertTrue(desc.hasEmptySchema());
executeString("drop table d10.schemaless").close();
executeString("drop database d10").close();
}
private void assertOutputResult(
String expectedResultFile, String actual, String[] paramKeys, String[] paramValues)
throws Exception {
FileSystem fs =
currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile);
assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile));
String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
if (paramKeys != null) {
for (int i = 0; i < paramKeys.length; i++) {
if (i < paramValues.length) {
expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]);
}
}
}
assertEquals(expectedResult.trim(), actual.trim());
}
private Path getDataSetFile(String fileName) {
return StorageUtil.concatPath(currentDatasetPath, fileName);
}
private Path getResultFile(String fileName) {
return StorageUtil.concatPath(currentResultPath, fileName);
}
private Path getQueryFilePath(String fileName) {
return StorageUtil.concatPath(currentQueryPath, fileName);
}
@Before
public void setup() throws Exception {
this.randomValues = new HashMap<Integer, Integer>();
this.conf = new TajoConf();
util = new TajoTestingCluster();
util.startCatalogCluster();
catalog = util.getMiniCatalogCluster().getCatalog();
Path workDir = CommonTestingUtil.getTestDir();
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString());
catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
sm = StorageManagerFactory.getStorageManager(conf, workDir);
idxPath = new Path(workDir, "test.idx");
Schema schema = new Schema();
schema.addColumn("managerid", Type.INT4);
schema.addColumn("empid", Type.INT4);
schema.addColumn("deptname", Type.TEXT);
this.idxSchema = new Schema();
idxSchema.addColumn("managerid", Type.INT4);
SortSpec[] sortKeys = new SortSpec[1];
sortKeys[0] = new SortSpec(idxSchema.getColumn("managerid"), true, false);
this.comp = new TupleComparator(idxSchema, sortKeys);
this.writer =
new BSTIndex(conf)
.getIndexWriter(idxPath, BSTIndex.TWO_LEVEL_INDEX, this.idxSchema, this.comp);
writer.setLoadNum(100);
writer.open();
long offset;
meta = CatalogUtil.newTableMeta(StoreType.CSV);
tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv");
fs = tablePath.getFileSystem(conf);
fs.mkdirs(tablePath.getParent());
FileAppender appender =
(FileAppender)
StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
Tuple tuple = new VTuple(schema.size());
for (int i = 0; i < 10000; i++) {
Tuple key = new VTuple(this.idxSchema.size());
int rndKey = rnd.nextInt(250);
if (this.randomValues.containsKey(rndKey)) {
int t = this.randomValues.remove(rndKey) + 1;
this.randomValues.put(rndKey, t);
} else {
this.randomValues.put(rndKey, 1);
}
key.put(new Datum[] {DatumFactory.createInt4(rndKey)});
tuple.put(
new Datum[] {
DatumFactory.createInt4(rndKey),
DatumFactory.createInt4(rnd.nextInt(10)),
DatumFactory.createText("dept_" + rnd.nextInt(10))
});
offset = appender.getOffset();
appender.addTuple(tuple);
writer.write(key, offset);
}
appender.flush();
appender.close();
writer.close();
TableDesc desc =
new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"),
schema,
meta,
sm.getTablePath("employee"));
catalog.createTable(desc);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
optimizer = new LogicalOptimizer(conf);
}
@Test
public final void testCreateTableLike1() throws Exception {
// //HiveCatalogStore does not support varchar type in hive-0.12.0
if (testingCluster.isHiveCatalogStoreRunning()) {
// Basic create table with default database
executeString("CREATE TABLE table1 (c1 int, c2 text);").close();
executeString("CREATE TABLE table2 LIKE table1");
String testMsg = "testCreateTableLike1: Basic create table with default db";
assertTrue(testMsg, isClonedTable("table1", "table2"));
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
// Basic create table with database
executeString("CREATE DATABASE d1").close();
executeString("CREATE TABLE d1.table1 (c1 int, c2 text);").close();
executeString("CREATE TABLE d1.table2 LIKE d1.table1");
testMsg = "testCreateTableLike1: Basic create table with db test failed";
assertTrue(testMsg, isClonedTable("d1.table1", "d1.table2"));
executeString("DROP TABLE d1.table1");
executeString("DROP TABLE d1.table2");
// Table with non-default store type
executeString("CREATE TABLE table1 (c1 int, c2 text) USING rcfile;").close();
executeString("CREATE TABLE table2 LIKE table1");
testMsg = "testCreateTableLike1: Table with non-default store type test failed";
assertTrue(testMsg, isClonedTable("table1", "table2"));
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
// Table with non-default meta options
executeString(
"CREATE TABLE table1 (c1 int, c2 text) USING text WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');")
.close();
executeString("CREATE TABLE table2 LIKE table1");
testMsg = "testCreateTableLike1: Table with non-default meta options test failed";
assertTrue(testMsg, isClonedTable("table1", "table2"));
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
// Table with partitions (default partition type)
executeString(
"CREATE TABLE table1 (c1 int, c2 text) PARTITION BY COLUMN (c3 int, c4 float, c5 text);")
.close();
executeString("CREATE TABLE table2 LIKE table1");
testMsg = "testCreateTableLike1: Table with partitions test failed";
assertTrue(testMsg, isClonedTable("table1", "table2"));
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
// Table with external flag
// Use existing file as input for creating external table
String className = getClass().getSimpleName();
Path currentDatasetPath = new Path(datasetBasePath, className);
Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
executeString(
"CREATE EXTERNAL TABLE table3 (c1 int, c2 text) USING rcfile LOCATION '"
+ filePath.toUri()
+ "'")
.close();
executeString("CREATE TABLE table2 LIKE table3");
testMsg = "testCreateTableLike1: Table with external table flag test failed";
assertTrue(testMsg, isClonedTable("table3", "table2"));
executeString("DROP TABLE table3");
executeString("DROP TABLE table2");
// Table created using CTAS
executeString("CREATE TABLE table3 (c1 int, c2 text) PARTITION BY COLUMN (c3 int);").close();
executeString("CREATE TABLE table4 AS SELECT c1 * c1 as m_c1, c2, c2 as c2_a,c3 from table3;")
.close();
executeString("CREATE TABLE table2 LIKE table4");
testMsg = "testCreateTableLike1: Table using CTAS test failed";
assertTrue(testMsg, isClonedTable("table4", "table2"));
executeString("DROP TABLE table3");
executeString("DROP TABLE table4");
executeString("DROP TABLE table2");
} else {
// Basic create table with default database
executeString("CREATE TABLE table1 (c1 int, c2 varchar);").close();
executeString("CREATE TABLE table2 LIKE table1");
String testMsg = "testCreateTableLike1: Basic create table with default db";
assertTrue(testMsg, isClonedTable("table1", "table2"));
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
// Basic create table with database
executeString("CREATE DATABASE d1").close();
executeString("CREATE TABLE d1.table1 (c1 int, c2 varchar);").close();
executeString("CREATE TABLE d1.table2 LIKE d1.table1");
testMsg = "testCreateTableLike1: Basic create table with db test failed";
assertTrue(testMsg, isClonedTable("d1.table1", "d1.table2"));
executeString("DROP TABLE d1.table1");
executeString("DROP TABLE d1.table2");
// Table with non-default store type
executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING rcfile;").close();
executeString("CREATE TABLE table2 LIKE table1");
testMsg = "testCreateTableLike1: Table with non-default store type test failed";
assertTrue(testMsg, isClonedTable("table1", "table2"));
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
// Table with non-default meta options
executeString(
"CREATE TABLE table1 (c1 int, c2 varchar) USING text WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');")
.close();
executeString("CREATE TABLE table2 LIKE table1");
testMsg = "testCreateTableLike1: Table with non-default meta options test failed";
assertTrue(testMsg, isClonedTable("table1", "table2"));
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
// Table with partitions (default partition type)
executeString(
"CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int, c4 float, c5 text);")
.close();
executeString("CREATE TABLE table2 LIKE table1");
testMsg = "testCreateTableLike1: Table with partitions test failed";
assertTrue(testMsg, isClonedTable("table1", "table2"));
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
// Table with external flag
// Use existing file as input for creating external table
String className = getClass().getSimpleName();
Path currentDatasetPath = new Path(datasetBasePath, className);
Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
executeString(
"CREATE EXTERNAL TABLE table3 (c1 int, c2 varchar) USING rcfile LOCATION '"
+ filePath.toUri()
+ "'")
.close();
executeString("CREATE TABLE table2 LIKE table3");
testMsg = "testCreateTableLike1: Table with external table flag test failed";
assertTrue(testMsg, isClonedTable("table3", "table2"));
executeString("DROP TABLE table3");
executeString("DROP TABLE table2");
// Table created using CTAS
executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);")
.close();
executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2 as c2_a,c3 from table3;").close();
executeString("CREATE TABLE table2 LIKE table4");
testMsg = "testCreateTableLike1: Table using CTAS test failed";
assertTrue(testMsg, isClonedTable("table4", "table2"));
executeString("DROP TABLE table3");
executeString("DROP TABLE table4");
executeString("DROP TABLE table2");
/* Enable when view is supported
// View
executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close();
executeString("CREATE VIEW table4(c1,c2,c3) AS SELECT c1*c1, c2, c2,c3 from table3;").close();
executeString("CREATE TABLE table2 LIKE table4");
testMsg = "testCreateTableLike1: Table using VIEW test failed";
assertTrue(testMsg, isClonedTable("table4","table2"));
executeString("DROP TABLE table3");
executeString("DROP TABLE table4");
executeString("DROP TABLE table2");
*/
/* Enable when partition type other than column is supported
// Table with partitions (range partition)
executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY RANGE (c1) ( PARTITION c1 VALUES LESS THAN (2), PARTITION c1 VALUES LESS THAN (5), PARTITION c1 VALUES LESS THAN (MAXVALUE) );").close();
executeString("CREATE TABLE table2 LIKE table1");
testMsg = "testCreateTableLike1: Table using non-default partition type failed";
assertTrue(testMsg, isClonedTable("table1","table2"));
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
*/
}
}
private static Path writeTmpTable(String tableName) throws IOException {
Path tablePath = StorageUtil.concatPath(testDir, tableName);
BackendTestingUtil.writeTmpTable(conf, tablePath);
return tablePath;
}