/** It asserts the equality between an original table desc and a restored table desc. */
private static void assertSchemaEquality(String tableName, Schema schema)
throws IOException, TajoException {
Path path = new Path(CommonTestingUtil.getTestDir(), tableName);
TableDesc tableDesc =
new TableDesc(
IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName),
schema,
"TEXT",
new KeyValueSet(),
path.toUri());
// schema creation
assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
catalog.createTable(tableDesc);
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
// change it for the equals test.
schema.setQualifier(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
TableDesc restored = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
assertEquals(schema, restored.getSchema());
// drop test
catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
}
@Test
public final void testSelfDescTable1() throws Exception {
executeString("create database d9;").close();
assertTableNotExists("d9.schemaless");
executeQuery();
assertTableExists("d9.schemaless");
TableDesc desc = getClient().getTableDesc("d9.schemaless");
assertTrue(desc.hasEmptySchema());
executeString("drop table d9.schemaless").close();
executeString("drop database d9").close();
}
@Test
public final void testSelfDescTable2() throws Exception {
executeString("create database d10;").close();
String className = getClass().getSimpleName();
Path currentDatasetPath = new Path(datasetBasePath, className);
Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
String sql =
"create external table d10.schemaless (*) using json with ('compression.codec'='none') location '"
+ filePath.toString()
+ "'";
executeString(sql).close();
assertTableExists("d10.schemaless");
TableDesc desc = getClient().getTableDesc("d10.schemaless");
assertTrue(desc.hasEmptySchema());
executeString("drop table d10.schemaless").close();
executeString("drop database d10").close();
}
@Test
public final void testRightOuter_MergeJoin3() throws IOException, TajoException {
Expr expr = analyzer.parse(QUERIES[3]);
LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
FileTablespace.splitNG(
conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getUri()), Integer.MAX_VALUE);
FileFragment[] dep4Frags =
FileTablespace.splitNG(
conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getUri()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags);
Path workDir =
CommonTestingUtil.getTestDir(
TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testRightOuter_MergeJoin3");
TaskAttemptContext ctx =
new TaskAttemptContext(
new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
ProjectionExec proj = (ProjectionExec) exec;
assertTrue(proj.getChild() instanceof RightOuterMergeJoinExec);
int count = 0;
exec.init();
while (exec.next() != null) {
// TODO check contents
count = count + 1;
}
assertNull(exec.next());
exec.close();
assertEquals(13, count);
}
@Test
public void testAddAndDelIndex() throws Exception {
TableDesc desc = prepareTable();
prepareIndexDescs();
catalog.createTable(desc);
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
assertFalse(
catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"id"}));
catalog.createIndex(desc1);
assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
assertTrue(
catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"id"}));
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
assertFalse(
catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"score"}));
catalog.createIndex(desc2);
assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
assertTrue(
catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"score"}));
Set<IndexDesc> indexDescs = new HashSet<>();
indexDescs.add(desc1);
indexDescs.add(desc2);
indexDescs.add(desc3);
for (IndexDesc index : catalog.getAllIndexesByTable(DEFAULT_DATABASE_NAME, "indexed")) {
assertTrue(indexDescs.contains(index));
}
catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getName());
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
catalog.dropIndex(DEFAULT_DATABASE_NAME, desc2.getName());
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
catalog.dropTable(desc.getName());
assertFalse(catalog.existsTable(desc.getName()));
}
@Test
public final void testAddAndDeleteTablePartitionByRange() throws Exception {
Schema schema =
SchemaBuilder.builder()
.add("id", Type.INT4)
.add("name", Type.TEXT)
.add("age", Type.INT4)
.add("score", Type.FLOAT8)
.build();
String tableName =
IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).build();
PartitionMethodDesc partitionDesc =
new PartitionMethodDesc(
DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.RANGE, "id", partSchema);
TableDesc desc =
new TableDesc(
tableName,
schema,
meta,
new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
assertTrue(catalog.existsTable(tableName));
TableDesc retrieved = catalog.getTableDesc(tableName);
assertEquals(retrieved.getName(), tableName);
assertEquals(
retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.RANGE);
assertEquals(
retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");
catalog.dropTable(tableName);
assertFalse(catalog.existsTable(tableName));
}
private final void assertPathOfCreatedTable(
final String databaseName,
final String originalTableName,
final String newTableName,
String createTableStmt)
throws Exception {
// create one table
executeString("CREATE DATABASE " + CatalogUtil.denormalizeIdentifier(databaseName)).close();
getClient().existDatabase(CatalogUtil.denormalizeIdentifier(databaseName));
final String oldFQTableName = CatalogUtil.buildFQName(databaseName, originalTableName);
ResultSet res = executeString(createTableStmt);
res.close();
assertTableExists(oldFQTableName);
TableDesc oldTableDesc = client.getTableDesc(oldFQTableName);
// checking the existence of the table directory and validating the path
Path warehouseDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration());
FileSystem fs = warehouseDir.getFileSystem(testingCluster.getConfiguration());
assertTrue(fs.exists(new Path(oldTableDesc.getUri())));
assertEquals(
StorageUtil.concatPath(warehouseDir, databaseName, originalTableName),
new Path(oldTableDesc.getUri()));
// Rename
client.executeQuery(
"ALTER TABLE "
+ CatalogUtil.denormalizeIdentifier(oldFQTableName)
+ " RENAME to "
+ CatalogUtil.denormalizeIdentifier(newTableName));
// checking the existence of the new table directory and validating the path
final String newFQTableName = CatalogUtil.buildFQName(databaseName, newTableName);
TableDesc newTableDesc = client.getTableDesc(newFQTableName);
assertTrue(fs.exists(new Path(newTableDesc.getUri())));
assertEquals(
StorageUtil.concatPath(warehouseDir, databaseName, newTableName),
new Path(newTableDesc.getUri()));
}
// TODO: This should be added at TAJO-1891
public final void testAddAndDeleteTablePartitionByColumn() throws Exception {
Schema schema =
SchemaBuilder.builder()
.add("id", Type.INT4)
.add("name", Type.TEXT)
.add("age", Type.INT4)
.add("score", Type.FLOAT8)
.build();
String simpleTableName = "addedtable";
String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName);
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).add("name", Type.TEXT).build();
PartitionMethodDesc partitionMethodDesc =
new PartitionMethodDesc(
DEFAULT_DATABASE_NAME,
tableName,
CatalogProtos.PartitionType.COLUMN,
"id,name",
partSchema);
TableDesc desc =
new TableDesc(
tableName,
schema,
meta,
new Path(CommonTestingUtil.getTestDir(), simpleTableName).toUri());
desc.setPartitionMethod(partitionMethodDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
assertTrue(catalog.existsTable(tableName));
TableDesc retrieved = catalog.getTableDesc(tableName);
assertEquals(retrieved.getName(), tableName);
assertEquals(
retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN);
assertEquals(
retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");
testAddPartition(tableName, "id=10/name=aaa");
testAddPartition(tableName, "id=20/name=bbb");
List<CatalogProtos.PartitionDescProto> partitions =
catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
assertNotNull(partitions);
assertEquals(partitions.size(), 2);
assertEquals(partitions.get(0).getNumBytes(), 0L);
testGetPartitionsByAlgebra(DEFAULT_DATABASE_NAME, simpleTableName);
testDropPartition(tableName, "id=10/name=aaa");
testDropPartition(tableName, "id=20/name=bbb");
partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
assertNotNull(partitions);
assertEquals(partitions.size(), 0);
catalog.dropTable(tableName);
assertFalse(catalog.existsTable(tableName));
}
@Test
public void testAlterTable() throws Exception {
// CREATE_TABLE
TableDesc tableRenameTestDesc = createMockupTable("default", "mycooltable");
catalog.createTable(tableRenameTestDesc);
// RENAME_TABLE
catalog.alterTable(createMockAlterTableName());
assertTrue(catalog.existsTable("default", "mynewcooltable"));
// RENAME_COLUMN
catalog.alterTable(createMockAlterTableRenameColumn());
TableDesc columnRenameDesc = catalog.getTableDesc("default", "mynewcooltable");
assertTrue(columnRenameDesc.getSchema().containsByName("ren" + FieldName1));
// ADD_COLUMN
catalog.alterTable(createMockAlterTableAddColumn());
TableDesc addColumnDesc = catalog.getTableDesc("default", "mynewcooltable");
assertTrue(addColumnDesc.getSchema().containsByName("mynewcol"));
// SET_PROPERTY
TableDesc setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
KeyValueSet options = new KeyValueSet();
options.set("timezone", "GMT+9"); // Seoul, Korea
setPropertyDesc.setMeta(new TableMeta("TEXT", options));
String prevTimeZone = setPropertyDesc.getMeta().getProperty("timezone");
String newTimeZone = "GMT-7"; // Silicon Valley, California
catalog.alterTable(createMockAlterTableSetProperty(newTimeZone));
setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
assertNotEquals(prevTimeZone, setPropertyDesc.getMeta().getProperty("timezone"));
assertEquals(newTimeZone, setPropertyDesc.getMeta().getProperty("timezone"));
// UNSET_PROPERTY
catalog.alterTable(createMockAlterTableUnsetProperty(Sets.newHashSet("dummy")));
setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
assertTrue(setPropertyDesc.getMeta().getPropertySet().containsKey("timezone"));
assertFalse(setPropertyDesc.getMeta().getPropertySet().containsKey("dummy"));
}
private boolean isClonedTable(String orignalTable, String newTable) throws Exception {
assertTableExists(newTable);
TableDesc origTableDesc = client.getTableDesc(orignalTable);
TableDesc newTableDesc = client.getTableDesc(newTable);
if (isClonedSchema(origTableDesc.getSchema(), newTableDesc.getSchema()) == false) {
fail("Schema of input tables do not match");
return false;
}
// Check partition information
PartitionMethodDesc origPartMethod = origTableDesc.getPartitionMethod();
PartitionMethodDesc newPartMethod = newTableDesc.getPartitionMethod();
if (origPartMethod != null) {
if (newPartMethod == null) {
fail("New table does not have partition info");
return false;
}
if (isClonedSchema(origPartMethod.getExpressionSchema(), newPartMethod.getExpressionSchema())
== false) {
fail("Partition columns of input tables do not match");
return false;
}
if (origPartMethod.getPartitionType().equals(newPartMethod.getPartitionType()) == false) {
fail("Partition type of input tables do not match");
return false;
}
}
// Check external flag
if (origTableDesc.isExternal() != newTableDesc.isExternal()) {
fail("External table flag on input tables not equal");
return false;
}
if (origTableDesc.getMeta() != null) {
TableMeta origMeta = origTableDesc.getMeta();
TableMeta newMeta = newTableDesc.getMeta();
if (origMeta.getDataFormat().equals(newMeta.getDataFormat()) == false) {
fail("Store type of input tables not equal");
return false;
}
KeyValueSet origOptions = origMeta.getPropertySet();
KeyValueSet newOptions = newMeta.getPropertySet();
if (origOptions.equals(newOptions) == false) {
fail("Meta options of input tables not equal");
return false;
}
}
return true;
}