@Test public void testCreateAndDropTable() throws Exception { catalog.createDatabase("tmpdb1", TajoConstants.DEFAULT_TABLESPACE_NAME); assertTrue(catalog.existDatabase("tmpdb1")); catalog.createDatabase("tmpdb2", TajoConstants.DEFAULT_TABLESPACE_NAME); assertTrue(catalog.existDatabase("tmpdb2")); TableDesc table1 = createMockupTable("tmpdb1", "table1"); catalog.createTable(table1); TableDesc table2 = createMockupTable("tmpdb2", "table2"); catalog.createTable(table2); Set<String> tmpdb1 = Sets.newHashSet(catalog.getAllTableNames("tmpdb1")); assertEquals(1, tmpdb1.size()); assertTrue(tmpdb1.contains("table1")); Set<String> tmpdb2 = Sets.newHashSet(catalog.getAllTableNames("tmpdb2")); assertEquals(1, tmpdb2.size()); assertTrue(tmpdb2.contains("table2")); catalog.dropDatabase("tmpdb1"); assertFalse(catalog.existDatabase("tmpdb1")); tmpdb2 = Sets.newHashSet(catalog.getAllTableNames("tmpdb2")); assertEquals(1, tmpdb2.size()); assertTrue(tmpdb2.contains("table2")); catalog.dropDatabase("tmpdb2"); assertFalse(catalog.existDatabase("tmpdb2")); }
@Test public void testCreateSameTables() throws IOException, TajoException { catalog.createDatabase("tmpdb3", TajoConstants.DEFAULT_TABLESPACE_NAME); assertTrue(catalog.existDatabase("tmpdb3")); catalog.createDatabase("tmpdb4", TajoConstants.DEFAULT_TABLESPACE_NAME); assertTrue(catalog.existDatabase("tmpdb4")); TableDesc table1 = createMockupTable("tmpdb3", "table1"); catalog.createTable(table1); TableDesc table2 = createMockupTable("tmpdb3", "table2"); catalog.createTable(table2); assertTrue(catalog.existsTable("tmpdb3", "table1")); assertTrue(catalog.existsTable("tmpdb3", "table2")); TableDesc table3 = createMockupTable("tmpdb4", "table1"); catalog.createTable(table3); TableDesc table4 = createMockupTable("tmpdb4", "table2"); catalog.createTable(table4); assertTrue(catalog.existsTable("tmpdb4", "table1")); assertTrue(catalog.existsTable("tmpdb4", "table2")); catalog.dropTable("tmpdb3.table1"); catalog.dropTable("tmpdb3.table2"); catalog.dropTable("tmpdb4.table1"); catalog.dropTable("tmpdb4.table2"); assertFalse(catalog.existsTable("tmpdb3.table1")); assertFalse(catalog.existsTable("tmpdb3.table2")); assertFalse(catalog.existsTable("tmpdb4.table1")); assertFalse(catalog.existsTable("tmpdb4.table2")); }
/** It asserts the equality between an original table desc and a restored table desc. */ private static void assertSchemaEquality(String tableName, Schema schema) throws IOException, TajoException { Path path = new Path(CommonTestingUtil.getTestDir(), tableName); TableDesc tableDesc = new TableDesc( IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName), schema, "TEXT", new KeyValueSet(), path.toUri()); // schema creation assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); catalog.createTable(tableDesc); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); // change it for the equals test. schema.setQualifier(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); TableDesc restored = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); assertEquals(schema, restored.getSchema()); // drop test catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); }
@Test public final void testAddAndDeleteTablePartitionByRange() throws Exception { Schema schema = SchemaBuilder.builder() .add("id", Type.INT4) .add("name", Type.TEXT) .add("age", Type.INT4) .add("score", Type.FLOAT8) .build(); String tableName = IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).build(); PartitionMethodDesc partitionDesc = new PartitionMethodDesc( DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.RANGE, "id", partSchema); TableDesc desc = new TableDesc( tableName, schema, meta, new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri()); desc.setPartitionMethod(partitionDesc); assertFalse(catalog.existsTable(tableName)); catalog.createTable(desc); assertTrue(catalog.existsTable(tableName)); TableDesc retrieved = catalog.getTableDesc(tableName); assertEquals(retrieved.getName(), tableName); assertEquals( retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.RANGE); assertEquals( retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id"); catalog.dropTable(tableName); assertFalse(catalog.existsTable(tableName)); }
@Test public void testAlterTable() throws Exception { // CREATE_TABLE TableDesc tableRenameTestDesc = createMockupTable("default", "mycooltable"); catalog.createTable(tableRenameTestDesc); // RENAME_TABLE catalog.alterTable(createMockAlterTableName()); assertTrue(catalog.existsTable("default", "mynewcooltable")); // RENAME_COLUMN catalog.alterTable(createMockAlterTableRenameColumn()); TableDesc columnRenameDesc = catalog.getTableDesc("default", "mynewcooltable"); assertTrue(columnRenameDesc.getSchema().containsByName("ren" + FieldName1)); // ADD_COLUMN catalog.alterTable(createMockAlterTableAddColumn()); TableDesc addColumnDesc = catalog.getTableDesc("default", "mynewcooltable"); assertTrue(addColumnDesc.getSchema().containsByName("mynewcol")); // SET_PROPERTY TableDesc setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable"); KeyValueSet options = new KeyValueSet(); options.set("timezone", "GMT+9"); // Seoul, Korea setPropertyDesc.setMeta(new TableMeta("TEXT", options)); String prevTimeZone = setPropertyDesc.getMeta().getProperty("timezone"); String newTimeZone = "GMT-7"; // Silicon Valley, California catalog.alterTable(createMockAlterTableSetProperty(newTimeZone)); setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable"); assertNotEquals(prevTimeZone, setPropertyDesc.getMeta().getProperty("timezone")); assertEquals(newTimeZone, setPropertyDesc.getMeta().getProperty("timezone")); // UNSET_PROPERTY catalog.alterTable(createMockAlterTableUnsetProperty(Sets.newHashSet("dummy"))); setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable"); assertTrue(setPropertyDesc.getMeta().getPropertySet().containsKey("timezone")); assertFalse(setPropertyDesc.getMeta().getPropertySet().containsKey("dummy")); }
@Test public void testAddAndDelIndex() throws Exception { TableDesc desc = prepareTable(); prepareIndexDescs(); catalog.createTable(desc); assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName())); assertFalse( catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"id"})); catalog.createIndex(desc1); assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName())); assertTrue( catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"id"})); assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName())); assertFalse( catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"score"})); catalog.createIndex(desc2); assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName())); assertTrue( catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"score"})); Set<IndexDesc> indexDescs = new HashSet<>(); indexDescs.add(desc1); indexDescs.add(desc2); indexDescs.add(desc3); for (IndexDesc index : catalog.getAllIndexesByTable(DEFAULT_DATABASE_NAME, "indexed")) { assertTrue(indexDescs.contains(index)); } catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getName()); assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName())); catalog.dropIndex(DEFAULT_DATABASE_NAME, desc2.getName()); assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName())); catalog.dropTable(desc.getName()); assertFalse(catalog.existsTable(desc.getName())); }
@Test public void testGetTable() throws Exception { schema1 = SchemaBuilder.builder() .add(FieldName1, Type.BLOB) .add(FieldName2, Type.INT4) .add(FieldName3, Type.INT8) .build(); Path path = new Path(CommonTestingUtil.getTestDir(), "table1"); TableDesc meta = new TableDesc( IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"), schema1, "TEXT", new KeyValueSet(), path.toUri()); assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); catalog.createTable(meta); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable")); assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); }
private Map<String, List<String>> createBaseDatabaseAndTables() throws IOException, TajoException { Map<String, List<String>> createdDatabaseAndTablesMap = new HashMap<>(); // add and divide all tables to multiple databases in a round robin manner for (int tableId = 0; tableId < TOTAL_TABLE_NUM; tableId++) { int dbIdx = tableId % DB_NUM; String databaseName = dbPrefix + dbIdx; if (!catalog.existDatabase(databaseName)) { catalog.createDatabase(databaseName, TajoConstants.DEFAULT_TABLESPACE_NAME); } String tableName = tablePrefix + tableId; TableDesc table = createMockupTable(databaseName, tableName); catalog.createTable(table); TUtil.putToNestedList(createdDatabaseAndTablesMap, databaseName, tableName); } // checking all tables for each database for (int dbIdx = 0; dbIdx < DB_NUM; dbIdx++) { String databaseName = dbPrefix + dbIdx; Collection<String> tableNames = catalog.getAllTableNames(databaseName); assertTrue(createdDatabaseAndTablesMap.containsKey(databaseName)); assertEquals(createdDatabaseAndTablesMap.get(databaseName).size(), tableNames.size()); for (String tableName : tableNames) { assertTrue(createdDatabaseAndTablesMap.get(databaseName).contains(tableName)); } } return createdDatabaseAndTablesMap; }
// TODO: This should be added at TAJO-1891 public final void testAddAndDeleteTablePartitionByColumn() throws Exception { Schema schema = SchemaBuilder.builder() .add("id", Type.INT4) .add("name", Type.TEXT) .add("age", Type.INT4) .add("score", Type.FLOAT8) .build(); String simpleTableName = "addedtable"; String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).add("name", Type.TEXT).build(); PartitionMethodDesc partitionMethodDesc = new PartitionMethodDesc( DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.COLUMN, "id,name", partSchema); TableDesc desc = new TableDesc( tableName, schema, meta, new Path(CommonTestingUtil.getTestDir(), simpleTableName).toUri()); desc.setPartitionMethod(partitionMethodDesc); assertFalse(catalog.existsTable(tableName)); catalog.createTable(desc); assertTrue(catalog.existsTable(tableName)); TableDesc retrieved = catalog.getTableDesc(tableName); assertEquals(retrieved.getName(), tableName); assertEquals( retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN); assertEquals( retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id"); testAddPartition(tableName, "id=10/name=aaa"); testAddPartition(tableName, "id=20/name=bbb"); List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName); assertNotNull(partitions); assertEquals(partitions.size(), 2); assertEquals(partitions.get(0).getNumBytes(), 0L); testGetPartitionsByAlgebra(DEFAULT_DATABASE_NAME, simpleTableName); testDropPartition(tableName, "id=10/name=aaa"); testDropPartition(tableName, "id=20/name=bbb"); partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName); assertNotNull(partitions); assertEquals(partitions.size(), 0); catalog.dropTable(tableName); assertFalse(catalog.existsTable(tableName)); }
@Before public void setUp() throws Exception { util = new TajoTestingCluster(); util.initTestDir(); util.startCatalogCluster(); catalog = util.getCatalogService(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); // ----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id // -------------------------------- // 0 | dep_0 | 1000 // 1 | dep_1 | 1001 // 2 | dep_2 | 1002 // 3 | dep_3 | 1003 // 4 | dep_4 | 1004 // 5 | dep_5 | 1005 // 6 | dep_6 | 1006 // 7 | dep_7 | 1007 // 8 | dep_8 | 1008 // 9 | dep_9 | 1009 Schema dep3Schema = new Schema(); dep3Schema.addColumn("dep_id", Type.INT4); dep3Schema.addColumn("dep_name", Type.TEXT); dep3Schema.addColumn("loc_id", Type.INT4); TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); VTuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { tuple.put( new Datum[] { DatumFactory.createInt4(i), DatumFactory.createText("dept_" + i), DatumFactory.createInt4(1000 + i) }); appender1.addTuple(tuple); } appender1.flush(); appender1.close(); dep3 = CatalogUtil.newTableDesc(DEP3_NAME, dep3Schema, dep3Meta, dep3Path); catalog.createTable(dep3); // ----------------- dep4 ------------------------------ // dep_id | dep_name | loc_id // -------------------------------- // 0 | dep_0 | 1000 // 1 | dep_1 | 1001 // 2 | dep_2 | 1002 // 3 | dep_3 | 1003 // 4 | dep_4 | 1004 // 5 | dep_5 | 1005 // 6 | dep_6 | 1006 // 7 | dep_7 | 1007 // 8 | dep_8 | 1008 // 9 | dep_9 | 1009 // 10 | dep_10 | 1010 Schema dep4Schema = new Schema(); dep4Schema.addColumn("dep_id", Type.INT4); dep4Schema.addColumn("dep_name", Type.TEXT); dep4Schema.addColumn("loc_id", Type.INT4); TableMeta dep4Meta = CatalogUtil.newTableMeta("TEXT"); Path dep4Path = new Path(testDir, "dep4.csv"); Appender appender4 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); VTuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { tuple4.put( new Datum[] { DatumFactory.createInt4(i), DatumFactory.createText("dept_" + i), DatumFactory.createInt4(1000 + i) }); appender4.addTuple(tuple4); } appender4.flush(); appender4.close(); dep4 = CatalogUtil.newTableDesc(DEP4_NAME, dep4Schema, dep4Meta, dep4Path); catalog.createTable(dep4); // ----------------- job3 ------------------------------ // job_id | job_title // ---------------------- // 101 | job_101 // 102 | job_102 // 103 | job_103 Schema job3Schema = new Schema(); job3Schema.addColumn("job_id", Type.INT4); job3Schema.addColumn("job_title", Type.TEXT); TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); VTuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { int x = 100 + i; tuple2.put( new Datum[] {DatumFactory.createInt4(100 + i), DatumFactory.createText("job_" + x)}); appender2.addTuple(tuple2); } appender2.flush(); appender2.close(); job3 = CatalogUtil.newTableDesc(JOB3_NAME, job3Schema, job3Meta, job3Path); catalog.createTable(job3); // ---------------------emp3 -------------------- // emp_id | first_name | last_name | dep_id | salary | job_id // ------------------------------------------------------------ // 11 | fn_11 | ln_11 | 1 | 123 | 101 // 13 | fn_13 | ln_13 | 3 | 369 | 103 // 15 | fn_15 | ln_15 | 5 | 615 | null // 17 | fn_17 | ln_17 | 7 | 861 | null // 19 | fn_19 | ln_19 | 9 | 1107 | null // 21 | fn_21 | ln_21 | 1 | 123 | 101 // 23 | fn_23 | ln_23 | 3 | 369 | 103 Schema emp3Schema = new Schema(); emp3Schema.addColumn("emp_id", Type.INT4); emp3Schema.addColumn("first_name", Type.TEXT); emp3Schema.addColumn("last_name", Type.TEXT); emp3Schema.addColumn("dep_id", Type.INT4); emp3Schema.addColumn("salary", Type.FLOAT4); emp3Schema.addColumn("job_id", Type.INT4); TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); VTuple tuple3 = new VTuple(emp3Schema.size()); for (int i = 1; i < 4; i += 2) { int x = 10 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(10 + i), DatumFactory.createText("firstname_" + x), DatumFactory.createText("lastname_" + x), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createInt4(100 + i) }); appender3.addTuple(tuple3); int y = 20 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(20 + i), DatumFactory.createText("firstname_" + y), DatumFactory.createText("lastname_" + y), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createInt4(100 + i) }); appender3.addTuple(tuple3); } for (int i = 5; i < 10; i += 2) { int x = 10 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(10 + i), DatumFactory.createText("firstname_" + x), DatumFactory.createText("lastname_" + x), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createNullDatum() }); appender3.addTuple(tuple3); } appender3.flush(); appender3.close(); emp3 = CatalogUtil.newTableDesc(EMP3_NAME, emp3Schema, emp3Meta, emp3Path); catalog.createTable(emp3); // ---------------------phone3 -------------------- // emp_id | phone_number // ----------------------------------------------- // this table is empty, no rows Schema phone3Schema = new Schema(); phone3Schema.addColumn("emp_id", Type.INT4); phone3Schema.addColumn("phone_number", Type.TEXT); TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT"); Path phone3Path = new Path(testDir, "phone3.csv"); Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); appender5.close(); phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path); catalog.createTable(phone3); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); defaultContext = LocalTajoTestingUtility.createDummyContext(conf); }