@Test public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOException, ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testFailCreateTablePartitionedOtherExceptColumn"; assertFalse(client.existTable(tableName)); String rangeSql = "create table " + tableName + " (deptname text, score int4)"; rangeSql += "PARTITION BY RANGE (score)"; rangeSql += "( PARTITION sub_part1 VALUES LESS THAN (2),"; rangeSql += "PARTITION sub_part2 VALUES LESS THAN (5),"; rangeSql += "PARTITION sub_part2 VALUES LESS THAN (MAXVALUE) )"; assertFalse(client.updateQuery(rangeSql)); String listSql = "create table " + tableName + " (deptname text, score int4)"; listSql += "PARTITION BY LIST (deptname)"; listSql += "( PARTITION sub_part1 VALUES('r&d', 'design'),"; listSql += "PARTITION sub_part2 VALUES('sales', 'hr') )"; assertFalse(client.updateQuery(listSql)); String hashSql = "create table " + tableName + " (deptname text, score int4)"; hashSql += "PARTITION BY HASH (deptname)"; hashSql += "PARTITIONS 2"; assertFalse(client.updateQuery(hashSql)); }
@BeforeClass public static void setUp() throws Exception { cluster = TpchTestBase.getInstance().getTestingCluster(); conf = cluster.getConfiguration(); client = new TajoClient(conf); testDir = CommonTestingUtil.getTestDir(); }
@Test public final void testCreateAndPurgeExternalTableByExecuteQuery() throws IOException, ServiceException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndPurgeExternalTableByExecuteQuery"; Path tablePath = writeTmpTable(tableName); assertFalse(client.existTable(tableName)); String sql = "create external table " + tableName + " (deptname text, score int4) " + "using csv location '" + tablePath + "'"; client.executeQueryAndGetResult(sql); assertTrue(client.existTable(tableName)); client.updateQuery("drop table " + tableName + " purge"); assertFalse(client.existTable(tableName)); FileSystem localFS = FileSystem.getLocal(conf); assertFalse(localFS.exists(tablePath)); }
@BeforeClass public static void setUp() throws Exception { util = new TajoTestingCluster(); util.startCatalogCluster(); catalog = util.getMiniCatalogCluster().getCatalog(); Schema schema = new Schema(); schema.addColumn("name", Type.TEXT); schema.addColumn("empId", CatalogUtil.newSimpleDataType(Type.INT4)); schema.addColumn("deptName", Type.TEXT); Schema schema2 = new Schema(); schema2.addColumn("deptName", Type.TEXT); schema2.addColumn("manager", Type.TEXT); Schema schema3 = new Schema(); schema3.addColumn("deptName", Type.TEXT); schema3.addColumn("score", CatalogUtil.newSimpleDataType(Type.INT4)); TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); TableDesc people = new TableDesc("employee", schema, meta, CommonTestingUtil.getTestDir()); catalog.addTable(people); TableDesc student = new TableDesc( "dept", schema2, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir()); catalog.addTable(student); TableDesc score = new TableDesc( "score", schema3, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir()); catalog.addTable(score); FunctionDesc funcDesc = new FunctionDesc( "sumtest", SumInt.class, FunctionType.AGGREGATION, CatalogUtil.newSimpleDataType(Type.INT4), CatalogUtil.newSimpleDataTypeArray(Type.INT4)); catalog.createFunction(funcDesc); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); }
@Test public final void testCtasWithColumnedPartition() throws Exception { ResultSet res = executeQuery(); res.close(); String tableName = CatalogUtil.normalizeIdentifier("testCtasWithColumnedPartition"); TajoTestingCluster cluster = testBase.getTestingCluster(); CatalogService catalog = cluster.getMaster().getCatalog(); TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); PartitionMethodDesc partitionDesc = desc.getPartitionMethod(); assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN); assertEquals( "key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName()); FileSystem fs = FileSystem.get(cluster.getConfiguration()); Path path = new Path(desc.getUri()); assertTrue(fs.isDirectory(path)); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0"))); if (!cluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } ResultSet res2 = executeFile("check2.sql"); Map<Double, int[]> resultRows1 = Maps.newHashMap(); resultRows1.put(45.0d, new int[] {3, 2}); resultRows1.put(38.0d, new int[] {2, 2}); int i = 0; while (res2.next()) { assertEquals(resultRows1.get(res2.getDouble(3))[0], res2.getInt(1)); assertEquals(resultRows1.get(res2.getDouble(3))[1], res2.getInt(2)); i++; } res2.close(); assertEquals(2, i); }
@Test public void testDescTable() throws Exception { String tableName; if (cluster.isHiveCatalogStoreRunning()) { tableName = "TEST_DESC_TABLE".toLowerCase(); } else { tableName = "TEST_DESC_TABLE"; } String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4);"; verifyDescTable(sql, tableName, "testDescTable.result"); }
@Test public void testDescTableForNestedSchema() throws Exception { String tableName; if (cluster.isHiveCatalogStoreRunning()) { tableName = "TEST_DESC_TABLE_NESTED".toLowerCase(); } else { tableName = "TEST_DESC_TABLE_NESTED"; } String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4, col3 record (col4 record (col5 text)));"; verifyDescTable(sql, tableName, "testDescTableForNestedSchema.result"); }
@Test(timeout = 3000) public void testNonForwardQueryPause() throws Exception { final String sql = "select * from default.lineitem"; TajoCli cli = null; try { TableDesc tableDesc = cluster.getMaster().getCatalog().getTableDesc("default", "lineitem"); assertNotNull(tableDesc); assertEquals(0L, tableDesc.getStats().getNumRows().longValue()); InputStream testInput = new ByteArrayInputStream(new byte[] {(byte) DefaultTajoCliOutputFormatter.QUIT_COMMAND}); cli = new TajoCli(cluster.getConfiguration(), new String[] {}, testInput, out); setVar(cli, SessionVars.CLI_PAGE_ROWS, "2"); setVar(cli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); cli.executeScript(sql); String consoleResult; consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); } finally { cli.close(); } }
@Before public void setUp() throws Exception { this.conf = new TajoConf(); util = new TajoTestingCluster(); catalog = util.startCatalogCluster().getCatalog(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); sm = StorageManagerFactory.getStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerId", Type.INT4); schema.addColumn("empId", Type.INT4); schema.addColumn("deptName", Type.TEXT); TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = StorageManagerFactory.getStorageManager(conf) .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.getColumnNum()); for (int i = 0; i < numTuple; i++) { tuple.put( new Datum[] { DatumFactory.createInt4(rnd.nextInt(50)), DatumFactory.createInt4(rnd.nextInt(100)), DatumFactory.createText("dept_" + i), }); appender.addTuple(tuple); } appender.flush(); appender.close(); System.out.println( appender.getStats().getNumRows() + " rows (" + (appender.getStats().getNumBytes() / 1048576) + " MB)"); employee = new TableDesc("employee", schema, employeeMeta, employeePath); catalog.addTable(employee); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); }
@Test public final void testGetFunctions() throws IOException, ServiceException, SQLException { Collection<FunctionDesc> catalogFunctions = cluster.getMaster().getCatalog().getFunctions(); String functionName = "sum"; int numFunctions = 0; for (FunctionDesc eachFunction : catalogFunctions) { if (functionName.equals(eachFunction.getSignature())) { numFunctions++; } } List<CatalogProtos.FunctionDescProto> functions = client.getFunctions(functionName); assertEquals(numFunctions, functions.size()); functions = client.getFunctions("notmatched"); assertEquals(0, functions.size()); functions = client.getFunctions(null); assertEquals(catalogFunctions.size(), functions.size()); }
private void verifyDescTable(String sql, String tableName, String resultFileName) throws Exception { setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); tajoCli.executeScript(sql); tajoCli.executeMetaCommand("\\d " + tableName); tajoCli.executeMetaCommand("\\d \"" + tableName + "\""); String consoleResult = new String(out.toByteArray()); if (!cluster.isHiveCatalogStoreRunning()) { assertOutputResult( resultFileName, consoleResult, new String[] {"${table.path}"}, new String[] { TablespaceManager.getDefault().getTableUri("default", tableName).toString() }); } }
@Test public final void testCreateAndPurgeTableByExecuteQuery() throws IOException, ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndPurgeTableByExecuteQuery"; assertFalse(client.existTable(tableName)); String sql = "create table " + tableName + " (deptname text, score int4)"; client.updateQuery(sql); assertTrue(client.existTable(tableName)); Path tablePath = client.getTableDesc(tableName).getPath(); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); client.updateQuery("drop table " + tableName + " purge"); assertFalse(client.existTable(tableName)); assertFalse(hdfs.exists(tablePath)); }
// @Test public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery() throws IOException, ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndDropTablePartitionedHash1ByExecuteQuery"; assertFalse(client.existTable(tableName)); String sql = "create table " + tableName + " (deptname text, score int4)"; sql += " PARTITION BY HASH (deptname)"; sql += " (PARTITION sub_part1, PARTITION sub_part2, PARTITION sub_part3)"; client.updateQuery(sql); assertTrue(client.existTable(tableName)); Path tablePath = client.getTableDesc(tableName).getPath(); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); client.updateQuery("drop table " + tableName); assertFalse(client.existTable(tableName)); assertTrue(hdfs.exists(tablePath)); }
@Test public void testConnectDatabase() throws Exception { String databaseName; if (cluster.isHiveCatalogStoreRunning()) { databaseName = "TEST_CONNECTION_DATABASE".toLowerCase(); } else { databaseName = "TEST_CONNECTION_DATABASE"; } String sql = "create database \"" + databaseName + "\";"; tajoCli.executeScript(sql); tajoCli.executeMetaCommand("\\c " + databaseName); assertEquals(databaseName, tajoCli.getContext().getCurrentDatabase()); tajoCli.executeMetaCommand("\\c default"); assertEquals("default", tajoCli.getContext().getCurrentDatabase()); tajoCli.executeMetaCommand("\\c \"" + databaseName + "\""); assertEquals(databaseName, tajoCli.getContext().getCurrentDatabase()); }
@Before public void setUp() throws Exception { util = new TajoTestingCluster(); util.initTestDir(); util.startCatalogCluster(); catalog = util.getCatalogService(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); // ----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id // -------------------------------- // 0 | dep_0 | 1000 // 1 | dep_1 | 1001 // 2 | dep_2 | 1002 // 3 | dep_3 | 1003 // 4 | dep_4 | 1004 // 5 | dep_5 | 1005 // 6 | dep_6 | 1006 // 7 | dep_7 | 1007 // 8 | dep_8 | 1008 // 9 | dep_9 | 1009 Schema dep3Schema = new Schema(); dep3Schema.addColumn("dep_id", Type.INT4); dep3Schema.addColumn("dep_name", Type.TEXT); dep3Schema.addColumn("loc_id", Type.INT4); TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); VTuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { tuple.put( new Datum[] { DatumFactory.createInt4(i), DatumFactory.createText("dept_" + i), DatumFactory.createInt4(1000 + i) }); appender1.addTuple(tuple); } appender1.flush(); appender1.close(); dep3 = CatalogUtil.newTableDesc(DEP3_NAME, dep3Schema, dep3Meta, dep3Path); catalog.createTable(dep3); // ----------------- dep4 ------------------------------ // dep_id | dep_name | loc_id // -------------------------------- // 0 | dep_0 | 1000 // 1 | dep_1 | 1001 // 2 | dep_2 | 1002 // 3 | dep_3 | 1003 // 4 | dep_4 | 1004 // 5 | dep_5 | 1005 // 6 | dep_6 | 1006 // 7 | dep_7 | 1007 // 8 | dep_8 | 1008 // 9 | dep_9 | 1009 // 10 | dep_10 | 1010 Schema dep4Schema = new Schema(); dep4Schema.addColumn("dep_id", Type.INT4); dep4Schema.addColumn("dep_name", Type.TEXT); dep4Schema.addColumn("loc_id", Type.INT4); TableMeta dep4Meta = CatalogUtil.newTableMeta("TEXT"); Path dep4Path = new Path(testDir, "dep4.csv"); Appender appender4 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); VTuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { tuple4.put( new Datum[] { DatumFactory.createInt4(i), DatumFactory.createText("dept_" + i), DatumFactory.createInt4(1000 + i) }); appender4.addTuple(tuple4); } appender4.flush(); appender4.close(); dep4 = CatalogUtil.newTableDesc(DEP4_NAME, dep4Schema, dep4Meta, dep4Path); catalog.createTable(dep4); // ----------------- job3 ------------------------------ // job_id | job_title // ---------------------- // 101 | job_101 // 102 | job_102 // 103 | job_103 Schema job3Schema = new Schema(); job3Schema.addColumn("job_id", Type.INT4); job3Schema.addColumn("job_title", Type.TEXT); TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); VTuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { int x = 100 + i; tuple2.put( new Datum[] {DatumFactory.createInt4(100 + i), DatumFactory.createText("job_" + x)}); appender2.addTuple(tuple2); } appender2.flush(); appender2.close(); job3 = CatalogUtil.newTableDesc(JOB3_NAME, job3Schema, job3Meta, job3Path); catalog.createTable(job3); // ---------------------emp3 -------------------- // emp_id | first_name | last_name | dep_id | salary | job_id // ------------------------------------------------------------ // 11 | fn_11 | ln_11 | 1 | 123 | 101 // 13 | fn_13 | ln_13 | 3 | 369 | 103 // 15 | fn_15 | ln_15 | 5 | 615 | null // 17 | fn_17 | ln_17 | 7 | 861 | null // 19 | fn_19 | ln_19 | 9 | 1107 | null // 21 | fn_21 | ln_21 | 1 | 123 | 101 // 23 | fn_23 | ln_23 | 3 | 369 | 103 Schema emp3Schema = new Schema(); emp3Schema.addColumn("emp_id", Type.INT4); emp3Schema.addColumn("first_name", Type.TEXT); emp3Schema.addColumn("last_name", Type.TEXT); emp3Schema.addColumn("dep_id", Type.INT4); emp3Schema.addColumn("salary", Type.FLOAT4); emp3Schema.addColumn("job_id", Type.INT4); TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); VTuple tuple3 = new VTuple(emp3Schema.size()); for (int i = 1; i < 4; i += 2) { int x = 10 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(10 + i), DatumFactory.createText("firstname_" + x), DatumFactory.createText("lastname_" + x), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createInt4(100 + i) }); appender3.addTuple(tuple3); int y = 20 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(20 + i), DatumFactory.createText("firstname_" + y), DatumFactory.createText("lastname_" + y), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createInt4(100 + i) }); appender3.addTuple(tuple3); } for (int i = 5; i < 10; i += 2) { int x = 10 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(10 + i), DatumFactory.createText("firstname_" + x), DatumFactory.createText("lastname_" + x), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createNullDatum() }); appender3.addTuple(tuple3); } appender3.flush(); appender3.close(); emp3 = CatalogUtil.newTableDesc(EMP3_NAME, emp3Schema, emp3Meta, emp3Path); catalog.createTable(emp3); // ---------------------phone3 -------------------- // emp_id | phone_number // ----------------------------------------------- // this table is empty, no rows Schema phone3Schema = new Schema(); phone3Schema.addColumn("emp_id", Type.INT4); phone3Schema.addColumn("phone_number", Type.TEXT); TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT"); Path phone3Path = new Path(testDir, "phone3.csv"); Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); appender5.close(); phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path); catalog.createTable(phone3); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); defaultContext = LocalTajoTestingUtility.createDummyContext(conf); }
@After public void tearDown() throws Exception { util.shutdownCatalogCluster(); }
@Before public void setup() throws Exception { this.randomValues = new HashMap<Integer, Integer>(); this.conf = new TajoConf(); util = new TajoTestingCluster(); util.startCatalogCluster(); catalog = util.getMiniCatalogCluster().getCatalog(); Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); sm = StorageManagerFactory.getStorageManager(conf, workDir); idxPath = new Path(workDir, "test.idx"); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); schema.addColumn("empid", Type.INT4); schema.addColumn("deptname", Type.TEXT); this.idxSchema = new Schema(); idxSchema.addColumn("managerid", Type.INT4); SortSpec[] sortKeys = new SortSpec[1]; sortKeys[0] = new SortSpec(idxSchema.getColumn("managerid"), true, false); this.comp = new TupleComparator(idxSchema, sortKeys); this.writer = new BSTIndex(conf) .getIndexWriter(idxPath, BSTIndex.TWO_LEVEL_INDEX, this.idxSchema, this.comp); writer.setLoadNum(100); writer.open(); long offset; meta = CatalogUtil.newTableMeta(StoreType.CSV); tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv"); fs = tablePath.getFileSystem(conf); fs.mkdirs(tablePath.getParent()); FileAppender appender = (FileAppender) StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 10000; i++) { Tuple key = new VTuple(this.idxSchema.size()); int rndKey = rnd.nextInt(250); if (this.randomValues.containsKey(rndKey)) { int t = this.randomValues.remove(rndKey) + 1; this.randomValues.put(rndKey, t); } else { this.randomValues.put(rndKey, 1); } key.put(new Datum[] {DatumFactory.createInt4(rndKey)}); tuple.put( new Datum[] { DatumFactory.createInt4(rndKey), DatumFactory.createInt4(rnd.nextInt(10)), DatumFactory.createText("dept_" + rnd.nextInt(10)) }); offset = appender.getOffset(); appender.addTuple(tuple); writer.write(key, offset); } appender.flush(); appender.close(); writer.close(); TableDesc desc = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, sm.getTablePath("employee")); catalog.createTable(desc); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); optimizer = new LogicalOptimizer(conf); }
@After public void tearDown() { util.shutdownCatalogCluster(); }
@Before public void setUp() throws Exception { out = new ByteArrayOutputStream(); tajoCli = new TajoCli(cluster.getConfiguration(), new String[] {}, System.in, out); }
@After public void tearDown() throws Exception { CommonTestingUtil.cleanupTestDir(TEST_PATH); util.shutdownCatalogCluster(); }