@Test public void testEqual() throws Exception { if (conf.getBoolean("tajo.storage.manager.v2", false)) { return; } this.rndKey = rnd.nextInt(250); final String QUERY = "select * from employee where managerId = " + rndKey; FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEqual"); TaskAttemptContext ctx = new TaskAttemptContext( conf, LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] {frags[0]}, workDir); Expr expr = analyzer.parse(QUERY); LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); LogicalNode rootNode = optimizer.optimize(plan); TmpPlanner phyPlanner = new TmpPlanner(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); int tupleCount = this.randomValues.get(rndKey); int counter = 0; exec.init(); while (exec.next() != null) { counter++; } exec.close(); assertEquals(tupleCount, counter); }
/** It asserts the equality between an original table desc and a restored table desc. */ private static void assertSchemaEquality(String tableName, Schema schema) throws IOException, TajoException { Path path = new Path(CommonTestingUtil.getTestDir(), tableName); TableDesc tableDesc = new TableDesc( IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName), schema, "TEXT", new KeyValueSet(), path.toUri()); // schema creation assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); catalog.createTable(tableDesc); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); // change it for the equals test. schema.setQualifier(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); TableDesc restored = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); assertEquals(schema, restored.getSchema()); // drop test catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); }
@BeforeClass public static void setUp() throws Exception { cluster = TpchTestBase.getInstance().getTestingCluster(); conf = cluster.getConfiguration(); client = new TajoClient(conf); testDir = CommonTestingUtil.getTestDir(); }
@BeforeClass public static void setUp() throws Exception { util = new TajoTestingCluster(); util.startCatalogCluster(); catalog = util.getMiniCatalogCluster().getCatalog(); Schema schema = new Schema(); schema.addColumn("name", Type.TEXT); schema.addColumn("empId", CatalogUtil.newSimpleDataType(Type.INT4)); schema.addColumn("deptName", Type.TEXT); Schema schema2 = new Schema(); schema2.addColumn("deptName", Type.TEXT); schema2.addColumn("manager", Type.TEXT); Schema schema3 = new Schema(); schema3.addColumn("deptName", Type.TEXT); schema3.addColumn("score", CatalogUtil.newSimpleDataType(Type.INT4)); TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); TableDesc people = new TableDesc("employee", schema, meta, CommonTestingUtil.getTestDir()); catalog.addTable(people); TableDesc student = new TableDesc( "dept", schema2, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir()); catalog.addTable(student); TableDesc score = new TableDesc( "score", schema3, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir()); catalog.addTable(score); FunctionDesc funcDesc = new FunctionDesc( "sumtest", SumInt.class, FunctionType.AGGREGATION, CatalogUtil.newSimpleDataType(Type.INT4), CatalogUtil.newSimpleDataTypeArray(Type.INT4)); catalog.createFunction(funcDesc); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); }
private TableDesc createMockupTable(String databaseName, String tableName) throws IOException { schema1 = SchemaBuilder.builder() .add(FieldName1, Type.BLOB) .add(FieldName2, Type.INT4) .add(FieldName3, Type.INT8) .build(); Path path = new Path(CommonTestingUtil.getTestDir(), tableName); TableDesc table = new TableDesc( IdentifierUtil.buildFQName(databaseName, tableName), schema1, new TableMeta("TEXT", new KeyValueSet()), path.toUri(), true); return table; }
@Before public void setUp() throws Exception { this.conf = new TajoConf(); util = new TajoTestingCluster(); catalog = util.startCatalogCluster().getCatalog(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); sm = StorageManagerFactory.getStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerId", Type.INT4); schema.addColumn("empId", Type.INT4); schema.addColumn("deptName", Type.TEXT); TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = StorageManagerFactory.getStorageManager(conf) .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.getColumnNum()); for (int i = 0; i < numTuple; i++) { tuple.put( new Datum[] { DatumFactory.createInt4(rnd.nextInt(50)), DatumFactory.createInt4(rnd.nextInt(100)), DatumFactory.createText("dept_" + i), }); appender.addTuple(tuple); } appender.flush(); appender.close(); System.out.println( appender.getStats().getNumRows() + " rows (" + (appender.getStats().getNumBytes() / 1048576) + " MB)"); employee = new TableDesc("employee", schema, employeeMeta, employeePath); catalog.addTable(employee); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); }
public static TableDesc prepareTable() throws IOException { relationSchema = SchemaBuilder.builder() .add(DEFAULT_DATABASE_NAME + ".indexed.id", Type.INT4) .add(DEFAULT_DATABASE_NAME + ".indexed.name", Type.TEXT) .add(DEFAULT_DATABASE_NAME + ".indexed.age", Type.INT4) .add(DEFAULT_DATABASE_NAME + ".indexed.score", Type.FLOAT8) .build(); String tableName = "indexed"; TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, server.getConf()); return new TableDesc( IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), relationSchema, meta, new Path(CommonTestingUtil.getTestDir(), "indexed").toUri()); }
@Test public final void testAddAndDeleteTablePartitionByRange() throws Exception { Schema schema = SchemaBuilder.builder() .add("id", Type.INT4) .add("name", Type.TEXT) .add("age", Type.INT4) .add("score", Type.FLOAT8) .build(); String tableName = IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).build(); PartitionMethodDesc partitionDesc = new PartitionMethodDesc( DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.RANGE, "id", partSchema); TableDesc desc = new TableDesc( tableName, schema, meta, new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri()); desc.setPartitionMethod(partitionDesc); assertFalse(catalog.existsTable(tableName)); catalog.createTable(desc); assertTrue(catalog.existsTable(tableName)); TableDesc retrieved = catalog.getTableDesc(tableName); assertEquals(retrieved.getName(), tableName); assertEquals( retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.RANGE); assertEquals( retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id"); catalog.dropTable(tableName); assertFalse(catalog.existsTable(tableName)); }
@Test public final void testRightOuter_MergeJoin3() throws IOException, TajoException { Expr expr = analyzer.parse(QUERIES[3]); LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = FileTablespace.splitNG( conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getUri()), Integer.MAX_VALUE); FileFragment[] dep4Frags = FileTablespace.splitNG( conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getUri()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir( TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testRightOuter_MergeJoin3"); TaskAttemptContext ctx = new TaskAttemptContext( new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; assertTrue(proj.getChild() instanceof RightOuterMergeJoinExec); int count = 0; exec.init(); while (exec.next() != null) { // TODO check contents count = count + 1; } assertNull(exec.next()); exec.close(); assertEquals(13, count); }
@Test public void testGetTable() throws Exception { schema1 = SchemaBuilder.builder() .add(FieldName1, Type.BLOB) .add(FieldName2, Type.INT4) .add(FieldName3, Type.INT8) .build(); Path path = new Path(CommonTestingUtil.getTestDir(), "table1"); TableDesc meta = new TableDesc( IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"), schema1, "TEXT", new KeyValueSet(), path.toUri()); assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); catalog.createTable(meta); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable")); assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); }
// TODO: This should be added at TAJO-1891 public final void testAddAndDeleteTablePartitionByColumn() throws Exception { Schema schema = SchemaBuilder.builder() .add("id", Type.INT4) .add("name", Type.TEXT) .add("age", Type.INT4) .add("score", Type.FLOAT8) .build(); String simpleTableName = "addedtable"; String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).add("name", Type.TEXT).build(); PartitionMethodDesc partitionMethodDesc = new PartitionMethodDesc( DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.COLUMN, "id,name", partSchema); TableDesc desc = new TableDesc( tableName, schema, meta, new Path(CommonTestingUtil.getTestDir(), simpleTableName).toUri()); desc.setPartitionMethod(partitionMethodDesc); assertFalse(catalog.existsTable(tableName)); catalog.createTable(desc); assertTrue(catalog.existsTable(tableName)); TableDesc retrieved = catalog.getTableDesc(tableName); assertEquals(retrieved.getName(), tableName); assertEquals( retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN); assertEquals( retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id"); testAddPartition(tableName, "id=10/name=aaa"); testAddPartition(tableName, "id=20/name=bbb"); List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName); assertNotNull(partitions); assertEquals(partitions.size(), 2); assertEquals(partitions.get(0).getNumBytes(), 0L); testGetPartitionsByAlgebra(DEFAULT_DATABASE_NAME, simpleTableName); testDropPartition(tableName, "id=10/name=aaa"); testDropPartition(tableName, "id=20/name=bbb"); partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName); assertNotNull(partitions); assertEquals(partitions.size(), 0); catalog.dropTable(tableName); assertFalse(catalog.existsTable(tableName)); }
@Before public void setUp() throws Exception { util = new TajoTestingCluster(); util.initTestDir(); util.startCatalogCluster(); catalog = util.getCatalogService(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); // ----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id // -------------------------------- // 0 | dep_0 | 1000 // 1 | dep_1 | 1001 // 2 | dep_2 | 1002 // 3 | dep_3 | 1003 // 4 | dep_4 | 1004 // 5 | dep_5 | 1005 // 6 | dep_6 | 1006 // 7 | dep_7 | 1007 // 8 | dep_8 | 1008 // 9 | dep_9 | 1009 Schema dep3Schema = new Schema(); dep3Schema.addColumn("dep_id", Type.INT4); dep3Schema.addColumn("dep_name", Type.TEXT); dep3Schema.addColumn("loc_id", Type.INT4); TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); VTuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { tuple.put( new Datum[] { DatumFactory.createInt4(i), DatumFactory.createText("dept_" + i), DatumFactory.createInt4(1000 + i) }); appender1.addTuple(tuple); } appender1.flush(); appender1.close(); dep3 = CatalogUtil.newTableDesc(DEP3_NAME, dep3Schema, dep3Meta, dep3Path); catalog.createTable(dep3); // ----------------- dep4 ------------------------------ // dep_id | dep_name | loc_id // -------------------------------- // 0 | dep_0 | 1000 // 1 | dep_1 | 1001 // 2 | dep_2 | 1002 // 3 | dep_3 | 1003 // 4 | dep_4 | 1004 // 5 | dep_5 | 1005 // 6 | dep_6 | 1006 // 7 | dep_7 | 1007 // 8 | dep_8 | 1008 // 9 | dep_9 | 1009 // 10 | dep_10 | 1010 Schema dep4Schema = new Schema(); dep4Schema.addColumn("dep_id", Type.INT4); dep4Schema.addColumn("dep_name", Type.TEXT); dep4Schema.addColumn("loc_id", Type.INT4); TableMeta dep4Meta = CatalogUtil.newTableMeta("TEXT"); Path dep4Path = new Path(testDir, "dep4.csv"); Appender appender4 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); VTuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { tuple4.put( new Datum[] { DatumFactory.createInt4(i), DatumFactory.createText("dept_" + i), DatumFactory.createInt4(1000 + i) }); appender4.addTuple(tuple4); } appender4.flush(); appender4.close(); dep4 = CatalogUtil.newTableDesc(DEP4_NAME, dep4Schema, dep4Meta, dep4Path); catalog.createTable(dep4); // ----------------- job3 ------------------------------ // job_id | job_title // ---------------------- // 101 | job_101 // 102 | job_102 // 103 | job_103 Schema job3Schema = new Schema(); job3Schema.addColumn("job_id", Type.INT4); job3Schema.addColumn("job_title", Type.TEXT); TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); VTuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { int x = 100 + i; tuple2.put( new Datum[] {DatumFactory.createInt4(100 + i), DatumFactory.createText("job_" + x)}); appender2.addTuple(tuple2); } appender2.flush(); appender2.close(); job3 = CatalogUtil.newTableDesc(JOB3_NAME, job3Schema, job3Meta, job3Path); catalog.createTable(job3); // ---------------------emp3 -------------------- // emp_id | first_name | last_name | dep_id | salary | job_id // ------------------------------------------------------------ // 11 | fn_11 | ln_11 | 1 | 123 | 101 // 13 | fn_13 | ln_13 | 3 | 369 | 103 // 15 | fn_15 | ln_15 | 5 | 615 | null // 17 | fn_17 | ln_17 | 7 | 861 | null // 19 | fn_19 | ln_19 | 9 | 1107 | null // 21 | fn_21 | ln_21 | 1 | 123 | 101 // 23 | fn_23 | ln_23 | 3 | 369 | 103 Schema emp3Schema = new Schema(); emp3Schema.addColumn("emp_id", Type.INT4); emp3Schema.addColumn("first_name", Type.TEXT); emp3Schema.addColumn("last_name", Type.TEXT); emp3Schema.addColumn("dep_id", Type.INT4); emp3Schema.addColumn("salary", Type.FLOAT4); emp3Schema.addColumn("job_id", Type.INT4); TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); VTuple tuple3 = new VTuple(emp3Schema.size()); for (int i = 1; i < 4; i += 2) { int x = 10 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(10 + i), DatumFactory.createText("firstname_" + x), DatumFactory.createText("lastname_" + x), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createInt4(100 + i) }); appender3.addTuple(tuple3); int y = 20 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(20 + i), DatumFactory.createText("firstname_" + y), DatumFactory.createText("lastname_" + y), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createInt4(100 + i) }); appender3.addTuple(tuple3); } for (int i = 5; i < 10; i += 2) { int x = 10 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(10 + i), DatumFactory.createText("firstname_" + x), DatumFactory.createText("lastname_" + x), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createNullDatum() }); appender3.addTuple(tuple3); } appender3.flush(); appender3.close(); emp3 = CatalogUtil.newTableDesc(EMP3_NAME, emp3Schema, emp3Meta, emp3Path); catalog.createTable(emp3); // ---------------------phone3 -------------------- // emp_id | phone_number // ----------------------------------------------- // this table is empty, no rows Schema phone3Schema = new Schema(); phone3Schema.addColumn("emp_id", Type.INT4); phone3Schema.addColumn("phone_number", Type.TEXT); TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT"); Path phone3Path = new Path(testDir, "phone3.csv"); Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); appender5.close(); phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path); catalog.createTable(phone3); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); defaultContext = LocalTajoTestingUtility.createDummyContext(conf); }
@Before public void setup() throws Exception { this.randomValues = new HashMap<Integer, Integer>(); this.conf = new TajoConf(); util = new TajoTestingCluster(); util.startCatalogCluster(); catalog = util.getMiniCatalogCluster().getCatalog(); Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); sm = StorageManagerFactory.getStorageManager(conf, workDir); idxPath = new Path(workDir, "test.idx"); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); schema.addColumn("empid", Type.INT4); schema.addColumn("deptname", Type.TEXT); this.idxSchema = new Schema(); idxSchema.addColumn("managerid", Type.INT4); SortSpec[] sortKeys = new SortSpec[1]; sortKeys[0] = new SortSpec(idxSchema.getColumn("managerid"), true, false); this.comp = new TupleComparator(idxSchema, sortKeys); this.writer = new BSTIndex(conf) .getIndexWriter(idxPath, BSTIndex.TWO_LEVEL_INDEX, this.idxSchema, this.comp); writer.setLoadNum(100); writer.open(); long offset; meta = CatalogUtil.newTableMeta(StoreType.CSV); tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv"); fs = tablePath.getFileSystem(conf); fs.mkdirs(tablePath.getParent()); FileAppender appender = (FileAppender) StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 10000; i++) { Tuple key = new VTuple(this.idxSchema.size()); int rndKey = rnd.nextInt(250); if (this.randomValues.containsKey(rndKey)) { int t = this.randomValues.remove(rndKey) + 1; this.randomValues.put(rndKey, t); } else { this.randomValues.put(rndKey, 1); } key.put(new Datum[] {DatumFactory.createInt4(rndKey)}); tuple.put( new Datum[] { DatumFactory.createInt4(rndKey), DatumFactory.createInt4(rnd.nextInt(10)), DatumFactory.createText("dept_" + rnd.nextInt(10)) }); offset = appender.getOffset(); appender.addTuple(tuple); writer.write(key, offset); } appender.flush(); appender.close(); writer.close(); TableDesc desc = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, sm.getTablePath("employee")); catalog.createTable(desc); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); optimizer = new LogicalOptimizer(conf); }
@After public void tearDown() throws Exception { CommonTestingUtil.cleanupTestDir(TEST_PATH); util.shutdownCatalogCluster(); }