Exemplo n.º 1
0
  private final void assertPathOfCreatedTable(
      final String databaseName,
      final String originalTableName,
      final String newTableName,
      String createTableStmt)
      throws Exception {
    // create one table
    executeString("CREATE DATABASE " + CatalogUtil.denormalizeIdentifier(databaseName)).close();
    getClient().existDatabase(CatalogUtil.denormalizeIdentifier(databaseName));
    final String oldFQTableName = CatalogUtil.buildFQName(databaseName, originalTableName);

    ResultSet res = executeString(createTableStmt);
    res.close();
    assertTableExists(oldFQTableName);
    TableDesc oldTableDesc = client.getTableDesc(oldFQTableName);

    // checking the existence of the table directory and validating the path
    Path warehouseDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration());
    FileSystem fs = warehouseDir.getFileSystem(testingCluster.getConfiguration());
    assertTrue(fs.exists(new Path(oldTableDesc.getUri())));
    assertEquals(
        StorageUtil.concatPath(warehouseDir, databaseName, originalTableName),
        new Path(oldTableDesc.getUri()));

    // Rename
    client.executeQuery(
        "ALTER TABLE "
            + CatalogUtil.denormalizeIdentifier(oldFQTableName)
            + " RENAME to "
            + CatalogUtil.denormalizeIdentifier(newTableName));

    // checking the existence of the new table directory and validating the path
    final String newFQTableName = CatalogUtil.buildFQName(databaseName, newTableName);
    TableDesc newTableDesc = client.getTableDesc(newFQTableName);
    assertTrue(fs.exists(new Path(newTableDesc.getUri())));
    assertEquals(
        StorageUtil.concatPath(warehouseDir, databaseName, newTableName),
        new Path(newTableDesc.getUri()));
  }
Exemplo n.º 2
0
  @Test
  public final void testSelfDescTable2() throws Exception {
    executeString("create database d10;").close();

    String className = getClass().getSimpleName();
    Path currentDatasetPath = new Path(datasetBasePath, className);
    Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
    String sql =
        "create external table d10.schemaless (*) using json with ('compression.codec'='none') location '"
            + filePath.toString()
            + "'";
    executeString(sql).close();
    assertTableExists("d10.schemaless");
    TableDesc desc = getClient().getTableDesc("d10.schemaless");
    assertTrue(desc.hasEmptySchema());

    executeString("drop table d10.schemaless").close();
    executeString("drop database d10").close();
  }
Exemplo n.º 3
0
  private void assertOutputResult(
      String expectedResultFile, String actual, String[] paramKeys, String[] paramValues)
      throws Exception {
    FileSystem fs =
        currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
    Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile);
    assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile));

    String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));

    if (paramKeys != null) {
      for (int i = 0; i < paramKeys.length; i++) {
        if (i < paramValues.length) {
          expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]);
        }
      }
    }
    assertEquals(expectedResult.trim(), actual.trim());
  }
Exemplo n.º 4
0
 private Path getDataSetFile(String fileName) {
   return StorageUtil.concatPath(currentDatasetPath, fileName);
 }
Exemplo n.º 5
0
 private Path getResultFile(String fileName) {
   return StorageUtil.concatPath(currentResultPath, fileName);
 }
Exemplo n.º 6
0
 private Path getQueryFilePath(String fileName) {
   return StorageUtil.concatPath(currentQueryPath, fileName);
 }
Exemplo n.º 7
0
  @Before
  public void setup() throws Exception {
    this.randomValues = new HashMap<Integer, Integer>();
    this.conf = new TajoConf();
    util = new TajoTestingCluster();
    util.startCatalogCluster();
    catalog = util.getMiniCatalogCluster().getCatalog();

    Path workDir = CommonTestingUtil.getTestDir();
    catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString());
    catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
    sm = StorageManagerFactory.getStorageManager(conf, workDir);

    idxPath = new Path(workDir, "test.idx");

    Schema schema = new Schema();
    schema.addColumn("managerid", Type.INT4);
    schema.addColumn("empid", Type.INT4);
    schema.addColumn("deptname", Type.TEXT);

    this.idxSchema = new Schema();
    idxSchema.addColumn("managerid", Type.INT4);
    SortSpec[] sortKeys = new SortSpec[1];
    sortKeys[0] = new SortSpec(idxSchema.getColumn("managerid"), true, false);
    this.comp = new TupleComparator(idxSchema, sortKeys);

    this.writer =
        new BSTIndex(conf)
            .getIndexWriter(idxPath, BSTIndex.TWO_LEVEL_INDEX, this.idxSchema, this.comp);
    writer.setLoadNum(100);
    writer.open();
    long offset;

    meta = CatalogUtil.newTableMeta(StoreType.CSV);
    tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv");
    fs = tablePath.getFileSystem(conf);
    fs.mkdirs(tablePath.getParent());

    FileAppender appender =
        (FileAppender)
            StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
    appender.init();
    Tuple tuple = new VTuple(schema.size());
    for (int i = 0; i < 10000; i++) {

      Tuple key = new VTuple(this.idxSchema.size());
      int rndKey = rnd.nextInt(250);
      if (this.randomValues.containsKey(rndKey)) {
        int t = this.randomValues.remove(rndKey) + 1;
        this.randomValues.put(rndKey, t);
      } else {
        this.randomValues.put(rndKey, 1);
      }

      key.put(new Datum[] {DatumFactory.createInt4(rndKey)});
      tuple.put(
          new Datum[] {
            DatumFactory.createInt4(rndKey),
            DatumFactory.createInt4(rnd.nextInt(10)),
            DatumFactory.createText("dept_" + rnd.nextInt(10))
          });
      offset = appender.getOffset();
      appender.addTuple(tuple);
      writer.write(key, offset);
    }
    appender.flush();
    appender.close();
    writer.close();

    TableDesc desc =
        new TableDesc(
            CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"),
            schema,
            meta,
            sm.getTablePath("employee"));
    catalog.createTable(desc);

    analyzer = new SQLAnalyzer();
    planner = new LogicalPlanner(catalog);
    optimizer = new LogicalOptimizer(conf);
  }
Exemplo n.º 8
0
  @Test
  public final void testCreateTableLike1() throws Exception {
    // //HiveCatalogStore does not support varchar type in hive-0.12.0
    if (testingCluster.isHiveCatalogStoreRunning()) {
      // Basic create table with default database
      executeString("CREATE TABLE table1 (c1 int, c2 text);").close();
      executeString("CREATE TABLE table2 LIKE table1");
      String testMsg = "testCreateTableLike1: Basic create table with default db";
      assertTrue(testMsg, isClonedTable("table1", "table2"));
      executeString("DROP TABLE table1");
      executeString("DROP TABLE table2");

      // Basic create table with database
      executeString("CREATE DATABASE d1").close();
      executeString("CREATE TABLE d1.table1 (c1 int, c2 text);").close();
      executeString("CREATE TABLE d1.table2 LIKE d1.table1");
      testMsg = "testCreateTableLike1: Basic create table with db test failed";
      assertTrue(testMsg, isClonedTable("d1.table1", "d1.table2"));
      executeString("DROP TABLE d1.table1");
      executeString("DROP TABLE d1.table2");

      // Table with non-default store type
      executeString("CREATE TABLE table1 (c1 int, c2 text) USING rcfile;").close();
      executeString("CREATE TABLE table2 LIKE table1");
      testMsg = "testCreateTableLike1: Table with non-default store type test failed";
      assertTrue(testMsg, isClonedTable("table1", "table2"));
      executeString("DROP TABLE table1");
      executeString("DROP TABLE table2");

      // Table with non-default meta options
      executeString(
              "CREATE TABLE table1 (c1 int, c2 text) USING text WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');")
          .close();
      executeString("CREATE TABLE table2 LIKE table1");
      testMsg = "testCreateTableLike1: Table with non-default meta options test failed";
      assertTrue(testMsg, isClonedTable("table1", "table2"));
      executeString("DROP TABLE table1");
      executeString("DROP TABLE table2");

      // Table with partitions (default partition type)
      executeString(
              "CREATE TABLE table1 (c1 int, c2 text) PARTITION BY COLUMN (c3 int, c4 float, c5 text);")
          .close();
      executeString("CREATE TABLE table2 LIKE table1");
      testMsg = "testCreateTableLike1: Table with partitions test failed";
      assertTrue(testMsg, isClonedTable("table1", "table2"));
      executeString("DROP TABLE table1");
      executeString("DROP TABLE table2");

      // Table with external flag
      // Use existing file as input for creating external table
      String className = getClass().getSimpleName();
      Path currentDatasetPath = new Path(datasetBasePath, className);
      Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
      executeString(
              "CREATE EXTERNAL TABLE table3 (c1 int, c2 text) USING rcfile LOCATION '"
                  + filePath.toUri()
                  + "'")
          .close();
      executeString("CREATE TABLE table2 LIKE table3");
      testMsg = "testCreateTableLike1: Table with external table flag test failed";
      assertTrue(testMsg, isClonedTable("table3", "table2"));
      executeString("DROP TABLE table3");
      executeString("DROP TABLE table2");

      // Table created using CTAS
      executeString("CREATE TABLE table3 (c1 int, c2 text) PARTITION BY COLUMN (c3 int);").close();
      executeString("CREATE TABLE table4 AS SELECT c1 * c1 as m_c1, c2, c2 as c2_a,c3 from table3;")
          .close();
      executeString("CREATE TABLE table2 LIKE table4");
      testMsg = "testCreateTableLike1: Table using CTAS test failed";
      assertTrue(testMsg, isClonedTable("table4", "table2"));
      executeString("DROP TABLE table3");
      executeString("DROP TABLE table4");
      executeString("DROP TABLE table2");
    } else {
      // Basic create table with default database
      executeString("CREATE TABLE table1 (c1 int, c2 varchar);").close();
      executeString("CREATE TABLE table2 LIKE table1");
      String testMsg = "testCreateTableLike1: Basic create table with default db";
      assertTrue(testMsg, isClonedTable("table1", "table2"));
      executeString("DROP TABLE table1");
      executeString("DROP TABLE table2");

      // Basic create table with database
      executeString("CREATE DATABASE d1").close();
      executeString("CREATE TABLE d1.table1 (c1 int, c2 varchar);").close();
      executeString("CREATE TABLE d1.table2 LIKE d1.table1");
      testMsg = "testCreateTableLike1: Basic create table with db test failed";
      assertTrue(testMsg, isClonedTable("d1.table1", "d1.table2"));
      executeString("DROP TABLE d1.table1");
      executeString("DROP TABLE d1.table2");

      // Table with non-default store type
      executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING rcfile;").close();
      executeString("CREATE TABLE table2 LIKE table1");
      testMsg = "testCreateTableLike1: Table with non-default store type test failed";
      assertTrue(testMsg, isClonedTable("table1", "table2"));
      executeString("DROP TABLE table1");
      executeString("DROP TABLE table2");

      // Table with non-default meta options
      executeString(
              "CREATE TABLE table1 (c1 int, c2 varchar) USING text WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');")
          .close();
      executeString("CREATE TABLE table2 LIKE table1");
      testMsg = "testCreateTableLike1: Table with non-default meta options test failed";
      assertTrue(testMsg, isClonedTable("table1", "table2"));
      executeString("DROP TABLE table1");
      executeString("DROP TABLE table2");

      // Table with partitions (default partition type)
      executeString(
              "CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int, c4 float, c5 text);")
          .close();
      executeString("CREATE TABLE table2 LIKE table1");
      testMsg = "testCreateTableLike1: Table with partitions test failed";
      assertTrue(testMsg, isClonedTable("table1", "table2"));
      executeString("DROP TABLE table1");
      executeString("DROP TABLE table2");

      // Table with external flag
      // Use existing file as input for creating external table
      String className = getClass().getSimpleName();
      Path currentDatasetPath = new Path(datasetBasePath, className);
      Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
      executeString(
              "CREATE EXTERNAL TABLE table3 (c1 int, c2 varchar) USING rcfile LOCATION '"
                  + filePath.toUri()
                  + "'")
          .close();
      executeString("CREATE TABLE table2 LIKE table3");
      testMsg = "testCreateTableLike1: Table with external table flag test failed";
      assertTrue(testMsg, isClonedTable("table3", "table2"));
      executeString("DROP TABLE table3");
      executeString("DROP TABLE table2");

      // Table created using CTAS
      executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);")
          .close();
      executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2 as c2_a,c3 from table3;").close();
      executeString("CREATE TABLE table2 LIKE table4");
      testMsg = "testCreateTableLike1: Table using CTAS test failed";
      assertTrue(testMsg, isClonedTable("table4", "table2"));
      executeString("DROP TABLE table3");
      executeString("DROP TABLE table4");
      executeString("DROP TABLE table2");

      /* Enable when view is supported
      // View
      executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close();
      executeString("CREATE VIEW table4(c1,c2,c3) AS SELECT c1*c1, c2, c2,c3 from table3;").close();
      executeString("CREATE TABLE table2 LIKE table4");
      testMsg = "testCreateTableLike1: Table using VIEW test failed";
      assertTrue(testMsg, isClonedTable("table4","table2"));
      executeString("DROP TABLE table3");
      executeString("DROP TABLE table4");
      executeString("DROP TABLE table2");
      */

      /*  Enable when partition type other than column is supported
      // Table with partitions (range partition)
      executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY RANGE (c1) (  PARTITION c1 VALUES LESS THAN (2),  PARTITION c1 VALUES LESS THAN (5),  PARTITION c1 VALUES LESS THAN (MAXVALUE) );").close();
      executeString("CREATE TABLE table2 LIKE table1");
      testMsg = "testCreateTableLike1: Table using non-default partition type failed";
      assertTrue(testMsg, isClonedTable("table1","table2"));
      executeString("DROP TABLE table1");
      executeString("DROP TABLE table2");
      */
    }
  }
Exemplo n.º 9
0
 private static Path writeTmpTable(String tableName) throws IOException {
   Path tablePath = StorageUtil.concatPath(testDir, tableName);
   BackendTestingUtil.writeTmpTable(conf, tablePath);
   return tablePath;
 }