Ejemplo n.º 1
0
  /** It asserts the equality between an original table desc and a restored table desc. */
  private static void assertSchemaEquality(String tableName, Schema schema)
      throws IOException, TajoException {
    Path path = new Path(CommonTestingUtil.getTestDir(), tableName);
    TableDesc tableDesc =
        new TableDesc(
            IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName),
            schema,
            "TEXT",
            new KeyValueSet(),
            path.toUri());

    // schema creation
    assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
    catalog.createTable(tableDesc);
    assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));

    // change it for the equals test.
    schema.setQualifier(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
    TableDesc restored = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
    assertEquals(schema, restored.getSchema());

    // drop test
    catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
    assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
  }
Ejemplo n.º 2
0
  @Test
  public final void testSelfDescTable1() throws Exception {
    executeString("create database d9;").close();

    assertTableNotExists("d9.schemaless");
    executeQuery();
    assertTableExists("d9.schemaless");
    TableDesc desc = getClient().getTableDesc("d9.schemaless");
    assertTrue(desc.hasEmptySchema());

    executeString("drop table d9.schemaless").close();
    executeString("drop database d9").close();
  }
Ejemplo n.º 3
0
  @Test
  public final void testSelfDescTable2() throws Exception {
    executeString("create database d10;").close();

    String className = getClass().getSimpleName();
    Path currentDatasetPath = new Path(datasetBasePath, className);
    Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
    String sql =
        "create external table d10.schemaless (*) using json with ('compression.codec'='none') location '"
            + filePath.toString()
            + "'";
    executeString(sql).close();
    assertTableExists("d10.schemaless");
    TableDesc desc = getClient().getTableDesc("d10.schemaless");
    assertTrue(desc.hasEmptySchema());

    executeString("drop table d10.schemaless").close();
    executeString("drop database d10").close();
  }
  @Test
  public final void testRightOuter_MergeJoin3() throws IOException, TajoException {
    Expr expr = analyzer.parse(QUERIES[3]);
    LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
    JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
    Enforcer enforcer = new Enforcer();
    enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);

    FileFragment[] emp3Frags =
        FileTablespace.splitNG(
            conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getUri()), Integer.MAX_VALUE);
    FileFragment[] dep4Frags =
        FileTablespace.splitNG(
            conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getUri()), Integer.MAX_VALUE);
    FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags);

    Path workDir =
        CommonTestingUtil.getTestDir(
            TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testRightOuter_MergeJoin3");
    TaskAttemptContext ctx =
        new TaskAttemptContext(
            new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
    ctx.setEnforcer(enforcer);

    PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
    PhysicalExec exec = phyPlanner.createPlan(ctx, plan);

    ProjectionExec proj = (ProjectionExec) exec;
    assertTrue(proj.getChild() instanceof RightOuterMergeJoinExec);

    int count = 0;
    exec.init();

    while (exec.next() != null) {
      // TODO check contents
      count = count + 1;
    }
    assertNull(exec.next());
    exec.close();
    assertEquals(13, count);
  }
Ejemplo n.º 5
0
  @Test
  public void testAddAndDelIndex() throws Exception {
    TableDesc desc = prepareTable();
    prepareIndexDescs();
    catalog.createTable(desc);

    assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
    assertFalse(
        catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"id"}));
    catalog.createIndex(desc1);
    assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
    assertTrue(
        catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"id"}));

    assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
    assertFalse(
        catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"score"}));
    catalog.createIndex(desc2);
    assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
    assertTrue(
        catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"score"}));

    Set<IndexDesc> indexDescs = new HashSet<>();
    indexDescs.add(desc1);
    indexDescs.add(desc2);
    indexDescs.add(desc3);
    for (IndexDesc index : catalog.getAllIndexesByTable(DEFAULT_DATABASE_NAME, "indexed")) {
      assertTrue(indexDescs.contains(index));
    }

    catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getName());
    assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
    catalog.dropIndex(DEFAULT_DATABASE_NAME, desc2.getName());
    assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));

    catalog.dropTable(desc.getName());
    assertFalse(catalog.existsTable(desc.getName()));
  }
Ejemplo n.º 6
0
  @Test
  public final void testAddAndDeleteTablePartitionByRange() throws Exception {
    Schema schema =
        SchemaBuilder.builder()
            .add("id", Type.INT4)
            .add("name", Type.TEXT)
            .add("age", Type.INT4)
            .add("score", Type.FLOAT8)
            .build();

    String tableName =
        IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
    KeyValueSet opts = new KeyValueSet();
    opts.set("file.delimiter", ",");
    TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);

    Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).build();
    PartitionMethodDesc partitionDesc =
        new PartitionMethodDesc(
            DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.RANGE, "id", partSchema);

    TableDesc desc =
        new TableDesc(
            tableName,
            schema,
            meta,
            new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
    desc.setPartitionMethod(partitionDesc);
    assertFalse(catalog.existsTable(tableName));
    catalog.createTable(desc);
    assertTrue(catalog.existsTable(tableName));

    TableDesc retrieved = catalog.getTableDesc(tableName);

    assertEquals(retrieved.getName(), tableName);
    assertEquals(
        retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.RANGE);
    assertEquals(
        retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");

    catalog.dropTable(tableName);
    assertFalse(catalog.existsTable(tableName));
  }
Ejemplo n.º 7
0
  private final void assertPathOfCreatedTable(
      final String databaseName,
      final String originalTableName,
      final String newTableName,
      String createTableStmt)
      throws Exception {
    // create one table
    executeString("CREATE DATABASE " + CatalogUtil.denormalizeIdentifier(databaseName)).close();
    getClient().existDatabase(CatalogUtil.denormalizeIdentifier(databaseName));
    final String oldFQTableName = CatalogUtil.buildFQName(databaseName, originalTableName);

    ResultSet res = executeString(createTableStmt);
    res.close();
    assertTableExists(oldFQTableName);
    TableDesc oldTableDesc = client.getTableDesc(oldFQTableName);

    // checking the existence of the table directory and validating the path
    Path warehouseDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration());
    FileSystem fs = warehouseDir.getFileSystem(testingCluster.getConfiguration());
    assertTrue(fs.exists(new Path(oldTableDesc.getUri())));
    assertEquals(
        StorageUtil.concatPath(warehouseDir, databaseName, originalTableName),
        new Path(oldTableDesc.getUri()));

    // Rename
    client.executeQuery(
        "ALTER TABLE "
            + CatalogUtil.denormalizeIdentifier(oldFQTableName)
            + " RENAME to "
            + CatalogUtil.denormalizeIdentifier(newTableName));

    // checking the existence of the new table directory and validating the path
    final String newFQTableName = CatalogUtil.buildFQName(databaseName, newTableName);
    TableDesc newTableDesc = client.getTableDesc(newFQTableName);
    assertTrue(fs.exists(new Path(newTableDesc.getUri())));
    assertEquals(
        StorageUtil.concatPath(warehouseDir, databaseName, newTableName),
        new Path(newTableDesc.getUri()));
  }
Ejemplo n.º 8
0
  // TODO: This should be added at TAJO-1891
  public final void testAddAndDeleteTablePartitionByColumn() throws Exception {
    Schema schema =
        SchemaBuilder.builder()
            .add("id", Type.INT4)
            .add("name", Type.TEXT)
            .add("age", Type.INT4)
            .add("score", Type.FLOAT8)
            .build();

    String simpleTableName = "addedtable";
    String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName);
    KeyValueSet opts = new KeyValueSet();
    opts.set("file.delimiter", ",");
    TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);

    Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).add("name", Type.TEXT).build();

    PartitionMethodDesc partitionMethodDesc =
        new PartitionMethodDesc(
            DEFAULT_DATABASE_NAME,
            tableName,
            CatalogProtos.PartitionType.COLUMN,
            "id,name",
            partSchema);

    TableDesc desc =
        new TableDesc(
            tableName,
            schema,
            meta,
            new Path(CommonTestingUtil.getTestDir(), simpleTableName).toUri());
    desc.setPartitionMethod(partitionMethodDesc);
    assertFalse(catalog.existsTable(tableName));
    catalog.createTable(desc);
    assertTrue(catalog.existsTable(tableName));

    TableDesc retrieved = catalog.getTableDesc(tableName);

    assertEquals(retrieved.getName(), tableName);
    assertEquals(
        retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN);
    assertEquals(
        retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");

    testAddPartition(tableName, "id=10/name=aaa");
    testAddPartition(tableName, "id=20/name=bbb");

    List<CatalogProtos.PartitionDescProto> partitions =
        catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
    assertNotNull(partitions);
    assertEquals(partitions.size(), 2);
    assertEquals(partitions.get(0).getNumBytes(), 0L);

    testGetPartitionsByAlgebra(DEFAULT_DATABASE_NAME, simpleTableName);

    testDropPartition(tableName, "id=10/name=aaa");
    testDropPartition(tableName, "id=20/name=bbb");

    partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
    assertNotNull(partitions);
    assertEquals(partitions.size(), 0);

    catalog.dropTable(tableName);
    assertFalse(catalog.existsTable(tableName));
  }
Ejemplo n.º 9
0
  @Test
  public void testAlterTable() throws Exception {

    // CREATE_TABLE
    TableDesc tableRenameTestDesc = createMockupTable("default", "mycooltable");
    catalog.createTable(tableRenameTestDesc);

    // RENAME_TABLE
    catalog.alterTable(createMockAlterTableName());
    assertTrue(catalog.existsTable("default", "mynewcooltable"));

    // RENAME_COLUMN
    catalog.alterTable(createMockAlterTableRenameColumn());
    TableDesc columnRenameDesc = catalog.getTableDesc("default", "mynewcooltable");
    assertTrue(columnRenameDesc.getSchema().containsByName("ren" + FieldName1));

    // ADD_COLUMN
    catalog.alterTable(createMockAlterTableAddColumn());
    TableDesc addColumnDesc = catalog.getTableDesc("default", "mynewcooltable");
    assertTrue(addColumnDesc.getSchema().containsByName("mynewcol"));

    // SET_PROPERTY
    TableDesc setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
    KeyValueSet options = new KeyValueSet();
    options.set("timezone", "GMT+9"); // Seoul, Korea
    setPropertyDesc.setMeta(new TableMeta("TEXT", options));
    String prevTimeZone = setPropertyDesc.getMeta().getProperty("timezone");
    String newTimeZone = "GMT-7"; // Silicon Valley, California
    catalog.alterTable(createMockAlterTableSetProperty(newTimeZone));
    setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
    assertNotEquals(prevTimeZone, setPropertyDesc.getMeta().getProperty("timezone"));
    assertEquals(newTimeZone, setPropertyDesc.getMeta().getProperty("timezone"));

    // UNSET_PROPERTY
    catalog.alterTable(createMockAlterTableUnsetProperty(Sets.newHashSet("dummy")));
    setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
    assertTrue(setPropertyDesc.getMeta().getPropertySet().containsKey("timezone"));
    assertFalse(setPropertyDesc.getMeta().getPropertySet().containsKey("dummy"));
  }
Ejemplo n.º 10
0
  private boolean isClonedTable(String orignalTable, String newTable) throws Exception {
    assertTableExists(newTable);
    TableDesc origTableDesc = client.getTableDesc(orignalTable);
    TableDesc newTableDesc = client.getTableDesc(newTable);

    if (isClonedSchema(origTableDesc.getSchema(), newTableDesc.getSchema()) == false) {
      fail("Schema of input tables do not match");
      return false;
    }

    // Check partition information
    PartitionMethodDesc origPartMethod = origTableDesc.getPartitionMethod();
    PartitionMethodDesc newPartMethod = newTableDesc.getPartitionMethod();
    if (origPartMethod != null) {
      if (newPartMethod == null) {
        fail("New table does not have partition info");
        return false;
      }
      if (isClonedSchema(origPartMethod.getExpressionSchema(), newPartMethod.getExpressionSchema())
          == false) {
        fail("Partition columns of input tables do not match");
        return false;
      }

      if (origPartMethod.getPartitionType().equals(newPartMethod.getPartitionType()) == false) {
        fail("Partition type of input tables do not match");
        return false;
      }
    }

    // Check external flag
    if (origTableDesc.isExternal() != newTableDesc.isExternal()) {
      fail("External table flag on input tables not equal");
      return false;
    }

    if (origTableDesc.getMeta() != null) {
      TableMeta origMeta = origTableDesc.getMeta();
      TableMeta newMeta = newTableDesc.getMeta();
      if (origMeta.getDataFormat().equals(newMeta.getDataFormat()) == false) {
        fail("Store type of input tables not equal");
        return false;
      }

      KeyValueSet origOptions = origMeta.getPropertySet();
      KeyValueSet newOptions = newMeta.getPropertySet();
      if (origOptions.equals(newOptions) == false) {
        fail("Meta options of input tables not equal");
        return false;
      }
    }
    return true;
  }