@Override
public void execute(
QueryMaster.QueryMasterContext context,
QueryContext queryContext,
Query query,
ExecutionBlockId finalExecBlockId,
Path finalOutputDir)
throws Exception {
CatalogService catalog = context.getWorkerContext().getCatalog();
SubQuery lastStage = query.getSubQuery(finalExecBlockId);
TableMeta meta = lastStage.getTableMeta();
TableStats stats = lastStage.getTableStat();
CreateTableNode createTableNode = (CreateTableNode) lastStage.getBlock().getPlan();
TableDesc tableDescTobeCreated =
new TableDesc(
createTableNode.getTableName(),
createTableNode.getTableSchema(),
meta,
finalOutputDir);
if (createTableNode.hasPartition()) {
tableDescTobeCreated.setPartitionMethod(createTableNode.getPartitionMethod());
}
stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir));
tableDescTobeCreated.setStats(stats);
query.setResultDesc(tableDescTobeCreated);
catalog.addTable(tableDescTobeCreated);
}
/** It asserts the equality between an original table desc and a restored table desc. */
private static void assertSchemaEquality(String tableName, Schema schema)
throws IOException, TajoException {
Path path = new Path(CommonTestingUtil.getTestDir(), tableName);
TableDesc tableDesc =
new TableDesc(
IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName),
schema,
"TEXT",
new KeyValueSet(),
path.toUri());
// schema creation
assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
catalog.createTable(tableDesc);
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
// change it for the equals test.
schema.setQualifier(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
TableDesc restored = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
assertEquals(schema, restored.getSchema());
// drop test
catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
}
private List<String> executeDDL(
String ddlFileName,
@Nullable String dataFileName,
boolean isLocalTable,
@Nullable String[] args)
throws Exception {
Path ddlFilePath = new Path(currentQueryPath, ddlFileName);
FileSystem fs = ddlFilePath.getFileSystem(conf);
assertTrue(ddlFilePath + " existence check", fs.exists(ddlFilePath));
String template = FileUtil.readTextFile(new File(ddlFilePath.toUri()));
String dataFilePath = null;
if (dataFileName != null) {
dataFilePath = getDataSetFile(dataFileName).toString();
}
String compiled = compileTemplate(template, dataFilePath, args);
List<ParsedResult> parsedResults = SimpleParser.parseScript(compiled);
List<String> createdTableNames = new ArrayList<String>();
for (ParsedResult parsedResult : parsedResults) {
// parse a statement
Expr expr = sqlParser.parse(parsedResult.getStatement());
assertNotNull(ddlFilePath + " cannot be parsed", expr);
if (expr.getType() == OpType.CreateTable) {
CreateTable createTable = (CreateTable) expr;
String tableName = createTable.getTableName();
assertTrue("Table creation is failed.", client.updateQuery(parsedResult.getStatement()));
TableDesc createdTable = client.getTableDesc(tableName);
String createdTableName = createdTable.getName();
assertTrue(
"table '" + createdTableName + "' creation check", client.existTable(createdTableName));
if (isLocalTable) {
createdTableGlobalSet.add(createdTableName);
createdTableNames.add(tableName);
}
} else if (expr.getType() == OpType.DropTable) {
DropTable dropTable = (DropTable) expr;
String tableName = dropTable.getTableName();
assertTrue(
"table '" + tableName + "' existence check",
client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
assertTrue("table drop is failed.", client.updateQuery(parsedResult.getStatement()));
assertFalse(
"table '" + tableName + "' dropped check",
client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
if (isLocalTable) {
createdTableGlobalSet.remove(tableName);
}
} else {
assertTrue(ddlFilePath + " is not a Create or Drop Table statement", false);
}
}
return createdTableNames;
}
@Test(timeout = 1000)
public void testGetTable() throws UndefinedTableException {
for (String tableName : PgSQLTestServer.TPCH_TABLES) {
TableDesc retrieved = client.getTableDesc(PgSQLTestServer.DATABASE_NAME + "." + tableName);
assertEquals(PgSQLTestServer.DATABASE_NAME + "." + tableName, retrieved.getName());
assertEquals(jdbcUrl + "&table=" + tableName, retrieved.getUri().toASCIIString());
}
}
@Test
public final void testSelfDescTable1() throws Exception {
executeString("create database d9;").close();
assertTableNotExists("d9.schemaless");
executeQuery();
assertTableExists("d9.schemaless");
TableDesc desc = getClient().getTableDesc("d9.schemaless");
assertTrue(desc.hasEmptySchema());
executeString("drop table d9.schemaless").close();
executeString("drop database d9").close();
}
@Test
public final void testCtasWithTextFile() throws Exception {
ResultSet res = executeFile("CtasWithTextFile.sql");
res.close();
ResultSet res2 = executeQuery();
resultSetToString(res2);
res2.close();
TableDesc desc =
client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
assertNotNull(desc);
assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getStoreType()));
}
public void executeDistributedQuery(
QueryContext queryContext,
Session session,
LogicalPlan plan,
String sql,
String jsonExpr,
SubmitQueryResponse.Builder responseBuilder)
throws Exception {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, plan.getRootBlock().getRoot());
if (tableDesc != null) {
Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
FormatProperty formatProperty = space.getFormatProperty(tableDesc.getMeta());
if (!formatProperty.isInsertable()) {
throw new UnsupportedException(
String.format("INSERT operation on %s tablespace", tableDesc.getUri().toString()));
}
space.prepareTable(rootNode.getChild());
}
hookManager.doHooks(queryContext, plan);
QueryManager queryManager = this.context.getQueryJobManager();
QueryInfo queryInfo;
queryInfo = queryManager.scheduleQuery(session, queryContext, sql, jsonExpr, rootNode);
responseBuilder.setState(OK);
responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
responseBuilder.setResultType(ResultType.FETCH);
if (queryInfo.getQueryMasterHost() != null) {
responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost());
}
responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort());
LOG.info(
"Query "
+ queryInfo.getQueryId().toString()
+ ","
+ queryInfo.getSql()
+ ","
+ " is forwarded to "
+ queryInfo.getQueryMasterHost()
+ ":"
+ queryInfo.getQueryMasterPort());
}
public MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, GlobalPlanner planner)
throws Exception {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
TableDesc tableDesc = PlannerUtil.getTableDesc(planner.getCatalog(), rootNode.getChild());
if (tableDesc != null) {
Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
space.rewritePlan(context, plan);
}
MasterPlan masterPlan = new MasterPlan(QueryIdFactory.NULL_QUERY_ID, context, plan);
planner.build(context, masterPlan);
return masterPlan;
}
private List<Tuple> fetchSystemTable(TableDesc tableDesc, Schema inSchema) {
List<Tuple> tuples = null;
String tableName = CatalogUtil.extractSimpleName(tableDesc.getName());
if ("tablespace".equalsIgnoreCase(tableName)) {
tuples = getTablespaces(inSchema);
} else if ("databases".equalsIgnoreCase(tableName)) {
tuples = getDatabases(inSchema);
} else if ("tables".equalsIgnoreCase(tableName)) {
tuples = getTables(inSchema);
} else if ("columns".equalsIgnoreCase(tableName)) {
tuples = getColumns(inSchema);
} else if ("indexes".equalsIgnoreCase(tableName)) {
tuples = getIndexes(inSchema);
} else if ("table_options".equalsIgnoreCase(tableName)) {
tuples = getAllTableOptions(inSchema);
} else if ("table_stats".equalsIgnoreCase(tableName)) {
tuples = getAllTableStats(inSchema);
} else if ("partitions".equalsIgnoreCase(tableName)) {
tuples = getAllPartitions(inSchema);
} else if ("cluster".equalsIgnoreCase(tableName)) {
tuples = getClusterInfo(inSchema);
} else if ("session".equalsIgnoreCase(tableName)) {
tuples = getSessionInfo(inSchema);
}
return tuples;
}
public void execSimpleQuery(
QueryContext queryContext,
Session session,
String query,
LogicalPlan plan,
SubmitQueryResponse.Builder response)
throws Exception {
ScanNode scanNode = plan.getRootBlock().getNode(NodeType.SCAN);
TableDesc desc = scanNode.getTableDesc();
if (desc.hasPartition()) {
scanNode = plan.getRootBlock().getNode(NodeType.PARTITIONS_SCAN);
}
int maxRow = Integer.MAX_VALUE;
if (plan.getRootBlock().hasNode(NodeType.LIMIT)) {
LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT);
maxRow = (int) limitNode.getFetchFirstNum();
}
if (desc.getStats().getNumRows() == 0) {
desc.getStats().setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER);
}
QueryInfo queryInfo =
context
.getQueryJobManager()
.createNewSimpleQuery(
queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot());
NonForwardQueryResultScanner queryResultScanner =
new NonForwardQueryResultFileScanner(
context.getConf(),
session.getSessionId(),
queryInfo.getQueryId(),
scanNode,
desc,
maxRow);
queryResultScanner.init();
session.addNonForwardQueryResultScanner(queryResultScanner);
response.setState(OK);
response.setQueryId(queryInfo.getQueryId().getProto());
response.setResultType(ResultType.ENCLOSED);
response.setMaxRowNum(maxRow);
response.setTableDesc(desc.getProto());
}
@Test
public final void testCtasWithoutTableDefinition() throws Exception {
ResultSet res = executeQuery();
res.close();
String tableName = CatalogUtil.normalizeIdentifier("testCtasWithoutTableDefinition");
CatalogService catalog = testBase.getTestingCluster().getMaster().getCatalog();
String qualifiedTableName = buildFQName(DEFAULT_DATABASE_NAME, tableName);
TableDesc desc = catalog.getTableDesc(qualifiedTableName);
assertTrue(catalog.existsTable(qualifiedTableName));
assertTrue(desc.getSchema().contains("default.testctaswithouttabledefinition.col1"));
PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN);
assertEquals(
"key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName());
FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration());
Path path = new Path(desc.getUri());
assertTrue(fs.isDirectory(path));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0")));
if (!testingCluster.isHiveCatalogStoreRunning()) {
assertEquals(5, desc.getStats().getNumRows().intValue());
}
ResultSet res2 = executeFile("check1.sql");
Map<Double, int[]> resultRows1 = Maps.newHashMap();
resultRows1.put(45.0d, new int[] {3, 2});
resultRows1.put(38.0d, new int[] {2, 2});
int i = 0;
while (res2.next()) {
assertEquals(resultRows1.get(res2.getDouble(3))[0], res2.getInt(1));
assertEquals(resultRows1.get(res2.getDouble(3))[1], res2.getInt(2));
i++;
}
res2.close();
assertEquals(2, i);
}
public Object clone() throws CloneNotSupportedException {
TableDesc desc = (TableDesc) super.clone();
desc.builder = TableDescProto.newBuilder();
desc.tableName = tableName;
desc.schema = (Schema) schema.clone();
desc.meta = (TableMeta) meta.clone();
desc.uri = uri;
desc.stats = stats != null ? (TableStats) stats.clone() : null;
desc.partitionMethodDesc =
partitionMethodDesc != null ? (PartitionMethodDesc) partitionMethodDesc.clone() : null;
return desc;
}
@Test
public final void testCtasWithOptions() throws Exception {
ResultSet res = executeFile("CtasWithOptions.sql");
res.close();
ResultSet res2 = executeQuery();
resultSetToString(res2);
res2.close();
TableDesc desc =
client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
assertNotNull(desc);
assertTrue("CSV".equalsIgnoreCase(desc.getMeta().getStoreType()));
KeyValueSet options = desc.getMeta().getOptions();
assertNotNull(options);
assertEquals(
StringEscapeUtils.escapeJava("\u0001"), options.get(StorageConstants.TEXT_DELIMITER));
}
@Test
public final void testSelfDescTable2() throws Exception {
executeString("create database d10;").close();
String className = getClass().getSimpleName();
Path currentDatasetPath = new Path(datasetBasePath, className);
Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
String sql =
"create external table d10.schemaless (*) using json with ('compression.codec'='none') location '"
+ filePath.toString()
+ "'";
executeString(sql).close();
assertTableExists("d10.schemaless");
TableDesc desc = getClient().getTableDesc("d10.schemaless");
assertTrue(desc.hasEmptySchema());
executeString("drop table d10.schemaless").close();
executeString("drop database d10").close();
}
@Override
public void execute(
QueryMaster.QueryMasterContext context,
QueryContext queryContext,
Query query,
ExecutionBlockId finalExecBlockId,
Path finalOutputDir)
throws Exception {
SubQuery lastStage = query.getSubQuery(finalExecBlockId);
TableMeta meta = lastStage.getTableMeta();
TableStats stats = lastStage.getTableStat();
TableDesc resultTableDesc =
new TableDesc(query.getId().toString(), lastStage.getSchema(), meta, finalOutputDir);
stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir));
resultTableDesc.setStats(stats);
query.setResultDesc(resultTableDesc);
}
@Test
public final void testGetTableDesc() throws IOException, ServiceException, SQLException {
final String tableName1 = "table3";
Path tablePath = writeTmpTable(tableName1);
LOG.error("Full path:" + tablePath.toUri().getRawPath());
FileSystem fs = tablePath.getFileSystem(conf);
assertTrue(fs.exists(tablePath));
assertNotNull(tablePath);
assertFalse(client.existTable(tableName1));
client.createExternalTable(
"table3", BackendTestingUtil.mockupSchema, tablePath, BackendTestingUtil.mockupMeta);
assertTrue(client.existTable(tableName1));
TableDesc desc = client.getTableDesc(tableName1);
assertNotNull(desc);
assertEquals(tableName1, desc.getName());
assertTrue(desc.getStats().getNumBytes() > 0);
}
@Test
public final void testRightOuter_MergeJoin3() throws IOException, TajoException {
Expr expr = analyzer.parse(QUERIES[3]);
LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
FileTablespace.splitNG(
conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getUri()), Integer.MAX_VALUE);
FileFragment[] dep4Frags =
FileTablespace.splitNG(
conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getUri()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags);
Path workDir =
CommonTestingUtil.getTestDir(
TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testRightOuter_MergeJoin3");
TaskAttemptContext ctx =
new TaskAttemptContext(
new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
ProjectionExec proj = (ProjectionExec) exec;
assertTrue(proj.getChild() instanceof RightOuterMergeJoinExec);
int count = 0;
exec.init();
while (exec.next() != null) {
// TODO check contents
count = count + 1;
}
assertNull(exec.next());
exec.close();
assertEquals(13, count);
}
@Test
public final void testCtasWithManagedTable() throws Exception {
ResultSet res = executeFile("CtasWithManagedTable.sql");
res.close();
if (testingCluster.isHiveCatalogStoreRunning()) {
assertTrue(client.existTable("managed_table1"));
TableDesc desc = client.getTableDesc("managed_table1");
assertNotNull(desc);
assertEquals("managed_table1", new Path(desc.getUri()).getName());
} else {
assertFalse(client.existTable("managed_Table1"));
assertTrue(client.existTable("MANAGED_TABLE1"));
TableDesc desc = client.getTableDesc("MANAGED_TABLE1");
assertNotNull(desc);
assertEquals("MANAGED_TABLE1", new Path(desc.getUri()).getName());
}
}
@Test
public void testAddAndDelIndex() throws Exception {
TableDesc desc = prepareTable();
prepareIndexDescs();
catalog.createTable(desc);
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
assertFalse(
catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"id"}));
catalog.createIndex(desc1);
assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
assertTrue(
catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"id"}));
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
assertFalse(
catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"score"}));
catalog.createIndex(desc2);
assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
assertTrue(
catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[] {"score"}));
Set<IndexDesc> indexDescs = new HashSet<>();
indexDescs.add(desc1);
indexDescs.add(desc2);
indexDescs.add(desc3);
for (IndexDesc index : catalog.getAllIndexesByTable(DEFAULT_DATABASE_NAME, "indexed")) {
assertTrue(indexDescs.contains(index));
}
catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getName());
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
catalog.dropIndex(DEFAULT_DATABASE_NAME, desc2.getName());
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
catalog.dropTable(desc.getName());
assertFalse(catalog.existsTable(desc.getName()));
}
@Test
public void testAlterTableAddDropPartition() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition");
tajoCli.executeScript(
"create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)");
tajoCli.executeScript("alter table " + tableName + " add partition (key2 = 0.1)");
tajoCli.executeScript("alter table " + tableName + " add partition (key = 0.1)");
tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)");
tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)");
tajoCli.executeScript("drop table " + tableName);
tajoCli.executeScript(
"create table "
+ tableName
+ " (col1 int4, col2 int4) partition by column(col3 float8, col4 int4)");
TajoClient client = testBase.getTestingCluster().newTajoClient();
TableDesc tableDesc = client.getTableDesc(tableName);
String partitionLocation = tableDesc.getUri().toString() + "/col5=0.1/col6=10";
tajoCli.executeScript(
"alter table "
+ tableName
+ " add partition (col3 = 0.1, col4 = 10)"
+ " location '"
+ partitionLocation
+ "'");
Path partitionPath = new Path(partitionLocation);
FileSystem fs = testBase.getTestingCluster().getDefaultFileSystem();
assertTrue(fs.exists(partitionPath));
tajoCli.executeScript("alter table " + tableName + " drop partition (col3 = 0.1, col4 = 10)");
String consoleResult = new String(out.toByteArray());
assertOutputResult(consoleResult);
}
@Test(timeout = 3000)
public void testNonForwardQueryPause() throws Exception {
final String sql = "select * from default.lineitem";
TajoCli cli = null;
try {
TableDesc tableDesc = cluster.getMaster().getCatalog().getTableDesc("default", "lineitem");
assertNotNull(tableDesc);
assertEquals(0L, tableDesc.getStats().getNumRows().longValue());
InputStream testInput =
new ByteArrayInputStream(new byte[] {(byte) DefaultTajoCliOutputFormatter.QUIT_COMMAND});
cli = new TajoCli(cluster.getConfiguration(), new String[] {}, testInput, out);
setVar(cli, SessionVars.CLI_PAGE_ROWS, "2");
setVar(cli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
cli.executeScript(sql);
String consoleResult;
consoleResult = new String(out.toByteArray());
assertOutputResult(consoleResult);
} finally {
cli.close();
}
}
@Override
public void execute(
QueryMaster.QueryMasterContext context,
QueryContext queryContext,
Query query,
ExecutionBlockId finalExecBlockId,
Path finalOutputDir)
throws Exception {
CatalogService catalog = context.getWorkerContext().getCatalog();
SubQuery lastStage = query.getSubQuery(finalExecBlockId);
TableMeta meta = lastStage.getTableMeta();
TableStats stats = lastStage.getTableStat();
InsertNode insertNode = (InsertNode) lastStage.getBlock().getPlan();
TableDesc finalTable;
if (insertNode.hasTargetTable()) {
String tableName = insertNode.getTableName();
finalTable = catalog.getTableDesc(tableName);
} else {
String tableName = query.getId().toString();
finalTable = new TableDesc(tableName, lastStage.getSchema(), meta, finalOutputDir);
}
long volume = getTableVolume(query.systemConf, finalOutputDir);
stats.setNumBytes(volume);
finalTable.setStats(stats);
if (insertNode.hasTargetTable()) {
catalog.deleteTable(insertNode.getTableName());
catalog.addTable(finalTable);
}
query.setResultDesc(finalTable);
}
@Test
public final void testAddAndDeleteTablePartitionByRange() throws Exception {
Schema schema =
SchemaBuilder.builder()
.add("id", Type.INT4)
.add("name", Type.TEXT)
.add("age", Type.INT4)
.add("score", Type.FLOAT8)
.build();
String tableName =
IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).build();
PartitionMethodDesc partitionDesc =
new PartitionMethodDesc(
DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.RANGE, "id", partSchema);
TableDesc desc =
new TableDesc(
tableName,
schema,
meta,
new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
assertTrue(catalog.existsTable(tableName));
TableDesc retrieved = catalog.getTableDesc(tableName);
assertEquals(retrieved.getName(), tableName);
assertEquals(
retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.RANGE);
assertEquals(
retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");
catalog.dropTable(tableName);
assertFalse(catalog.existsTable(tableName));
}
private final void assertPathOfCreatedTable(
final String databaseName,
final String originalTableName,
final String newTableName,
String createTableStmt)
throws Exception {
// create one table
executeString("CREATE DATABASE " + CatalogUtil.denormalizeIdentifier(databaseName)).close();
getClient().existDatabase(CatalogUtil.denormalizeIdentifier(databaseName));
final String oldFQTableName = CatalogUtil.buildFQName(databaseName, originalTableName);
ResultSet res = executeString(createTableStmt);
res.close();
assertTableExists(oldFQTableName);
TableDesc oldTableDesc = client.getTableDesc(oldFQTableName);
// checking the existence of the table directory and validating the path
Path warehouseDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration());
FileSystem fs = warehouseDir.getFileSystem(testingCluster.getConfiguration());
assertTrue(fs.exists(new Path(oldTableDesc.getUri())));
assertEquals(
StorageUtil.concatPath(warehouseDir, databaseName, originalTableName),
new Path(oldTableDesc.getUri()));
// Rename
client.executeQuery(
"ALTER TABLE "
+ CatalogUtil.denormalizeIdentifier(oldFQTableName)
+ " RENAME to "
+ CatalogUtil.denormalizeIdentifier(newTableName));
// checking the existence of the new table directory and validating the path
final String newFQTableName = CatalogUtil.buildFQName(databaseName, newTableName);
TableDesc newTableDesc = client.getTableDesc(newFQTableName);
assertTrue(fs.exists(new Path(newTableDesc.getUri())));
assertEquals(
StorageUtil.concatPath(warehouseDir, databaseName, newTableName),
new Path(newTableDesc.getUri()));
}
// TODO: This should be added at TAJO-1891
public final void testAddAndDeleteTablePartitionByColumn() throws Exception {
Schema schema =
SchemaBuilder.builder()
.add("id", Type.INT4)
.add("name", Type.TEXT)
.add("age", Type.INT4)
.add("score", Type.FLOAT8)
.build();
String simpleTableName = "addedtable";
String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName);
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).add("name", Type.TEXT).build();
PartitionMethodDesc partitionMethodDesc =
new PartitionMethodDesc(
DEFAULT_DATABASE_NAME,
tableName,
CatalogProtos.PartitionType.COLUMN,
"id,name",
partSchema);
TableDesc desc =
new TableDesc(
tableName,
schema,
meta,
new Path(CommonTestingUtil.getTestDir(), simpleTableName).toUri());
desc.setPartitionMethod(partitionMethodDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
assertTrue(catalog.existsTable(tableName));
TableDesc retrieved = catalog.getTableDesc(tableName);
assertEquals(retrieved.getName(), tableName);
assertEquals(
retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN);
assertEquals(
retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");
testAddPartition(tableName, "id=10/name=aaa");
testAddPartition(tableName, "id=20/name=bbb");
List<CatalogProtos.PartitionDescProto> partitions =
catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
assertNotNull(partitions);
assertEquals(partitions.size(), 2);
assertEquals(partitions.get(0).getNumBytes(), 0L);
testGetPartitionsByAlgebra(DEFAULT_DATABASE_NAME, simpleTableName);
testDropPartition(tableName, "id=10/name=aaa");
testDropPartition(tableName, "id=20/name=bbb");
partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
assertNotNull(partitions);
assertEquals(partitions.size(), 0);
catalog.dropTable(tableName);
assertFalse(catalog.existsTable(tableName));
}
@Test
public void testAlterTable() throws Exception {
// CREATE_TABLE
TableDesc tableRenameTestDesc = createMockupTable("default", "mycooltable");
catalog.createTable(tableRenameTestDesc);
// RENAME_TABLE
catalog.alterTable(createMockAlterTableName());
assertTrue(catalog.existsTable("default", "mynewcooltable"));
// RENAME_COLUMN
catalog.alterTable(createMockAlterTableRenameColumn());
TableDesc columnRenameDesc = catalog.getTableDesc("default", "mynewcooltable");
assertTrue(columnRenameDesc.getSchema().containsByName("ren" + FieldName1));
// ADD_COLUMN
catalog.alterTable(createMockAlterTableAddColumn());
TableDesc addColumnDesc = catalog.getTableDesc("default", "mynewcooltable");
assertTrue(addColumnDesc.getSchema().containsByName("mynewcol"));
// SET_PROPERTY
TableDesc setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
KeyValueSet options = new KeyValueSet();
options.set("timezone", "GMT+9"); // Seoul, Korea
setPropertyDesc.setMeta(new TableMeta("TEXT", options));
String prevTimeZone = setPropertyDesc.getMeta().getProperty("timezone");
String newTimeZone = "GMT-7"; // Silicon Valley, California
catalog.alterTable(createMockAlterTableSetProperty(newTimeZone));
setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
assertNotEquals(prevTimeZone, setPropertyDesc.getMeta().getProperty("timezone"));
assertEquals(newTimeZone, setPropertyDesc.getMeta().getProperty("timezone"));
// UNSET_PROPERTY
catalog.alterTable(createMockAlterTableUnsetProperty(Sets.newHashSet("dummy")));
setPropertyDesc = catalog.getTableDesc("default", "mynewcooltable");
assertTrue(setPropertyDesc.getMeta().getPropertySet().containsKey("timezone"));
assertFalse(setPropertyDesc.getMeta().getPropertySet().containsKey("dummy"));
}
@Override
public final CatalogProtos.TableDescProto getTable(String databaseName, final String tableName)
throws CatalogException {
org.apache.hadoop.hive.ql.metadata.Table table = null;
HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
Path path = null;
String storeType = null;
org.apache.tajo.catalog.Schema schema = null;
KeyValueSet options = null;
TableStats stats = null;
PartitionMethodDesc partitions = null;
//////////////////////////////////
// set tajo table schema.
//////////////////////////////////
try {
// get hive table schema
try {
client = clientPool.getClient();
table = HiveCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName);
path = table.getPath();
} catch (NoSuchObjectException nsoe) {
throw new UndefinedTableException(tableName);
} catch (Exception e) {
throw new TajoInternalError(e);
}
// convert HiveCatalogStore field schema into tajo field schema.
schema = new org.apache.tajo.catalog.Schema();
List<FieldSchema> fieldSchemaList = table.getCols();
boolean isPartitionKey = false;
for (FieldSchema eachField : fieldSchemaList) {
isPartitionKey = false;
if (table.getPartitionKeys() != null) {
for (FieldSchema partitionKey : table.getPartitionKeys()) {
if (partitionKey.getName().equals(eachField.getName())) {
isPartitionKey = true;
}
}
}
if (!isPartitionKey) {
String fieldName =
databaseName
+ CatalogConstants.IDENTIFIER_DELIMITER
+ tableName
+ CatalogConstants.IDENTIFIER_DELIMITER
+ eachField.getName();
TajoDataTypes.Type dataType =
HiveCatalogUtil.getTajoFieldType(eachField.getType().toString());
schema.addColumn(fieldName, dataType);
}
}
// validate field schema.
HiveCatalogUtil.validateSchema(table);
stats = new TableStats();
options = new KeyValueSet();
options.putAll(table.getParameters());
options.remove("EXTERNAL");
Properties properties = table.getMetadata();
if (properties != null) {
// set field delimiter
String fieldDelimiter = "", nullFormat = "";
if (properties.getProperty(serdeConstants.FIELD_DELIM) != null) {
fieldDelimiter = properties.getProperty(serdeConstants.FIELD_DELIM);
} else {
// if hive table used default row format delimiter, Properties doesn't have it.
// So, Tajo must set as follows:
fieldDelimiter = "\u0001";
}
// set null format
if (properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT) != null) {
nullFormat = properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT);
} else {
nullFormat = "\\N";
}
options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT);
// set file output format
String fileOutputformat =
properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT);
storeType = HiveCatalogUtil.getStoreType(fileOutputformat);
if (storeType.equalsIgnoreCase("TEXT")) {
options.set(
StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat));
} else if (storeType.equals("RCFILE")) {
options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) {
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (ColumnarSerDe.class.getName().equals(serde)) {
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
} else if (storeType.equals("SEQUENCEFILE")) {
options.set(
StorageConstants.SEQUENCEFILE_DELIMITER,
StringEscapeUtils.escapeJava(fieldDelimiter));
options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinarySerDe.class.getName().equals(serde)) {
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (LazySimpleSerDe.class.getName().equals(serde)) {
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
}
// set data size
long totalSize = 0;
if (properties.getProperty("totalSize") != null) {
totalSize = Long.parseLong(properties.getProperty("totalSize"));
} else {
try {
FileSystem fs = path.getFileSystem(conf);
if (fs.exists(path)) {
totalSize = fs.getContentSummary(path).getLength();
}
} catch (IOException ioe) {
throw new TajoInternalError(ioe);
}
}
stats.setNumBytes(totalSize);
}
// set partition keys
List<FieldSchema> partitionKeys = table.getPartitionKeys();
if (null != partitionKeys) {
org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema();
StringBuilder sb = new StringBuilder();
if (partitionKeys.size() > 0) {
for (int i = 0; i < partitionKeys.size(); i++) {
FieldSchema fieldSchema = partitionKeys.get(i);
TajoDataTypes.Type dataType =
HiveCatalogUtil.getTajoFieldType(fieldSchema.getType().toString());
String fieldName =
databaseName
+ CatalogConstants.IDENTIFIER_DELIMITER
+ tableName
+ CatalogConstants.IDENTIFIER_DELIMITER
+ fieldSchema.getName();
expressionSchema.addColumn(new Column(fieldName, dataType));
if (i > 0) {
sb.append(",");
}
sb.append(fieldSchema.getName());
}
partitions =
new PartitionMethodDesc(
databaseName, tableName, PartitionType.COLUMN, sb.toString(), expressionSchema);
}
}
} finally {
if (client != null) client.release();
}
TableMeta meta = new TableMeta(storeType, options);
TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri());
if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) {
tableDesc.setExternal(true);
}
if (stats != null) {
tableDesc.setStats(stats);
}
if (partitions != null) {
tableDesc.setPartitionMethod(partitions);
}
return tableDesc.getProto();
}
@Override
public final void createTable(final CatalogProtos.TableDescProto tableDescProto)
throws CatalogException {
HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
TableDesc tableDesc = new TableDesc(tableDescProto);
String[] splitted = CatalogUtil.splitFQTableName(tableDesc.getName());
String databaseName = splitted[0];
String tableName = splitted[1];
try {
client = clientPool.getClient();
org.apache.hadoop.hive.metastore.api.Table table =
new org.apache.hadoop.hive.metastore.api.Table();
table.setDbName(databaseName);
table.setTableName(tableName);
table.setParameters(
new HashMap<String, String>(tableDesc.getMeta().getOptions().getAllKeyValus()));
// TODO: set owner
// table.setOwner();
StorageDescriptor sd = new StorageDescriptor();
sd.setSerdeInfo(new SerDeInfo());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.getSerdeInfo().setName(table.getTableName());
// if tajo set location method, thrift client make exception as follows:
// Caused by: MetaException(message:java.lang.NullPointerException)
// If you want to modify table path, you have to modify on Hive cli.
if (tableDesc.isExternal()) {
table.setTableType(TableType.EXTERNAL_TABLE.name());
table.putToParameters("EXTERNAL", "TRUE");
Path tablePath = new Path(tableDesc.getUri());
FileSystem fs = tablePath.getFileSystem(conf);
if (fs.isFile(tablePath)) {
LOG.warn("A table path is a file, but HiveCatalogStore does not allow a file path.");
sd.setLocation(tablePath.getParent().toString());
} else {
sd.setLocation(tablePath.toString());
}
}
// set column information
List<Column> columns = tableDesc.getSchema().getRootColumns();
ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(columns.size());
for (Column eachField : columns) {
cols.add(
new FieldSchema(
eachField.getSimpleName(),
HiveCatalogUtil.getHiveFieldType(eachField.getDataType()),
""));
}
sd.setCols(cols);
// set partition keys
if (tableDesc.hasPartition()
&& tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
List<FieldSchema> partitionKeys = new ArrayList<FieldSchema>();
for (Column eachPartitionKey :
tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionKeys.add(
new FieldSchema(
eachPartitionKey.getSimpleName(),
HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()),
""));
}
table.setPartitionKeys(partitionKeys);
}
if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.RCFILE)) {
String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
} else {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName());
}
if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) {
table.putToParameters(
serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(
tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
}
} else if (tableDesc.getMeta().getStoreType().equals(BuiltinStorages.TEXT)) {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
sd.setOutputFormat(
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());
String fieldDelimiter =
tableDesc
.getMeta()
.getOption(
StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
// And hive will un-espace this value again.
// As a result, user can use right field delimiter.
// So, we have to un-escape this value.
sd.getSerdeInfo()
.putToParameters(
serdeConstants.SERIALIZATION_FORMAT,
StringEscapeUtils.unescapeJava(fieldDelimiter));
sd.getSerdeInfo()
.putToParameters(
serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter));
table.getParameters().remove(StorageConstants.TEXT_DELIMITER);
if (tableDesc.getMeta().containsOption(StorageConstants.TEXT_NULL)) {
table.putToParameters(
serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(
tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL)));
table.getParameters().remove(StorageConstants.TEXT_NULL);
}
} else if (tableDesc
.getMeta()
.getStoreType()
.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName());
sd.setOutputFormat(
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName());
if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
String fieldDelimiter =
tableDesc
.getMeta()
.getOption(
StorageConstants.SEQUENCEFILE_DELIMITER,
StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
// And hive will un-espace this value again.
// As a result, user can use right field delimiter.
// So, we have to un-escape this value.
sd.getSerdeInfo()
.putToParameters(
serdeConstants.SERIALIZATION_FORMAT,
StringEscapeUtils.unescapeJava(fieldDelimiter));
sd.getSerdeInfo()
.putToParameters(
serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter));
table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER);
} else {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
}
if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) {
table.putToParameters(
serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(
tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
}
} else {
if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.PARQUET)) {
sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName());
sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName());
sd.getSerdeInfo()
.setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName());
} else {
throw new UnsupportedException(
tableDesc.getMeta().getStoreType() + " in HivecatalogStore");
}
}
sd.setSortCols(new ArrayList<Order>());
table.setSd(sd);
client.getHiveClient().createTable(table);
} catch (Throwable t) {
throw new TajoInternalError(t);
} finally {
if (client != null) client.release();
}
}
private boolean isClonedTable(String orignalTable, String newTable) throws Exception {
assertTableExists(newTable);
TableDesc origTableDesc = client.getTableDesc(orignalTable);
TableDesc newTableDesc = client.getTableDesc(newTable);
if (isClonedSchema(origTableDesc.getSchema(), newTableDesc.getSchema()) == false) {
fail("Schema of input tables do not match");
return false;
}
// Check partition information
PartitionMethodDesc origPartMethod = origTableDesc.getPartitionMethod();
PartitionMethodDesc newPartMethod = newTableDesc.getPartitionMethod();
if (origPartMethod != null) {
if (newPartMethod == null) {
fail("New table does not have partition info");
return false;
}
if (isClonedSchema(origPartMethod.getExpressionSchema(), newPartMethod.getExpressionSchema())
== false) {
fail("Partition columns of input tables do not match");
return false;
}
if (origPartMethod.getPartitionType().equals(newPartMethod.getPartitionType()) == false) {
fail("Partition type of input tables do not match");
return false;
}
}
// Check external flag
if (origTableDesc.isExternal() != newTableDesc.isExternal()) {
fail("External table flag on input tables not equal");
return false;
}
if (origTableDesc.getMeta() != null) {
TableMeta origMeta = origTableDesc.getMeta();
TableMeta newMeta = newTableDesc.getMeta();
if (origMeta.getDataFormat().equals(newMeta.getDataFormat()) == false) {
fail("Store type of input tables not equal");
return false;
}
KeyValueSet origOptions = origMeta.getPropertySet();
KeyValueSet newOptions = newMeta.getPropertySet();
if (origOptions.equals(newOptions) == false) {
fail("Meta options of input tables not equal");
return false;
}
}
return true;
}
/** Insert row values */
private void insertRowValues(
QueryContext queryContext,
InsertNode insertNode,
SubmitQueryResponse.Builder responseBuilder) {
try {
String nodeUniqName =
insertNode.getTableName() == null
? new Path(insertNode.getUri()).getName()
: insertNode.getTableName();
String queryId = nodeUniqName + "_" + System.currentTimeMillis();
URI finalOutputUri = insertNode.getUri();
Tablespace space = TablespaceManager.get(finalOutputUri).get();
TableMeta tableMeta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions());
tableMeta.putOption(StorageConstants.INSERT_DIRECTLY, Boolean.TRUE.toString());
FormatProperty formatProperty = space.getFormatProperty(tableMeta);
TaskAttemptContext taskAttemptContext;
if (formatProperty
.directInsertSupported()) { // if this format and storage supports direct insertion
taskAttemptContext = new TaskAttemptContext(queryContext, null, null, null, null);
taskAttemptContext.setOutputPath(new Path(finalOutputUri));
EvalExprExec evalExprExec =
new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild());
InsertRowsExec exec = new InsertRowsExec(taskAttemptContext, insertNode, evalExprExec);
try {
exec.init();
exec.next();
} finally {
exec.close();
}
} else {
URI stagingSpaceUri =
space.prepareStagingSpace(context.getConf(), queryId, queryContext, tableMeta);
Path stagingDir = new Path(stagingSpaceUri);
Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME);
taskAttemptContext = new TaskAttemptContext(queryContext, null, null, null, stagingDir);
taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000"));
insertRowsThroughStaging(
taskAttemptContext, insertNode, new Path(finalOutputUri), stagingDir, stagingResultDir);
}
// set insert stats (how many rows and bytes)
TableStats stats = new TableStats();
stats.setNumBytes(taskAttemptContext.getResultStats().getNumBytes());
stats.setNumRows(taskAttemptContext.getResultStats().getNumRows());
if (insertNode.hasTargetTable()) {
CatalogProtos.UpdateTableStatsProto.Builder builder =
CatalogProtos.UpdateTableStatsProto.newBuilder();
builder.setTableName(insertNode.getTableName());
builder.setStats(stats.getProto());
catalog.updateTableStats(builder.build());
TableDesc desc =
new TableDesc(
insertNode.getTableName(), insertNode.getTargetSchema(), tableMeta, finalOutputUri);
responseBuilder.setTableDesc(desc.getProto());
} else { // If INSERT INTO LOCATION
// Empty TableDesc
List<CatalogProtos.ColumnProto> columns = new ArrayList<CatalogProtos.ColumnProto>();
CatalogProtos.TableDescProto tableDescProto =
CatalogProtos.TableDescProto.newBuilder()
.setTableName(nodeUniqName)
.setMeta(
CatalogProtos.TableProto.newBuilder()
.setStoreType(BuiltinStorages.TEXT)
.build())
.setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build())
.setStats(stats.getProto())
.build();
responseBuilder.setTableDesc(tableDescProto);
}
// If queryId == NULL_QUERY_ID and MaxRowNum == -1, TajoCli prints only number of inserted
// rows.
responseBuilder.setMaxRowNum(-1);
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
responseBuilder.setResultType(ResultType.NO_RESULT);
responseBuilder.setState(OK);
} catch (Throwable t) {
throw new RuntimeException(t);
}
}
