@Override public void execute( QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { CatalogService catalog = context.getWorkerContext().getCatalog(); SubQuery lastStage = query.getSubQuery(finalExecBlockId); TableMeta meta = lastStage.getTableMeta(); TableStats stats = lastStage.getTableStat(); CreateTableNode createTableNode = (CreateTableNode) lastStage.getBlock().getPlan(); TableDesc tableDescTobeCreated = new TableDesc( createTableNode.getTableName(), createTableNode.getTableSchema(), meta, finalOutputDir); if (createTableNode.hasPartition()) { tableDescTobeCreated.setPartitionMethod(createTableNode.getPartitionMethod()); } stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir)); tableDescTobeCreated.setStats(stats); query.setResultDesc(tableDescTobeCreated); catalog.addTable(tableDescTobeCreated); }
@Override public void execute( QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { SubQuery lastStage = query.getSubQuery(finalExecBlockId); TableMeta meta = lastStage.getTableMeta(); TableStats stats = lastStage.getTableStat(); TableDesc resultTableDesc = new TableDesc(query.getId().toString(), lastStage.getSchema(), meta, finalOutputDir); stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir)); resultTableDesc.setStats(stats); query.setResultDesc(resultTableDesc); }
@Override public void execute( QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { CatalogService catalog = context.getWorkerContext().getCatalog(); SubQuery lastStage = query.getSubQuery(finalExecBlockId); TableMeta meta = lastStage.getTableMeta(); TableStats stats = lastStage.getTableStat(); InsertNode insertNode = (InsertNode) lastStage.getBlock().getPlan(); TableDesc finalTable; if (insertNode.hasTargetTable()) { String tableName = insertNode.getTableName(); finalTable = catalog.getTableDesc(tableName); } else { String tableName = query.getId().toString(); finalTable = new TableDesc(tableName, lastStage.getSchema(), meta, finalOutputDir); } long volume = getTableVolume(query.systemConf, finalOutputDir); stats.setNumBytes(volume); finalTable.setStats(stats); if (insertNode.hasTargetTable()) { catalog.deleteTable(insertNode.getTableName()); catalog.addTable(finalTable); } query.setResultDesc(finalTable); }
@Override public final CatalogProtos.TableDescProto getTable(String databaseName, final String tableName) throws CatalogException { org.apache.hadoop.hive.ql.metadata.Table table = null; HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; Path path = null; String storeType = null; org.apache.tajo.catalog.Schema schema = null; KeyValueSet options = null; TableStats stats = null; PartitionMethodDesc partitions = null; ////////////////////////////////// // set tajo table schema. ////////////////////////////////// try { // get hive table schema try { client = clientPool.getClient(); table = HiveCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); path = table.getPath(); } catch (NoSuchObjectException nsoe) { throw new UndefinedTableException(tableName); } catch (Exception e) { throw new TajoInternalError(e); } // convert HiveCatalogStore field schema into tajo field schema. schema = new org.apache.tajo.catalog.Schema(); List<FieldSchema> fieldSchemaList = table.getCols(); boolean isPartitionKey = false; for (FieldSchema eachField : fieldSchemaList) { isPartitionKey = false; if (table.getPartitionKeys() != null) { for (FieldSchema partitionKey : table.getPartitionKeys()) { if (partitionKey.getName().equals(eachField.getName())) { isPartitionKey = true; } } } if (!isPartitionKey) { String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName(); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(eachField.getType().toString()); schema.addColumn(fieldName, dataType); } } // validate field schema. HiveCatalogUtil.validateSchema(table); stats = new TableStats(); options = new KeyValueSet(); options.putAll(table.getParameters()); options.remove("EXTERNAL"); Properties properties = table.getMetadata(); if (properties != null) { // set field delimiter String fieldDelimiter = "", nullFormat = ""; if (properties.getProperty(serdeConstants.FIELD_DELIM) != null) { fieldDelimiter = properties.getProperty(serdeConstants.FIELD_DELIM); } else { // if hive table used default row format delimiter, Properties doesn't have it. // So, Tajo must set as follows: fieldDelimiter = "\u0001"; } // set null format if (properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT) != null) { nullFormat = properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT); } else { nullFormat = "\\N"; } options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT); // set file output format String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); storeType = HiveCatalogUtil.getStoreType(fileOutputformat); if (storeType.equalsIgnoreCase("TEXT")) { options.set( StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat)); } else if (storeType.equals("RCFILE")) { options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); } else if (ColumnarSerDe.class.getName().equals(serde)) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } } else if (storeType.equals("SEQUENCEFILE")) { options.set( StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinarySerDe.class.getName().equals(serde)) { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); } else if (LazySimpleSerDe.class.getName().equals(serde)) { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } } // set data size long totalSize = 0; if (properties.getProperty("totalSize") != null) { totalSize = Long.parseLong(properties.getProperty("totalSize")); } else { try { FileSystem fs = path.getFileSystem(conf); if (fs.exists(path)) { totalSize = fs.getContentSummary(path).getLength(); } } catch (IOException ioe) { throw new TajoInternalError(ioe); } } stats.setNumBytes(totalSize); } // set partition keys List<FieldSchema> partitionKeys = table.getPartitionKeys(); if (null != partitionKeys) { org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema(); StringBuilder sb = new StringBuilder(); if (partitionKeys.size() > 0) { for (int i = 0; i < partitionKeys.size(); i++) { FieldSchema fieldSchema = partitionKeys.get(i); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType().toString()); String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); expressionSchema.addColumn(new Column(fieldName, dataType)); if (i > 0) { sb.append(","); } sb.append(fieldSchema.getName()); } partitions = new PartitionMethodDesc( databaseName, tableName, PartitionType.COLUMN, sb.toString(), expressionSchema); } } } finally { if (client != null) client.release(); } TableMeta meta = new TableMeta(storeType, options); TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri()); if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) { tableDesc.setExternal(true); } if (stats != null) { tableDesc.setStats(stats); } if (partitions != null) { tableDesc.setPartitionMethod(partitions); } return tableDesc.getProto(); }