public static Table fromMetastoreApiTable(org.apache.hadoop.hive.metastore.api.Table table) {
StorageDescriptor storageDescriptor = table.getSd();
if (storageDescriptor == null) {
throw new PrestoException(HIVE_INVALID_METADATA, "Table is missing storage descriptor");
}
Table.Builder tableBuilder =
Table.builder()
.setDatabaseName(table.getDbName())
.setTableName(table.getTableName())
.setOwner(nullToEmpty(table.getOwner()))
.setTableType(table.getTableType())
.setDataColumns(
storageDescriptor
.getCols()
.stream()
.map(MetastoreUtil::fromMetastoreApiFieldSchema)
.collect(toList()))
.setPartitionColumns(
table
.getPartitionKeys()
.stream()
.map(MetastoreUtil::fromMetastoreApiFieldSchema)
.collect(toList()))
.setParameters(
table.getParameters() == null ? ImmutableMap.of() : table.getParameters())
.setViewOriginalText(Optional.ofNullable(emptyToNull(table.getViewOriginalText())))
.setViewExpandedText(Optional.ofNullable(emptyToNull(table.getViewExpandedText())));
fromMetastoreApiStorageDescriptor(
storageDescriptor, tableBuilder.getStorageBuilder(), table.getTableName());
return tableBuilder.build();
}
@Override
public synchronized void createTable(Table table) {
SchemaTableName schemaTableName = new SchemaTableName(table.getDbName(), table.getTableName());
Table tableCopy = table.deepCopy();
if (tableCopy.getSd() == null) {
tableCopy.setSd(new StorageDescriptor());
} else if (tableCopy.getSd().getLocation() != null) {
File directory = new File(new Path(tableCopy.getSd().getLocation()).toUri());
checkArgument(directory.exists(), "Table directory does not exist");
checkArgument(
isParentDir(directory, baseDirectory),
"Table directory must be inside of the metastore base directory");
}
if (relations.putIfAbsent(schemaTableName, tableCopy) != null) {
throw new TableAlreadyExistsException(schemaTableName);
}
if (tableCopy.getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
views.put(schemaTableName, tableCopy);
}
PrincipalPrivilegeSet privileges = table.getPrivileges();
if (privileges != null) {
for (Entry<String, List<PrivilegeGrantInfo>> entry :
privileges.getUserPrivileges().entrySet()) {
String user = entry.getKey();
Set<HivePrivilegeInfo> userPrivileges =
entry
.getValue()
.stream()
.map(HivePrivilegeInfo::parsePrivilege)
.flatMap(Collection::stream)
.collect(toImmutableSet());
setTablePrivileges(user, USER, table.getDbName(), table.getTableName(), userPrivileges);
}
for (Entry<String, List<PrivilegeGrantInfo>> entry :
privileges.getRolePrivileges().entrySet()) {
String role = entry.getKey();
Set<HivePrivilegeInfo> rolePrivileges =
entry
.getValue()
.stream()
.map(HivePrivilegeInfo::parsePrivilege)
.flatMap(Collection::stream)
.collect(toImmutableSet());
setTablePrivileges(role, ROLE, table.getDbName(), table.getTableName(), rolePrivileges);
}
}
}
@Override
public void alterTable(String databaseName, String tableName, Table newTable) {
SchemaTableName oldName = new SchemaTableName(databaseName, tableName);
SchemaTableName newName = new SchemaTableName(newTable.getDbName(), newTable.getTableName());
// if the name did not change, this is a simple schema change
if (oldName.equals(newName)) {
if (relations.replace(oldName, newTable) == null) {
throw new TableNotFoundException(oldName);
}
return;
}
// remove old table definition and add the new one
// TODO: use locking to do this properly
Table table = relations.get(oldName);
if (table == null) {
throw new TableNotFoundException(oldName);
}
if (relations.putIfAbsent(newName, newTable) != null) {
throw new TableAlreadyExistsException(newName);
}
relations.remove(oldName);
}
/**
* Removes the cache directive associated with the table from HDFS, uncaching all data. Also
* updates the table's metadata. No-op if the table is not cached.
*/
public static void uncacheTbl(org.apache.hadoop.hive.metastore.api.Table table)
throws ImpalaRuntimeException {
Preconditions.checkNotNull(table);
LOG.debug("Uncaching table: " + table.getDbName() + "." + table.getTableName());
Long id = getCacheDirectiveId(table.getParameters());
if (id == null) return;
HdfsCachingUtil.removeDirective(id);
table.getParameters().remove(CACHE_DIR_ID_PROP_NAME);
table.getParameters().remove(CACHE_DIR_REPLICATION_PROP_NAME);
}
@Override
public void createTable(Table table) throws HiveMetastoreException {
if (!existsDb(table.getDbName())) {
throw new HiveMetastoreException("DB " + table.getDbName() + " does not exist!");
}
HiveObjectSpec tableSpec = new HiveObjectSpec(table.getDbName(), table.getTableName());
if (specToTable.containsKey(tableSpec)) {
throw new HiveMetastoreException("Table already exists: " + tableSpec);
}
specToTable.put(tableSpec, table);
}
@Before
public void createTable() throws Exception {
// Use Junit's Assume to skip running this fixture against any storage formats whose
// SerDe is in the disabled serdes list.
Assume.assumeTrue(!DISABLED_SERDES.contains(serdeClass));
String databaseName = (dbName == null) ? MetaStoreUtils.DEFAULT_DATABASE_NAME : dbName;
try {
client.dropTable(databaseName, tableName);
} catch (Exception e) {
// Can fail with NoSuchObjectException.
}
Table tbl = new Table();
tbl.setDbName(databaseName);
tbl.setTableName(tableName);
if (isTableExternal()) {
tbl.setTableType(TableType.EXTERNAL_TABLE.toString());
} else {
tbl.setTableType(TableType.MANAGED_TABLE.toString());
}
StorageDescriptor sd = new StorageDescriptor();
sd.setCols(getTableColumns());
tbl.setPartitionKeys(getPartitionKeys());
tbl.setSd(sd);
sd.setBucketCols(new ArrayList<String>(2));
sd.setSerdeInfo(new SerDeInfo());
sd.getSerdeInfo().setName(tbl.getTableName());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
if (isTableExternal()) {
sd.getSerdeInfo().getParameters().put("EXTERNAL", "TRUE");
}
sd.getSerdeInfo().setSerializationLib(serdeClass);
sd.setInputFormat(inputFormatClass);
sd.setOutputFormat(outputFormatClass);
Map<String, String> tableParams = new HashMap<String, String>();
if (isTableExternal()) {
tableParams.put("EXTERNAL", "TRUE");
}
if (isTableImmutable()) {
tableParams.put(hive_metastoreConstants.IS_IMMUTABLE, "true");
}
StatsSetupConst.setBasicStatsState(tableParams, StatsSetupConst.TRUE);
tbl.setParameters(tableParams);
client.createTable(tbl);
}
@Override
public void alterTable(String dbName, String tableName, Table table)
throws HiveMetastoreException {
HiveObjectSpec existingTableSpec = new HiveObjectSpec(dbName, tableName);
HiveObjectSpec newTableSpec = new HiveObjectSpec(table.getDbName(), table.getTableName());
if (!specToTable.containsKey(existingTableSpec)) {
throw new HiveMetastoreException("Unknown table: " + existingTableSpec);
}
Table removedTable = specToTable.remove(existingTableSpec);
if (removedTable == null) {
throw new RuntimeException("Shouldn't happen!");
}
specToTable.put(newTableSpec, table);
}
@Override
public void createTable(Table table) {
try {
retry()
.stopOn(
AlreadyExistsException.class,
InvalidObjectException.class,
MetaException.class,
NoSuchObjectException.class)
.stopOnIllegalExceptions()
.run(
"createTable",
stats
.getCreateTable()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
client.createTable(table);
}
return null;
}));
} catch (AlreadyExistsException e) {
throw new TableAlreadyExistsException(
new SchemaTableName(table.getDbName(), table.getTableName()));
} catch (NoSuchObjectException e) {
throw new SchemaNotFoundException(table.getDbName());
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
throw Throwables.propagate(e);
} finally {
invalidateTable(table.getDbName(), table.getTableName());
}
}
// This method is completely copied from Hive's HBaseStorageHandler.java.
private String getHBaseTableName(org.apache.hadoop.hive.metastore.api.Table tbl) {
// Give preference to TBLPROPERTIES over SERDEPROPERTIES
// (really we should only use TBLPROPERTIES, so this is just
// for backwards compatibility with the original specs).
String tableName = tbl.getParameters().get(HBaseSerDe.HBASE_TABLE_NAME);
if (tableName == null) {
tableName = tbl.getSd().getSerdeInfo().getParameters().get(HBaseSerDe.HBASE_TABLE_NAME);
}
if (tableName == null) {
tableName = tbl.getDbName() + "." + tbl.getTableName();
if (tableName.startsWith(DEFAULT_PREFIX)) {
tableName = tableName.substring(DEFAULT_PREFIX.length());
}
}
return tableName;
}
private static void createTable(String tableName, String tablePerm) throws Exception {
Table tbl = new Table();
tbl.setDbName(DATABASE);
tbl.setTableName(tableName);
StorageDescriptor sd = new StorageDescriptor();
sd.setCols(ColumnHolder.colMapping.get(tableName));
tbl.setSd(sd);
sd.setParameters(new HashMap<String, String>());
sd.setSerdeInfo(new SerDeInfo());
sd.getSerdeInfo().setName(tbl.getTableName());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
sd.getSerdeInfo()
.setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
tbl.setPartitionKeys(ColumnHolder.partitionCols);
hmsc.createTable(tbl);
FileSystem fs = FileSystem.get(mrConf);
fs.setPermission(new Path(warehousedir, tableName), new FsPermission(tablePerm));
}
private void validateTable(Table expectedTable, Table actualTable) {
assertEquals(expectedTable.getTableName(), actualTable.getTableName());
assertEquals(expectedTable.getDbName(), actualTable.getDbName());
assertEquals(expectedTable.getSd().getLocation(), actualTable.getSd().getLocation());
}
@Override
public final void createTable(final CatalogProtos.TableDescProto tableDescProto)
throws CatalogException {
HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
TableDesc tableDesc = new TableDesc(tableDescProto);
String[] splitted = CatalogUtil.splitFQTableName(tableDesc.getName());
String databaseName = splitted[0];
String tableName = splitted[1];
try {
client = clientPool.getClient();
org.apache.hadoop.hive.metastore.api.Table table =
new org.apache.hadoop.hive.metastore.api.Table();
table.setDbName(databaseName);
table.setTableName(tableName);
table.setParameters(
new HashMap<String, String>(tableDesc.getMeta().getOptions().getAllKeyValus()));
// TODO: set owner
// table.setOwner();
StorageDescriptor sd = new StorageDescriptor();
sd.setSerdeInfo(new SerDeInfo());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.getSerdeInfo().setName(table.getTableName());
// if tajo set location method, thrift client make exception as follows:
// Caused by: MetaException(message:java.lang.NullPointerException)
// If you want to modify table path, you have to modify on Hive cli.
if (tableDesc.isExternal()) {
table.setTableType(TableType.EXTERNAL_TABLE.name());
table.putToParameters("EXTERNAL", "TRUE");
Path tablePath = new Path(tableDesc.getUri());
FileSystem fs = tablePath.getFileSystem(conf);
if (fs.isFile(tablePath)) {
LOG.warn("A table path is a file, but HiveCatalogStore does not allow a file path.");
sd.setLocation(tablePath.getParent().toString());
} else {
sd.setLocation(tablePath.toString());
}
}
// set column information
List<Column> columns = tableDesc.getSchema().getRootColumns();
ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(columns.size());
for (Column eachField : columns) {
cols.add(
new FieldSchema(
eachField.getSimpleName(),
HiveCatalogUtil.getHiveFieldType(eachField.getDataType()),
""));
}
sd.setCols(cols);
// set partition keys
if (tableDesc.hasPartition()
&& tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
List<FieldSchema> partitionKeys = new ArrayList<FieldSchema>();
for (Column eachPartitionKey :
tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionKeys.add(
new FieldSchema(
eachPartitionKey.getSimpleName(),
HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()),
""));
}
table.setPartitionKeys(partitionKeys);
}
if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.RCFILE)) {
String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
} else {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName());
}
if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) {
table.putToParameters(
serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(
tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
}
} else if (tableDesc.getMeta().getStoreType().equals(BuiltinStorages.TEXT)) {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
sd.setOutputFormat(
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());
String fieldDelimiter =
tableDesc
.getMeta()
.getOption(
StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
// And hive will un-espace this value again.
// As a result, user can use right field delimiter.
// So, we have to un-escape this value.
sd.getSerdeInfo()
.putToParameters(
serdeConstants.SERIALIZATION_FORMAT,
StringEscapeUtils.unescapeJava(fieldDelimiter));
sd.getSerdeInfo()
.putToParameters(
serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter));
table.getParameters().remove(StorageConstants.TEXT_DELIMITER);
if (tableDesc.getMeta().containsOption(StorageConstants.TEXT_NULL)) {
table.putToParameters(
serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(
tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL)));
table.getParameters().remove(StorageConstants.TEXT_NULL);
}
} else if (tableDesc
.getMeta()
.getStoreType()
.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName());
sd.setOutputFormat(
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName());
if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
String fieldDelimiter =
tableDesc
.getMeta()
.getOption(
StorageConstants.SEQUENCEFILE_DELIMITER,
StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
// And hive will un-espace this value again.
// As a result, user can use right field delimiter.
// So, we have to un-escape this value.
sd.getSerdeInfo()
.putToParameters(
serdeConstants.SERIALIZATION_FORMAT,
StringEscapeUtils.unescapeJava(fieldDelimiter));
sd.getSerdeInfo()
.putToParameters(
serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter));
table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER);
} else {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
}
if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) {
table.putToParameters(
serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(
tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
}
} else {
if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.PARQUET)) {
sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName());
sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName());
sd.getSerdeInfo()
.setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName());
} else {
throw new UnsupportedException(
tableDesc.getMeta().getStoreType() + " in HivecatalogStore");
}
}
sd.setSortCols(new ArrayList<Order>());
table.setSd(sd);
client.getHiveClient().createTable(table);
} catch (Throwable t) {
throw new TajoInternalError(t);
} finally {
if (client != null) client.release();
}
}