@Override
public void dropPartition(String databaseName, String tableName, List<String> parts) {
try {
retry()
.stopOn(NoSuchObjectException.class, MetaException.class)
.stopOnIllegalExceptions()
.run(
"dropPartition",
stats
.getDropPartition()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
client.dropPartition(databaseName, tableName, parts, true);
}
return null;
}));
} catch (NoSuchObjectException e) {
throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
throw Throwables.propagate(e);
} finally {
invalidatePartitionCache(databaseName, tableName);
}
}
private Optional<Partition> loadPartitionByName(HivePartitionName partitionName)
throws Exception {
requireNonNull(partitionName, "partitionName is null");
try {
return retry()
.stopOn(NoSuchObjectException.class)
.stopOnIllegalExceptions()
.run(
"getPartitionsByNames",
stats
.getGetPartitionByName()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
return Optional.of(
client.getPartitionByName(
partitionName.getHiveTableName().getDatabaseName(),
partitionName.getHiveTableName().getTableName(),
partitionName.getPartitionName()));
}
}));
} catch (NoSuchObjectException e) {
return Optional.empty();
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
private Optional<List<String>> loadPartitionNamesByParts(PartitionFilter partitionFilter)
throws Exception {
try {
return retry()
.stopOn(NoSuchObjectException.class)
.stopOnIllegalExceptions()
.run(
"getPartitionNamesByParts",
stats
.getGetPartitionNamesPs()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
return Optional.of(
client.getPartitionNamesFiltered(
partitionFilter.getHiveTableName().getDatabaseName(),
partitionFilter.getHiveTableName().getTableName(),
partitionFilter.getParts()));
}
}));
} catch (NoSuchObjectException e) {
return Optional.empty();
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
private Optional<Table> loadTable(HiveTableName hiveTableName) throws Exception {
try {
return retry()
.stopOn(NoSuchObjectException.class, HiveViewNotSupportedException.class)
.stopOnIllegalExceptions()
.run(
"getTable",
stats
.getGetTable()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
Table table =
client.getTable(
hiveTableName.getDatabaseName(), hiveTableName.getTableName());
if (table.getTableType().equals(TableType.VIRTUAL_VIEW.name())
&& (!isPrestoView(table))) {
throw new HiveViewNotSupportedException(
new SchemaTableName(
hiveTableName.getDatabaseName(), hiveTableName.getTableName()));
}
return Optional.of(table);
}
}));
} catch (NoSuchObjectException e) {
return Optional.empty();
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
private Map<HivePartitionName, Optional<Partition>> loadPartitionsByNames(
Iterable<? extends HivePartitionName> partitionNames) throws Exception {
requireNonNull(partitionNames, "partitionNames is null");
checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty");
HivePartitionName firstPartition = Iterables.get(partitionNames, 0);
HiveTableName hiveTableName = firstPartition.getHiveTableName();
String databaseName = hiveTableName.getDatabaseName();
String tableName = hiveTableName.getTableName();
List<String> partitionsToFetch = new ArrayList<>();
for (HivePartitionName partitionName : partitionNames) {
checkArgument(
partitionName.getHiveTableName().equals(hiveTableName),
"Expected table name %s but got %s",
hiveTableName,
partitionName.getHiveTableName());
partitionsToFetch.add(partitionName.getPartitionName());
}
List<String> partitionColumnNames =
ImmutableList.copyOf(
Warehouse.makeSpecFromName(firstPartition.getPartitionName()).keySet());
try {
return retry()
.stopOn(NoSuchObjectException.class)
.stopOnIllegalExceptions()
.run(
"getPartitionsByNames",
stats
.getGetPartitionsByNames()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
ImmutableMap.Builder<HivePartitionName, Optional<Partition>> partitions =
ImmutableMap.builder();
for (Partition partition :
client.getPartitionsByNames(
databaseName, tableName, partitionsToFetch)) {
String partitionId =
FileUtils.makePartName(
partitionColumnNames, partition.getValues(), null);
partitions.put(
HivePartitionName.partition(databaseName, tableName, partitionId),
Optional.of(partition));
}
return partitions.build();
}
}));
} catch (NoSuchObjectException e) {
// assume none of the partitions in the batch are available
return stream(partitionNames.spliterator(), false)
.collect(toMap(identity(), (name) -> Optional.empty()));
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
private Optional<List<String>> loadAllTables(String databaseName) throws Exception {
Callable<List<String>> getAllTables =
stats
.getGetAllTables()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
return client.getAllTables(databaseName);
}
});
Callable<Void> getDatabase =
stats
.getGetDatabase()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
client.getDatabase(databaseName);
return null;
}
});
try {
return retry()
.stopOn(NoSuchObjectException.class)
.stopOnIllegalExceptions()
.run(
"getAllTables",
() -> {
List<String> tables = getAllTables.call();
if (tables.isEmpty()) {
// Check to see if the database exists
getDatabase.call();
}
return Optional.of(tables);
});
} catch (NoSuchObjectException e) {
return Optional.empty();
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
@Override
public void grantTablePrivileges(
String databaseName,
String tableName,
String grantee,
Set<PrivilegeGrantInfo> privilegeGrantInfoSet) {
try {
retry()
.stopOnIllegalExceptions()
.run(
"grantTablePrivileges",
stats
.getGrantTablePrivileges()
.wrap(
() -> {
try (HiveMetastoreClient metastoreClient =
clientProvider.createMetastoreClient()) {
PrincipalType principalType;
if (metastoreClient.getRoleNames().contains(grantee)) {
principalType = ROLE;
} else {
principalType = USER;
}
ImmutableList.Builder<HiveObjectPrivilege> privilegeBagBuilder =
ImmutableList.builder();
for (PrivilegeGrantInfo privilegeGrantInfo : privilegeGrantInfoSet) {
privilegeBagBuilder.add(
new HiveObjectPrivilege(
new HiveObjectRef(
HiveObjectType.TABLE, databaseName, tableName, null, null),
grantee,
principalType,
privilegeGrantInfo));
}
// TODO: Check whether the user/role exists in the hive metastore.
// TODO: Check whether the user already has the given privilege.
metastoreClient.grantPrivileges(
new PrivilegeBag(privilegeBagBuilder.build()));
}
return null;
}));
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
throw Throwables.propagate(e);
} finally {
userTablePrivileges.invalidate(new UserTableKey(grantee, tableName, databaseName));
}
}
private Set<String> loadRoles(String user) {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
List<Role> roles = client.listRoles(user, USER);
if (roles == null) {
return ImmutableSet.of();
}
return ImmutableSet.copyOf(roles.stream().map(Role::getRoleName).collect(toSet()));
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
@Override
public void addPartitions(String databaseName, String tableName, List<Partition> partitions) {
if (partitions.isEmpty()) {
return;
}
try {
retry()
.stopOn(
AlreadyExistsException.class,
InvalidObjectException.class,
MetaException.class,
NoSuchObjectException.class,
PrestoException.class)
.stopOnIllegalExceptions()
.run(
"addPartitions",
stats
.getAddPartitions()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
int partitionsAdded = client.addPartitions(partitions);
if (partitionsAdded != partitions.size()) {
throw new PrestoException(
HIVE_METASTORE_ERROR,
format(
"Hive metastore only added %s of %s partitions",
partitionsAdded, partitions.size()));
}
return null;
}
}));
} catch (AlreadyExistsException e) {
// todo partition already exists exception
throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
} catch (NoSuchObjectException e) {
throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
throw Throwables.propagate(e);
} finally {
// todo do we need to invalidate all partitions?
invalidatePartitionCache(databaseName, tableName);
}
}
private List<String> loadAllDatabases() throws Exception {
try {
return retry()
.stopOnIllegalExceptions()
.run(
"getAllDatabases",
stats
.getGetAllDatabases()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
return client.getAllDatabases();
}
}));
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
private Optional<Database> loadDatabase(String databaseName) throws Exception {
try {
return retry()
.stopOn(NoSuchObjectException.class)
.stopOnIllegalExceptions()
.run(
"getDatabase",
stats
.getGetDatabase()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
return Optional.of(client.getDatabase(databaseName));
}
}));
} catch (NoSuchObjectException e) {
return Optional.empty();
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
private Set<HivePrivilege> getPrivileges(String user, HiveObjectRef objectReference) {
ImmutableSet.Builder<HivePrivilege> privileges = ImmutableSet.builder();
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
PrincipalPrivilegeSet privilegeSet = client.getPrivilegeSet(objectReference, user, null);
if (privilegeSet != null) {
Map<String, List<PrivilegeGrantInfo>> userPrivileges = privilegeSet.getUserPrivileges();
if (userPrivileges != null) {
privileges.addAll(toGrants(userPrivileges.get(user)));
}
for (List<PrivilegeGrantInfo> rolePrivileges : privilegeSet.getRolePrivileges().values()) {
privileges.addAll(toGrants(rolePrivileges));
}
// We do not add the group permissions as Hive does not seem to process these
}
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
return privileges.build();
}
@Override
public boolean hasPrivilegeWithGrantOptionOnTable(
String user, String databaseName, String tableName, HivePrivilege hivePrivilege) {
try (HiveMetastoreClient metastoreClient = clientProvider.createMetastoreClient()) {
PrincipalPrivilegeSet principalPrivilegeSet =
metastoreClient.getPrivilegeSet(
new HiveObjectRef(HiveObjectType.TABLE, databaseName, tableName, null, null),
user,
null);
for (PrivilegeGrantInfo privilegeGrantInfo :
principalPrivilegeSet.getUserPrivileges().get(user)) {
if (privilegeGrantInfo.getPrivilege().equalsIgnoreCase(hivePrivilege.name())
&& privilegeGrantInfo.isGrantOption()) {
return true;
}
}
return false;
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
@Override
public void alterTable(String databaseName, String tableName, Table table) {
try {
retry()
.stopOn(InvalidOperationException.class, MetaException.class)
.stopOnIllegalExceptions()
.run(
"alterTable",
stats
.getAlterTable()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
Optional<Table> source =
loadTable(new HiveTableName(databaseName, tableName));
if (!source.isPresent()) {
throw new TableNotFoundException(
new SchemaTableName(databaseName, tableName));
}
client.alterTable(databaseName, tableName, table);
}
return null;
}));
} catch (NoSuchObjectException e) {
throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
} catch (InvalidOperationException | MetaException e) {
throw Throwables.propagate(e);
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
throw Throwables.propagate(e);
} finally {
invalidateTable(databaseName, tableName);
}
}
private Optional<List<String>> loadAllViews(String databaseName) throws Exception {
try {
return retry()
.stopOn(UnknownDBException.class)
.stopOnIllegalExceptions()
.run(
"getAllViews",
stats
.getAllViews()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
String filter =
HIVE_FILTER_FIELD_PARAMS + PRESTO_VIEW_FLAG + " = \"true\"";
return Optional.of(client.getTableNamesByFilter(databaseName, filter));
}
}));
} catch (UnknownDBException e) {
return Optional.empty();
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
@Override
public void createTable(Table table) {
try {
retry()
.stopOn(
AlreadyExistsException.class,
InvalidObjectException.class,
MetaException.class,
NoSuchObjectException.class)
.stopOnIllegalExceptions()
.run(
"createTable",
stats
.getCreateTable()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
client.createTable(table);
}
return null;
}));
} catch (AlreadyExistsException e) {
throw new TableAlreadyExistsException(
new SchemaTableName(table.getDbName(), table.getTableName()));
} catch (NoSuchObjectException e) {
throw new SchemaNotFoundException(table.getDbName());
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
throw Throwables.propagate(e);
} finally {
invalidateTable(table.getDbName(), table.getTableName());
}
}
@Override
public void dropPartitionByName(String databaseName, String tableName, String partitionName) {
try {
retry()
.stopOn(NoSuchObjectException.class, MetaException.class)
.stopOnIllegalExceptions()
.run(
"dropPartitionByName",
stats
.getDropPartitionByName()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
// It is observed that: (examples below assumes a table with one partition
// column `ds`)
// * When a partition doesn't exist (e.g. ds=2015-09-99), this thrift
// call is a no-op. It doesn't throw any exception.
// * When a typo exists in partition column name (e.g. dxs=2015-09-01),
// this thrift call will delete ds=2015-09-01.
client.dropPartitionByName(databaseName, tableName, partitionName, true);
}
return null;
}));
} catch (NoSuchObjectException e) {
throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
throw Throwables.propagate(e);
} finally {
invalidatePartitionCache(databaseName, tableName);
}
}