Exemplo n.º 1
0
 @Override
 public void dropPartition(String databaseName, String tableName, List<String> parts) {
   try {
     retry()
         .stopOn(NoSuchObjectException.class, MetaException.class)
         .stopOnIllegalExceptions()
         .run(
             "dropPartition",
             stats
                 .getDropPartition()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         client.dropPartition(databaseName, tableName, parts, true);
                       }
                       return null;
                     }));
   } catch (NoSuchObjectException e) {
     throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   } catch (Exception e) {
     if (e instanceof InterruptedException) {
       Thread.currentThread().interrupt();
     }
     throw Throwables.propagate(e);
   } finally {
     invalidatePartitionCache(databaseName, tableName);
   }
 }
Exemplo n.º 2
0
 private Optional<Partition> loadPartitionByName(HivePartitionName partitionName)
     throws Exception {
   requireNonNull(partitionName, "partitionName is null");
   try {
     return retry()
         .stopOn(NoSuchObjectException.class)
         .stopOnIllegalExceptions()
         .run(
             "getPartitionsByNames",
             stats
                 .getGetPartitionByName()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         return Optional.of(
                             client.getPartitionByName(
                                 partitionName.getHiveTableName().getDatabaseName(),
                                 partitionName.getHiveTableName().getTableName(),
                                 partitionName.getPartitionName()));
                       }
                     }));
   } catch (NoSuchObjectException e) {
     return Optional.empty();
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   }
 }
Exemplo n.º 3
0
 private Optional<List<String>> loadPartitionNamesByParts(PartitionFilter partitionFilter)
     throws Exception {
   try {
     return retry()
         .stopOn(NoSuchObjectException.class)
         .stopOnIllegalExceptions()
         .run(
             "getPartitionNamesByParts",
             stats
                 .getGetPartitionNamesPs()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         return Optional.of(
                             client.getPartitionNamesFiltered(
                                 partitionFilter.getHiveTableName().getDatabaseName(),
                                 partitionFilter.getHiveTableName().getTableName(),
                                 partitionFilter.getParts()));
                       }
                     }));
   } catch (NoSuchObjectException e) {
     return Optional.empty();
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   }
 }
Exemplo n.º 4
0
 private Optional<Table> loadTable(HiveTableName hiveTableName) throws Exception {
   try {
     return retry()
         .stopOn(NoSuchObjectException.class, HiveViewNotSupportedException.class)
         .stopOnIllegalExceptions()
         .run(
             "getTable",
             stats
                 .getGetTable()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         Table table =
                             client.getTable(
                                 hiveTableName.getDatabaseName(), hiveTableName.getTableName());
                         if (table.getTableType().equals(TableType.VIRTUAL_VIEW.name())
                             && (!isPrestoView(table))) {
                           throw new HiveViewNotSupportedException(
                               new SchemaTableName(
                                   hiveTableName.getDatabaseName(), hiveTableName.getTableName()));
                         }
                         return Optional.of(table);
                       }
                     }));
   } catch (NoSuchObjectException e) {
     return Optional.empty();
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   }
 }
Exemplo n.º 5
0
  private Map<HivePartitionName, Optional<Partition>> loadPartitionsByNames(
      Iterable<? extends HivePartitionName> partitionNames) throws Exception {
    requireNonNull(partitionNames, "partitionNames is null");
    checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty");

    HivePartitionName firstPartition = Iterables.get(partitionNames, 0);

    HiveTableName hiveTableName = firstPartition.getHiveTableName();
    String databaseName = hiveTableName.getDatabaseName();
    String tableName = hiveTableName.getTableName();

    List<String> partitionsToFetch = new ArrayList<>();
    for (HivePartitionName partitionName : partitionNames) {
      checkArgument(
          partitionName.getHiveTableName().equals(hiveTableName),
          "Expected table name %s but got %s",
          hiveTableName,
          partitionName.getHiveTableName());
      partitionsToFetch.add(partitionName.getPartitionName());
    }

    List<String> partitionColumnNames =
        ImmutableList.copyOf(
            Warehouse.makeSpecFromName(firstPartition.getPartitionName()).keySet());

    try {
      return retry()
          .stopOn(NoSuchObjectException.class)
          .stopOnIllegalExceptions()
          .run(
              "getPartitionsByNames",
              stats
                  .getGetPartitionsByNames()
                  .wrap(
                      () -> {
                        try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                          ImmutableMap.Builder<HivePartitionName, Optional<Partition>> partitions =
                              ImmutableMap.builder();
                          for (Partition partition :
                              client.getPartitionsByNames(
                                  databaseName, tableName, partitionsToFetch)) {
                            String partitionId =
                                FileUtils.makePartName(
                                    partitionColumnNames, partition.getValues(), null);
                            partitions.put(
                                HivePartitionName.partition(databaseName, tableName, partitionId),
                                Optional.of(partition));
                          }
                          return partitions.build();
                        }
                      }));
    } catch (NoSuchObjectException e) {
      // assume none of the partitions in the batch are available
      return stream(partitionNames.spliterator(), false)
          .collect(toMap(identity(), (name) -> Optional.empty()));
    } catch (TException e) {
      throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
  }
Exemplo n.º 6
0
  private Optional<List<String>> loadAllTables(String databaseName) throws Exception {
    Callable<List<String>> getAllTables =
        stats
            .getGetAllTables()
            .wrap(
                () -> {
                  try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                    return client.getAllTables(databaseName);
                  }
                });

    Callable<Void> getDatabase =
        stats
            .getGetDatabase()
            .wrap(
                () -> {
                  try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                    client.getDatabase(databaseName);
                    return null;
                  }
                });

    try {
      return retry()
          .stopOn(NoSuchObjectException.class)
          .stopOnIllegalExceptions()
          .run(
              "getAllTables",
              () -> {
                List<String> tables = getAllTables.call();
                if (tables.isEmpty()) {
                  // Check to see if the database exists
                  getDatabase.call();
                }
                return Optional.of(tables);
              });
    } catch (NoSuchObjectException e) {
      return Optional.empty();
    } catch (TException e) {
      throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
  }
Exemplo n.º 7
0
  @Override
  public void grantTablePrivileges(
      String databaseName,
      String tableName,
      String grantee,
      Set<PrivilegeGrantInfo> privilegeGrantInfoSet) {
    try {
      retry()
          .stopOnIllegalExceptions()
          .run(
              "grantTablePrivileges",
              stats
                  .getGrantTablePrivileges()
                  .wrap(
                      () -> {
                        try (HiveMetastoreClient metastoreClient =
                            clientProvider.createMetastoreClient()) {
                          PrincipalType principalType;

                          if (metastoreClient.getRoleNames().contains(grantee)) {
                            principalType = ROLE;
                          } else {
                            principalType = USER;
                          }

                          ImmutableList.Builder<HiveObjectPrivilege> privilegeBagBuilder =
                              ImmutableList.builder();
                          for (PrivilegeGrantInfo privilegeGrantInfo : privilegeGrantInfoSet) {
                            privilegeBagBuilder.add(
                                new HiveObjectPrivilege(
                                    new HiveObjectRef(
                                        HiveObjectType.TABLE, databaseName, tableName, null, null),
                                    grantee,
                                    principalType,
                                    privilegeGrantInfo));
                          }
                          // TODO: Check whether the user/role exists in the hive metastore.
                          // TODO: Check whether the user already has the given privilege.
                          metastoreClient.grantPrivileges(
                              new PrivilegeBag(privilegeBagBuilder.build()));
                        }
                        return null;
                      }));
    } catch (TException e) {
      throw new PrestoException(HIVE_METASTORE_ERROR, e);
    } catch (Exception e) {
      if (e instanceof InterruptedException) {
        Thread.currentThread().interrupt();
      }
      throw Throwables.propagate(e);
    } finally {
      userTablePrivileges.invalidate(new UserTableKey(grantee, tableName, databaseName));
    }
  }
Exemplo n.º 8
0
 private Set<String> loadRoles(String user) {
   try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
     List<Role> roles = client.listRoles(user, USER);
     if (roles == null) {
       return ImmutableSet.of();
     }
     return ImmutableSet.copyOf(roles.stream().map(Role::getRoleName).collect(toSet()));
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   }
 }
Exemplo n.º 9
0
 @Override
 public void addPartitions(String databaseName, String tableName, List<Partition> partitions) {
   if (partitions.isEmpty()) {
     return;
   }
   try {
     retry()
         .stopOn(
             AlreadyExistsException.class,
             InvalidObjectException.class,
             MetaException.class,
             NoSuchObjectException.class,
             PrestoException.class)
         .stopOnIllegalExceptions()
         .run(
             "addPartitions",
             stats
                 .getAddPartitions()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         int partitionsAdded = client.addPartitions(partitions);
                         if (partitionsAdded != partitions.size()) {
                           throw new PrestoException(
                               HIVE_METASTORE_ERROR,
                               format(
                                   "Hive metastore only added %s of %s partitions",
                                   partitionsAdded, partitions.size()));
                         }
                         return null;
                       }
                     }));
   } catch (AlreadyExistsException e) {
     // todo partition already exists exception
     throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
   } catch (NoSuchObjectException e) {
     throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   } catch (Exception e) {
     if (e instanceof InterruptedException) {
       Thread.currentThread().interrupt();
     }
     throw Throwables.propagate(e);
   } finally {
     // todo do we need to invalidate all partitions?
     invalidatePartitionCache(databaseName, tableName);
   }
 }
Exemplo n.º 10
0
 private List<String> loadAllDatabases() throws Exception {
   try {
     return retry()
         .stopOnIllegalExceptions()
         .run(
             "getAllDatabases",
             stats
                 .getGetAllDatabases()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         return client.getAllDatabases();
                       }
                     }));
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   }
 }
Exemplo n.º 11
0
 private Optional<Database> loadDatabase(String databaseName) throws Exception {
   try {
     return retry()
         .stopOn(NoSuchObjectException.class)
         .stopOnIllegalExceptions()
         .run(
             "getDatabase",
             stats
                 .getGetDatabase()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         return Optional.of(client.getDatabase(databaseName));
                       }
                     }));
   } catch (NoSuchObjectException e) {
     return Optional.empty();
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   }
 }
Exemplo n.º 12
0
  private Set<HivePrivilege> getPrivileges(String user, HiveObjectRef objectReference) {
    ImmutableSet.Builder<HivePrivilege> privileges = ImmutableSet.builder();
    try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
      PrincipalPrivilegeSet privilegeSet = client.getPrivilegeSet(objectReference, user, null);

      if (privilegeSet != null) {
        Map<String, List<PrivilegeGrantInfo>> userPrivileges = privilegeSet.getUserPrivileges();
        if (userPrivileges != null) {
          privileges.addAll(toGrants(userPrivileges.get(user)));
        }
        for (List<PrivilegeGrantInfo> rolePrivileges : privilegeSet.getRolePrivileges().values()) {
          privileges.addAll(toGrants(rolePrivileges));
        }
        // We do not add the group permissions as Hive does not seem to process these
      }
    } catch (TException e) {
      throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }

    return privileges.build();
  }
Exemplo n.º 13
0
  @Override
  public boolean hasPrivilegeWithGrantOptionOnTable(
      String user, String databaseName, String tableName, HivePrivilege hivePrivilege) {
    try (HiveMetastoreClient metastoreClient = clientProvider.createMetastoreClient()) {
      PrincipalPrivilegeSet principalPrivilegeSet =
          metastoreClient.getPrivilegeSet(
              new HiveObjectRef(HiveObjectType.TABLE, databaseName, tableName, null, null),
              user,
              null);

      for (PrivilegeGrantInfo privilegeGrantInfo :
          principalPrivilegeSet.getUserPrivileges().get(user)) {
        if (privilegeGrantInfo.getPrivilege().equalsIgnoreCase(hivePrivilege.name())
            && privilegeGrantInfo.isGrantOption()) {
          return true;
        }
      }
      return false;
    } catch (TException e) {
      throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
  }
Exemplo n.º 14
0
 @Override
 public void alterTable(String databaseName, String tableName, Table table) {
   try {
     retry()
         .stopOn(InvalidOperationException.class, MetaException.class)
         .stopOnIllegalExceptions()
         .run(
             "alterTable",
             stats
                 .getAlterTable()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         Optional<Table> source =
                             loadTable(new HiveTableName(databaseName, tableName));
                         if (!source.isPresent()) {
                           throw new TableNotFoundException(
                               new SchemaTableName(databaseName, tableName));
                         }
                         client.alterTable(databaseName, tableName, table);
                       }
                       return null;
                     }));
   } catch (NoSuchObjectException e) {
     throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
   } catch (InvalidOperationException | MetaException e) {
     throw Throwables.propagate(e);
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   } catch (Exception e) {
     if (e instanceof InterruptedException) {
       Thread.currentThread().interrupt();
     }
     throw Throwables.propagate(e);
   } finally {
     invalidateTable(databaseName, tableName);
   }
 }
Exemplo n.º 15
0
 private Optional<List<String>> loadAllViews(String databaseName) throws Exception {
   try {
     return retry()
         .stopOn(UnknownDBException.class)
         .stopOnIllegalExceptions()
         .run(
             "getAllViews",
             stats
                 .getAllViews()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         String filter =
                             HIVE_FILTER_FIELD_PARAMS + PRESTO_VIEW_FLAG + " = \"true\"";
                         return Optional.of(client.getTableNamesByFilter(databaseName, filter));
                       }
                     }));
   } catch (UnknownDBException e) {
     return Optional.empty();
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   }
 }
Exemplo n.º 16
0
 @Override
 public void createTable(Table table) {
   try {
     retry()
         .stopOn(
             AlreadyExistsException.class,
             InvalidObjectException.class,
             MetaException.class,
             NoSuchObjectException.class)
         .stopOnIllegalExceptions()
         .run(
             "createTable",
             stats
                 .getCreateTable()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         client.createTable(table);
                       }
                       return null;
                     }));
   } catch (AlreadyExistsException e) {
     throw new TableAlreadyExistsException(
         new SchemaTableName(table.getDbName(), table.getTableName()));
   } catch (NoSuchObjectException e) {
     throw new SchemaNotFoundException(table.getDbName());
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   } catch (Exception e) {
     if (e instanceof InterruptedException) {
       Thread.currentThread().interrupt();
     }
     throw Throwables.propagate(e);
   } finally {
     invalidateTable(table.getDbName(), table.getTableName());
   }
 }
Exemplo n.º 17
0
 @Override
 public void dropPartitionByName(String databaseName, String tableName, String partitionName) {
   try {
     retry()
         .stopOn(NoSuchObjectException.class, MetaException.class)
         .stopOnIllegalExceptions()
         .run(
             "dropPartitionByName",
             stats
                 .getDropPartitionByName()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         // It is observed that: (examples below assumes a table with one partition
                         // column `ds`)
                         //  * When a partition doesn't exist (e.g. ds=2015-09-99), this thrift
                         // call is a no-op. It doesn't throw any exception.
                         //  * When a typo exists in partition column name (e.g. dxs=2015-09-01),
                         // this thrift call will delete ds=2015-09-01.
                         client.dropPartitionByName(databaseName, tableName, partitionName, true);
                       }
                       return null;
                     }));
   } catch (NoSuchObjectException e) {
     throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   } catch (Exception e) {
     if (e instanceof InterruptedException) {
       Thread.currentThread().interrupt();
     }
     throw Throwables.propagate(e);
   } finally {
     invalidatePartitionCache(databaseName, tableName);
   }
 }