Example #1
0
  public static Table fromMetastoreApiTable(org.apache.hadoop.hive.metastore.api.Table table) {
    StorageDescriptor storageDescriptor = table.getSd();
    if (storageDescriptor == null) {
      throw new PrestoException(HIVE_INVALID_METADATA, "Table is missing storage descriptor");
    }

    Table.Builder tableBuilder =
        Table.builder()
            .setDatabaseName(table.getDbName())
            .setTableName(table.getTableName())
            .setOwner(nullToEmpty(table.getOwner()))
            .setTableType(table.getTableType())
            .setDataColumns(
                storageDescriptor
                    .getCols()
                    .stream()
                    .map(MetastoreUtil::fromMetastoreApiFieldSchema)
                    .collect(toList()))
            .setPartitionColumns(
                table
                    .getPartitionKeys()
                    .stream()
                    .map(MetastoreUtil::fromMetastoreApiFieldSchema)
                    .collect(toList()))
            .setParameters(
                table.getParameters() == null ? ImmutableMap.of() : table.getParameters())
            .setViewOriginalText(Optional.ofNullable(emptyToNull(table.getViewOriginalText())))
            .setViewExpandedText(Optional.ofNullable(emptyToNull(table.getViewExpandedText())));

    fromMetastoreApiStorageDescriptor(
        storageDescriptor, tableBuilder.getStorageBuilder(), table.getTableName());

    return tableBuilder.build();
  }
  @Override
  public synchronized void createTable(Table table) {
    SchemaTableName schemaTableName = new SchemaTableName(table.getDbName(), table.getTableName());
    Table tableCopy = table.deepCopy();
    if (tableCopy.getSd() == null) {
      tableCopy.setSd(new StorageDescriptor());
    } else if (tableCopy.getSd().getLocation() != null) {
      File directory = new File(new Path(tableCopy.getSd().getLocation()).toUri());
      checkArgument(directory.exists(), "Table directory does not exist");
      checkArgument(
          isParentDir(directory, baseDirectory),
          "Table directory must be inside of the metastore base directory");
    }

    if (relations.putIfAbsent(schemaTableName, tableCopy) != null) {
      throw new TableAlreadyExistsException(schemaTableName);
    }

    if (tableCopy.getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
      views.put(schemaTableName, tableCopy);
    }

    PrincipalPrivilegeSet privileges = table.getPrivileges();
    if (privileges != null) {
      for (Entry<String, List<PrivilegeGrantInfo>> entry :
          privileges.getUserPrivileges().entrySet()) {
        String user = entry.getKey();
        Set<HivePrivilegeInfo> userPrivileges =
            entry
                .getValue()
                .stream()
                .map(HivePrivilegeInfo::parsePrivilege)
                .flatMap(Collection::stream)
                .collect(toImmutableSet());
        setTablePrivileges(user, USER, table.getDbName(), table.getTableName(), userPrivileges);
      }
      for (Entry<String, List<PrivilegeGrantInfo>> entry :
          privileges.getRolePrivileges().entrySet()) {
        String role = entry.getKey();
        Set<HivePrivilegeInfo> rolePrivileges =
            entry
                .getValue()
                .stream()
                .map(HivePrivilegeInfo::parsePrivilege)
                .flatMap(Collection::stream)
                .collect(toImmutableSet());
        setTablePrivileges(role, ROLE, table.getDbName(), table.getTableName(), rolePrivileges);
      }
    }
  }
Example #3
0
  @Override
  public void alterTable(String databaseName, String tableName, Table newTable) {
    SchemaTableName oldName = new SchemaTableName(databaseName, tableName);
    SchemaTableName newName = new SchemaTableName(newTable.getDbName(), newTable.getTableName());

    // if the name did not change, this is a simple schema change
    if (oldName.equals(newName)) {
      if (relations.replace(oldName, newTable) == null) {
        throw new TableNotFoundException(oldName);
      }
      return;
    }

    // remove old table definition and add the new one
    // TODO: use locking to do this properly
    Table table = relations.get(oldName);
    if (table == null) {
      throw new TableNotFoundException(oldName);
    }

    if (relations.putIfAbsent(newName, newTable) != null) {
      throw new TableAlreadyExistsException(newName);
    }
    relations.remove(oldName);
  }
 /**
  * Removes the cache directive associated with the table from HDFS, uncaching all data. Also
  * updates the table's metadata. No-op if the table is not cached.
  */
 public static void uncacheTbl(org.apache.hadoop.hive.metastore.api.Table table)
     throws ImpalaRuntimeException {
   Preconditions.checkNotNull(table);
   LOG.debug("Uncaching table: " + table.getDbName() + "." + table.getTableName());
   Long id = getCacheDirectiveId(table.getParameters());
   if (id == null) return;
   HdfsCachingUtil.removeDirective(id);
   table.getParameters().remove(CACHE_DIR_ID_PROP_NAME);
   table.getParameters().remove(CACHE_DIR_REPLICATION_PROP_NAME);
 }
  @Override
  public void createTable(Table table) throws HiveMetastoreException {
    if (!existsDb(table.getDbName())) {
      throw new HiveMetastoreException("DB " + table.getDbName() + " does not exist!");
    }

    HiveObjectSpec tableSpec = new HiveObjectSpec(table.getDbName(), table.getTableName());
    if (specToTable.containsKey(tableSpec)) {
      throw new HiveMetastoreException("Table already exists: " + tableSpec);
    }
    specToTable.put(tableSpec, table);
  }
  @Before
  public void createTable() throws Exception {
    // Use Junit's Assume to skip running this fixture against any storage formats whose
    // SerDe is in the disabled serdes list.
    Assume.assumeTrue(!DISABLED_SERDES.contains(serdeClass));

    String databaseName = (dbName == null) ? MetaStoreUtils.DEFAULT_DATABASE_NAME : dbName;
    try {
      client.dropTable(databaseName, tableName);
    } catch (Exception e) {
      // Can fail with NoSuchObjectException.
    }

    Table tbl = new Table();
    tbl.setDbName(databaseName);
    tbl.setTableName(tableName);
    if (isTableExternal()) {
      tbl.setTableType(TableType.EXTERNAL_TABLE.toString());
    } else {
      tbl.setTableType(TableType.MANAGED_TABLE.toString());
    }
    StorageDescriptor sd = new StorageDescriptor();
    sd.setCols(getTableColumns());

    tbl.setPartitionKeys(getPartitionKeys());
    tbl.setSd(sd);

    sd.setBucketCols(new ArrayList<String>(2));
    sd.setSerdeInfo(new SerDeInfo());
    sd.getSerdeInfo().setName(tbl.getTableName());
    sd.getSerdeInfo().setParameters(new HashMap<String, String>());
    sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
    if (isTableExternal()) {
      sd.getSerdeInfo().getParameters().put("EXTERNAL", "TRUE");
    }
    sd.getSerdeInfo().setSerializationLib(serdeClass);
    sd.setInputFormat(inputFormatClass);
    sd.setOutputFormat(outputFormatClass);

    Map<String, String> tableParams = new HashMap<String, String>();
    if (isTableExternal()) {
      tableParams.put("EXTERNAL", "TRUE");
    }
    if (isTableImmutable()) {
      tableParams.put(hive_metastoreConstants.IS_IMMUTABLE, "true");
    }
    StatsSetupConst.setBasicStatsState(tableParams, StatsSetupConst.TRUE);
    tbl.setParameters(tableParams);

    client.createTable(tbl);
  }
 @Override
 public void alterTable(String dbName, String tableName, Table table)
     throws HiveMetastoreException {
   HiveObjectSpec existingTableSpec = new HiveObjectSpec(dbName, tableName);
   HiveObjectSpec newTableSpec = new HiveObjectSpec(table.getDbName(), table.getTableName());
   if (!specToTable.containsKey(existingTableSpec)) {
     throw new HiveMetastoreException("Unknown table: " + existingTableSpec);
   }
   Table removedTable = specToTable.remove(existingTableSpec);
   if (removedTable == null) {
     throw new RuntimeException("Shouldn't happen!");
   }
   specToTable.put(newTableSpec, table);
 }
 @Override
 public void createTable(Table table) {
   try {
     retry()
         .stopOn(
             AlreadyExistsException.class,
             InvalidObjectException.class,
             MetaException.class,
             NoSuchObjectException.class)
         .stopOnIllegalExceptions()
         .run(
             "createTable",
             stats
                 .getCreateTable()
                 .wrap(
                     () -> {
                       try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                         client.createTable(table);
                       }
                       return null;
                     }));
   } catch (AlreadyExistsException e) {
     throw new TableAlreadyExistsException(
         new SchemaTableName(table.getDbName(), table.getTableName()));
   } catch (NoSuchObjectException e) {
     throw new SchemaNotFoundException(table.getDbName());
   } catch (TException e) {
     throw new PrestoException(HIVE_METASTORE_ERROR, e);
   } catch (Exception e) {
     if (e instanceof InterruptedException) {
       Thread.currentThread().interrupt();
     }
     throw Throwables.propagate(e);
   } finally {
     invalidateTable(table.getDbName(), table.getTableName());
   }
 }
Example #9
0
 // This method is completely copied from Hive's HBaseStorageHandler.java.
 private String getHBaseTableName(org.apache.hadoop.hive.metastore.api.Table tbl) {
   // Give preference to TBLPROPERTIES over SERDEPROPERTIES
   // (really we should only use TBLPROPERTIES, so this is just
   // for backwards compatibility with the original specs).
   String tableName = tbl.getParameters().get(HBaseSerDe.HBASE_TABLE_NAME);
   if (tableName == null) {
     tableName = tbl.getSd().getSerdeInfo().getParameters().get(HBaseSerDe.HBASE_TABLE_NAME);
   }
   if (tableName == null) {
     tableName = tbl.getDbName() + "." + tbl.getTableName();
     if (tableName.startsWith(DEFAULT_PREFIX)) {
       tableName = tableName.substring(DEFAULT_PREFIX.length());
     }
   }
   return tableName;
 }
  private static void createTable(String tableName, String tablePerm) throws Exception {
    Table tbl = new Table();
    tbl.setDbName(DATABASE);
    tbl.setTableName(tableName);
    StorageDescriptor sd = new StorageDescriptor();
    sd.setCols(ColumnHolder.colMapping.get(tableName));
    tbl.setSd(sd);
    sd.setParameters(new HashMap<String, String>());
    sd.setSerdeInfo(new SerDeInfo());
    sd.getSerdeInfo().setName(tbl.getTableName());
    sd.getSerdeInfo().setParameters(new HashMap<String, String>());
    sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
    sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
    sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
    sd.getSerdeInfo()
        .setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
    tbl.setPartitionKeys(ColumnHolder.partitionCols);

    hmsc.createTable(tbl);
    FileSystem fs = FileSystem.get(mrConf);
    fs.setPermission(new Path(warehousedir, tableName), new FsPermission(tablePerm));
  }
 private void validateTable(Table expectedTable, Table actualTable) {
   assertEquals(expectedTable.getTableName(), actualTable.getTableName());
   assertEquals(expectedTable.getDbName(), actualTable.getDbName());
   assertEquals(expectedTable.getSd().getLocation(), actualTable.getSd().getLocation());
 }
Example #12
0
  @Override
  public final void createTable(final CatalogProtos.TableDescProto tableDescProto)
      throws CatalogException {
    HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;

    TableDesc tableDesc = new TableDesc(tableDescProto);
    String[] splitted = CatalogUtil.splitFQTableName(tableDesc.getName());
    String databaseName = splitted[0];
    String tableName = splitted[1];

    try {
      client = clientPool.getClient();

      org.apache.hadoop.hive.metastore.api.Table table =
          new org.apache.hadoop.hive.metastore.api.Table();
      table.setDbName(databaseName);
      table.setTableName(tableName);
      table.setParameters(
          new HashMap<String, String>(tableDesc.getMeta().getOptions().getAllKeyValus()));
      // TODO: set owner
      // table.setOwner();

      StorageDescriptor sd = new StorageDescriptor();
      sd.setSerdeInfo(new SerDeInfo());
      sd.getSerdeInfo().setParameters(new HashMap<String, String>());
      sd.getSerdeInfo().setName(table.getTableName());

      // if tajo set location method, thrift client make exception as follows:
      // Caused by: MetaException(message:java.lang.NullPointerException)
      // If you want to modify table path, you have to modify on Hive cli.
      if (tableDesc.isExternal()) {
        table.setTableType(TableType.EXTERNAL_TABLE.name());
        table.putToParameters("EXTERNAL", "TRUE");

        Path tablePath = new Path(tableDesc.getUri());
        FileSystem fs = tablePath.getFileSystem(conf);
        if (fs.isFile(tablePath)) {
          LOG.warn("A table path is a file, but HiveCatalogStore does not allow a file path.");
          sd.setLocation(tablePath.getParent().toString());
        } else {
          sd.setLocation(tablePath.toString());
        }
      }

      // set column information
      List<Column> columns = tableDesc.getSchema().getRootColumns();
      ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(columns.size());

      for (Column eachField : columns) {
        cols.add(
            new FieldSchema(
                eachField.getSimpleName(),
                HiveCatalogUtil.getHiveFieldType(eachField.getDataType()),
                ""));
      }
      sd.setCols(cols);

      // set partition keys
      if (tableDesc.hasPartition()
          && tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
        List<FieldSchema> partitionKeys = new ArrayList<FieldSchema>();
        for (Column eachPartitionKey :
            tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
          partitionKeys.add(
              new FieldSchema(
                  eachPartitionKey.getSimpleName(),
                  HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()),
                  ""));
        }
        table.setPartitionKeys(partitionKeys);
      }

      if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.RCFILE)) {
        String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE);
        sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
        sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
        if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
          sd.getSerdeInfo()
              .setSerializationLib(
                  org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
        } else {
          sd.getSerdeInfo()
              .setSerializationLib(
                  org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName());
        }

        if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) {
          table.putToParameters(
              serdeConstants.SERIALIZATION_NULL_FORMAT,
              StringEscapeUtils.unescapeJava(
                  tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
        }
      } else if (tableDesc.getMeta().getStoreType().equals(BuiltinStorages.TEXT)) {
        sd.getSerdeInfo()
            .setSerializationLib(
                org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
        sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
        sd.setOutputFormat(
            org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());

        String fieldDelimiter =
            tableDesc
                .getMeta()
                .getOption(
                    StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);

        // User can use an unicode for filed delimiter such as \u0001, \001.
        // In this case, java console will convert this value into "\\u001".
        // And hive will un-espace this value again.
        // As a result, user can use right field delimiter.
        // So, we have to un-escape this value.
        sd.getSerdeInfo()
            .putToParameters(
                serdeConstants.SERIALIZATION_FORMAT,
                StringEscapeUtils.unescapeJava(fieldDelimiter));
        sd.getSerdeInfo()
            .putToParameters(
                serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter));
        table.getParameters().remove(StorageConstants.TEXT_DELIMITER);

        if (tableDesc.getMeta().containsOption(StorageConstants.TEXT_NULL)) {
          table.putToParameters(
              serdeConstants.SERIALIZATION_NULL_FORMAT,
              StringEscapeUtils.unescapeJava(
                  tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL)));
          table.getParameters().remove(StorageConstants.TEXT_NULL);
        }
      } else if (tableDesc
          .getMeta()
          .getStoreType()
          .equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
        String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
        sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName());
        sd.setOutputFormat(
            org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName());

        if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
          sd.getSerdeInfo()
              .setSerializationLib(
                  org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());

          String fieldDelimiter =
              tableDesc
                  .getMeta()
                  .getOption(
                      StorageConstants.SEQUENCEFILE_DELIMITER,
                      StorageConstants.DEFAULT_FIELD_DELIMITER);

          // User can use an unicode for filed delimiter such as \u0001, \001.
          // In this case, java console will convert this value into "\\u001".
          // And hive will un-espace this value again.
          // As a result, user can use right field delimiter.
          // So, we have to un-escape this value.
          sd.getSerdeInfo()
              .putToParameters(
                  serdeConstants.SERIALIZATION_FORMAT,
                  StringEscapeUtils.unescapeJava(fieldDelimiter));
          sd.getSerdeInfo()
              .putToParameters(
                  serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter));
          table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER);
        } else {
          sd.getSerdeInfo()
              .setSerializationLib(
                  org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
        }

        if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) {
          table.putToParameters(
              serdeConstants.SERIALIZATION_NULL_FORMAT,
              StringEscapeUtils.unescapeJava(
                  tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
          table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
        }
      } else {
        if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.PARQUET)) {
          sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName());
          sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName());
          sd.getSerdeInfo()
              .setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName());
        } else {
          throw new UnsupportedException(
              tableDesc.getMeta().getStoreType() + " in HivecatalogStore");
        }
      }

      sd.setSortCols(new ArrayList<Order>());

      table.setSd(sd);
      client.getHiveClient().createTable(table);
    } catch (Throwable t) {
      throw new TajoInternalError(t);
    } finally {
      if (client != null) client.release();
    }
  }