/**
  * Adds a table with the given name to the catalog and returns the new table, loading the metadata
  * if needed.
  */
 public Table addTable(String dbName, String tblName) throws TableNotFoundException {
   Db db = getDb(dbName);
   if (db == null) return null;
   Table incompleteTable = IncompleteTable.createUninitializedTable(getNextTableId(), db, tblName);
   incompleteTable.setCatalogVersion(incrementAndGetCatalogVersion());
   db.addTable(incompleteTable);
   return db.getTable(tblName);
 }
Пример #2
0
  @Test
  public void testLoadingUnsupportedTableTypes() throws TableLoadingException {
    Table table = catalog_.getDb("functional").getTable("hive_index_tbl");
    assertTrue(table instanceof IncompleteTable);
    IncompleteTable incompleteTable = (IncompleteTable) table;
    assertTrue(incompleteTable.getCause() instanceof TableLoadingException);
    assertEquals(
        "Unsupported table type 'INDEX_TABLE' for: functional.hive_index_tbl",
        incompleteTable.getCause().getMessage());

    // Table with unsupported SerDe library.
    table = catalog_.getDb("functional").getTable("bad_serde");
    assertTrue(table instanceof IncompleteTable);
    incompleteTable = (IncompleteTable) table;
    assertTrue(incompleteTable.getCause() instanceof TableLoadingException);
    assertEquals(
        "Failed to load metadata for table: bad_serde\n"
            + "CAUSED BY: InvalidStorageDescriptorException: "
            + "Impala does not support tables of this type. REASON: SerDe"
            + " library 'org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe' "
            + "is not supported.",
        incompleteTable.getCause().getMessage());
  }
Пример #3
0
 /**
  * Blocks until the table has finished loading and returns the result. If any errors were
  * encountered while loading the table an IncompleteTable will be returned.
  */
 public Table get() {
   Table tbl;
   try {
     tbl = tblTask_.get();
   } catch (Exception e) {
     tbl =
         IncompleteTable.createFailedMetadataLoadTable(
             TableId.createInvalidId(),
             catalog_.getDb(tblName_.getDb_name()),
             tblName_.getTable_name(),
             new TableLoadingException(e.getMessage(), e));
   }
   Preconditions.checkState(tbl.isLoaded());
   return tbl;
 }
Пример #4
0
 public void testArrayColumnFails() throws TableLoadingException {
   Table table = catalog_.getDb("functional").getTable("array_table");
   assertTrue(table instanceof IncompleteTable);
   IncompleteTable incompleteTable = (IncompleteTable) table;
   assertTrue(incompleteTable.getCause() instanceof TableLoadingException);
 }
Пример #5
0
 public void testMapColumnFailsOnHBaseTable() throws TableLoadingException {
   Table table = catalog_.getDb("functional_hbase").getTable("map_table_hbase");
   assertTrue(table instanceof IncompleteTable);
   IncompleteTable incompleteTable = (IncompleteTable) table;
   assertTrue(incompleteTable.getCause() instanceof TableLoadingException);
 }
  /**
   * Invalidates the table in the catalog cache, potentially adding/removing the table from the
   * cache based on whether it exists in the Hive Metastore. The invalidation logic is: - If the
   * table exists in the metastore, add it to the catalog as an uninitialized IncompleteTable
   * (replacing any existing entry). The table metadata will be loaded lazily, on the next access.
   * If the parent database for this table does not yet exist in Impala's cache it will also be
   * added. - If the table does not exist in the metastore, remove it from the catalog cache. - If
   * we are unable to determine whether the table exists in the metastore (there was an exception
   * thrown making the RPC), invalidate any existing Table by replacing it with an uninitialized
   * IncompleteTable.
   *
   * <p>The parameter updatedObjects is a Pair that contains details on what catalog objects were
   * modified as a result of the invalidateTable() call. The first item in the Pair is a Db which
   * will only be set if a new database was added as a result of this call, otherwise it will be
   * null. The second item in the Pair is the Table that was modified/added/removed. Returns a flag
   * that indicates whether the items in updatedObjects were removed (returns true) or
   * added/modified (return false). Only Tables should ever be removed.
   */
  public boolean invalidateTable(TTableName tableName, Pair<Db, Table> updatedObjects) {
    Preconditions.checkNotNull(updatedObjects);
    updatedObjects.first = null;
    updatedObjects.second = null;
    LOG.debug(
        String.format(
            "Invalidating table metadata: %s.%s",
            tableName.getDb_name(), tableName.getTable_name()));
    String dbName = tableName.getDb_name();
    String tblName = tableName.getTable_name();

    // Stores whether the table exists in the metastore. Can have three states:
    // 1) true - Table exists in metastore.
    // 2) false - Table does not exist in metastore.
    // 3) unknown (null) - There was exception thrown by the metastore client.
    Boolean tableExistsInMetaStore;
    MetaStoreClient msClient = getMetaStoreClient();
    try {
      tableExistsInMetaStore = msClient.getHiveClient().tableExists(dbName, tblName);
    } catch (UnknownDBException e) {
      // The parent database does not exist in the metastore. Treat this the same
      // as if the table does not exist.
      tableExistsInMetaStore = false;
    } catch (TException e) {
      LOG.error("Error executing tableExists() metastore call: " + tblName, e);
      tableExistsInMetaStore = null;
    } finally {
      msClient.release();
    }

    if (tableExistsInMetaStore != null && !tableExistsInMetaStore) {
      updatedObjects.second = removeTable(dbName, tblName);
      return true;
    } else {
      Db db = getDb(dbName);
      if ((db == null || !db.containsTable(tblName)) && tableExistsInMetaStore == null) {
        // The table does not exist in our cache AND it is unknown whether the table
        // exists in the metastore. Do nothing.
        return false;
      } else if (db == null && tableExistsInMetaStore) {
        // The table exists in the metastore, but our cache does not contain the parent
        // database. A new db will be added to the cache along with the new table.
        db = new Db(dbName, this);
        db.setCatalogVersion(incrementAndGetCatalogVersion());
        addDb(db);
        updatedObjects.first = db;
      }

      // Add a new uninitialized table to the table cache, effectively invalidating
      // any existing entry. The metadata for the table will be loaded lazily, on the
      // on the next access to the table.
      Table newTable = IncompleteTable.createUninitializedTable(getNextTableId(), db, tblName);
      newTable.setCatalogVersion(incrementAndGetCatalogVersion());
      db.addTable(newTable);
      if (loadInBackground_) {
        tableLoadingMgr_.backgroundLoad(
            new TTableName(dbName.toLowerCase(), tblName.toLowerCase()));
      }
      updatedObjects.second = newTable;
      return false;
    }
  }
  /** Resets this catalog instance by clearing all cached table and database metadata. */
  public void reset() throws CatalogException {
    // First update the policy metadata.
    if (sentryProxy_ != null) {
      // Sentry Service is enabled.
      try {
        // Update the authorization policy, waiting for the result to complete.
        sentryProxy_.refresh();
      } catch (Exception e) {
        throw new CatalogException("Error updating authorization policy: ", e);
      }
    }

    catalogLock_.writeLock().lock();
    try {
      nextTableId_.set(0);

      // Since UDFs/UDAs are not persisted in the metastore, we won't clear
      // them across reset. To do this, we store all the functions before
      // clearing and restore them after.
      // TODO: Everything about this. Persist them.
      List<Pair<String, HashMap<String, List<Function>>>> functions = Lists.newArrayList();
      for (Db db : dbCache_.get().values()) {
        if (db.numFunctions() == 0) continue;
        functions.add(Pair.create(db.getName(), db.getAllFunctions()));
      }

      // Build a new DB cache, populate it, and replace the existing cache in one
      // step.
      ConcurrentHashMap<String, Db> newDbCache = new ConcurrentHashMap<String, Db>();
      List<TTableName> tblsToBackgroundLoad = Lists.newArrayList();
      MetaStoreClient msClient = metaStoreClientPool_.getClient();
      try {
        for (String dbName : msClient.getHiveClient().getAllDatabases()) {
          Db db = new Db(dbName, this);
          db.setCatalogVersion(incrementAndGetCatalogVersion());
          newDbCache.put(db.getName().toLowerCase(), db);

          for (String tableName : msClient.getHiveClient().getAllTables(dbName)) {
            Table incompleteTbl =
                IncompleteTable.createUninitializedTable(getNextTableId(), db, tableName);
            incompleteTbl.setCatalogVersion(incrementAndGetCatalogVersion());
            db.addTable(incompleteTbl);
            if (loadInBackground_) {
              tblsToBackgroundLoad.add(
                  new TTableName(dbName.toLowerCase(), tableName.toLowerCase()));
            }
          }
        }
      } finally {
        msClient.release();
      }

      // Restore UDFs/UDAs.
      for (Pair<String, HashMap<String, List<Function>>> dbFns : functions) {
        Db db = null;
        try {
          db = newDbCache.get(dbFns.first);
        } catch (Exception e) {
          continue;
        }
        if (db == null) {
          // DB no longer exists - it was probably dropped externally.
          // TODO: We could restore this DB and then add the functions back?
          continue;
        }

        for (List<Function> fns : dbFns.second.values()) {
          for (Function fn : fns) {
            if (fn.getBinaryType() == TFunctionBinaryType.BUILTIN) continue;
            fn.setCatalogVersion(incrementAndGetCatalogVersion());
            db.addFunction(fn);
          }
        }
      }
      dbCache_.set(newDbCache);
      addDb(builtinsDb_);
      // Submit tables for background loading.
      for (TTableName tblName : tblsToBackgroundLoad) {
        tableLoadingMgr_.backgroundLoad(tblName);
      }
    } catch (Exception e) {
      LOG.error(e);
      throw new CatalogException("Error initializing Catalog. Catalog may be empty.", e);
    } finally {
      catalogLock_.writeLock().unlock();
    }
  }