/** * Loads a table asynchronously, returning a LoadRequest that can be used to get the result (a * Table). If there is already a load in flight for this table name, the same underlying loading * task (Future) will be used, helping to prevent duplicate loads of the same table. Can also be * used to perform an incremental refresh of an existing table, by passing the previous Table * value in previousTbl. This may speedup the loading process, but may return a stale object. */ public LoadRequest loadAsync(final TTableName tblName, final Table previousTbl) throws DatabaseNotFoundException { final Db parentDb = catalog_.getDb(tblName.getDb_name()); if (parentDb == null) { throw new DatabaseNotFoundException("Database '" + tblName.getDb_name() + "' was not found."); } FutureTask<Table> tableLoadTask = new FutureTask<Table>( new Callable<Table>() { @Override public Table call() throws Exception { return tblLoader_.load(parentDb, tblName.table_name, previousTbl); } }); FutureTask<Table> existingValue = loadingTables_.putIfAbsent(tblName, tableLoadTask); if (existingValue == null) { // There was no existing value, submit a new load request. tblLoadingPool_.execute(tableLoadTask); } else { tableLoadTask = existingValue; } return new LoadRequest(tblName, tableLoadTask); }
/** * Updates the cached lastDdlTime for the given table. The lastDdlTime is used during the metadata * refresh() operations to determine if there have been any external (outside of Impala) * modifications to the table. */ public void updateLastDdlTime(TTableName tblName, long ddlTime) { Db db = getDb(tblName.getDb_name()); if (db == null) return; Table tbl = db.getTable(tblName.getTable_name()); if (tbl == null) return; tbl.updateLastDdlTime(ddlTime); }
/** * Gets the next table name to load off the head of the table loading queue. If the queue is * empty, this will block until a new table is added. */ private void loadNextTable() throws InterruptedException { // Always get the next table from the head of the deque. final TTableName tblName = tableLoadingDeque_.takeFirst(); tableLoadingSet_.remove(tblName); LOG.debug("Loading next table. Remaining items in queue: " + tableLoadingDeque_.size()); try { // TODO: Instead of calling "getOrLoad" here we could call "loadAsync". We would // just need to add a mechanism for moving loaded tables into the Catalog. catalog_.getOrLoadTable(tblName.getDb_name(), tblName.getTable_name()); } catch (CatalogException e) { // Ignore. } }
/** * Renames a table. Equivalent to an atomic drop + add of the table. Returns the new Table object * with an incremented catalog version or null if operation was not successful. */ public Table renameTable(TTableName oldTableName, TTableName newTableName) throws CatalogException { // Ensure the removal of the old table and addition of the new table happen // atomically. catalogLock_.writeLock().lock(); try { // Remove the old table name from the cache and add the new table. Db db = getDb(oldTableName.getDb_name()); if (db != null) db.removeTable(oldTableName.getTable_name()); return addTable(newTableName.getDb_name(), newTableName.getTable_name()); } finally { catalogLock_.writeLock().unlock(); } }
/** * Blocks until the table has finished loading and returns the result. If any errors were * encountered while loading the table an IncompleteTable will be returned. */ public Table get() { Table tbl; try { tbl = tblTask_.get(); } catch (Exception e) { tbl = IncompleteTable.createFailedMetadataLoadTable( TableId.createInvalidId(), catalog_.getDb(tblName_.getDb_name()), tblName_.getTable_name(), new TableLoadingException(e.getMessage(), e)); } Preconditions.checkState(tbl.isLoaded()); return tbl; }
/** * Reloads a table's metadata, reusing any existing cached metadata to speed up the operation. * Returns the updated Table object or null if no table with this name exists in the catalog. If * the existing table is dropped or modified (indicated by the catalog version changing) while the * reload is in progress, the loaded value will be discarded and the current cached value will be * returned. This may mean that a missing table (not yet loaded table) will be returned. */ public Table reloadTable(TTableName tblName) throws CatalogException { LOG.debug( String.format( "Refreshing table metadata: %s.%s", tblName.getDb_name(), tblName.getTable_name())); long previousCatalogVersion; TableLoadingMgr.LoadRequest loadReq; catalogLock_.readLock().lock(); try { Table tbl = getTable(tblName.getDb_name(), tblName.getTable_name()); if (tbl == null) return null; previousCatalogVersion = tbl.getCatalogVersion(); loadReq = tableLoadingMgr_.loadAsync(tblName, tbl); } finally { catalogLock_.readLock().unlock(); } Preconditions.checkNotNull(loadReq); try { return replaceTableIfUnchanged(loadReq.get(), previousCatalogVersion); } finally { loadReq.close(); } }
/** * Invalidates the table in the catalog cache, potentially adding/removing the table from the * cache based on whether it exists in the Hive Metastore. The invalidation logic is: - If the * table exists in the metastore, add it to the catalog as an uninitialized IncompleteTable * (replacing any existing entry). The table metadata will be loaded lazily, on the next access. * If the parent database for this table does not yet exist in Impala's cache it will also be * added. - If the table does not exist in the metastore, remove it from the catalog cache. - If * we are unable to determine whether the table exists in the metastore (there was an exception * thrown making the RPC), invalidate any existing Table by replacing it with an uninitialized * IncompleteTable. * * <p>The parameter updatedObjects is a Pair that contains details on what catalog objects were * modified as a result of the invalidateTable() call. The first item in the Pair is a Db which * will only be set if a new database was added as a result of this call, otherwise it will be * null. The second item in the Pair is the Table that was modified/added/removed. Returns a flag * that indicates whether the items in updatedObjects were removed (returns true) or * added/modified (return false). Only Tables should ever be removed. */ public boolean invalidateTable(TTableName tableName, Pair<Db, Table> updatedObjects) { Preconditions.checkNotNull(updatedObjects); updatedObjects.first = null; updatedObjects.second = null; LOG.debug( String.format( "Invalidating table metadata: %s.%s", tableName.getDb_name(), tableName.getTable_name())); String dbName = tableName.getDb_name(); String tblName = tableName.getTable_name(); // Stores whether the table exists in the metastore. Can have three states: // 1) true - Table exists in metastore. // 2) false - Table does not exist in metastore. // 3) unknown (null) - There was exception thrown by the metastore client. Boolean tableExistsInMetaStore; MetaStoreClient msClient = getMetaStoreClient(); try { tableExistsInMetaStore = msClient.getHiveClient().tableExists(dbName, tblName); } catch (UnknownDBException e) { // The parent database does not exist in the metastore. Treat this the same // as if the table does not exist. tableExistsInMetaStore = false; } catch (TException e) { LOG.error("Error executing tableExists() metastore call: " + tblName, e); tableExistsInMetaStore = null; } finally { msClient.release(); } if (tableExistsInMetaStore != null && !tableExistsInMetaStore) { updatedObjects.second = removeTable(dbName, tblName); return true; } else { Db db = getDb(dbName); if ((db == null || !db.containsTable(tblName)) && tableExistsInMetaStore == null) { // The table does not exist in our cache AND it is unknown whether the table // exists in the metastore. Do nothing. return false; } else if (db == null && tableExistsInMetaStore) { // The table exists in the metastore, but our cache does not contain the parent // database. A new db will be added to the cache along with the new table. db = new Db(dbName, this); db.setCatalogVersion(incrementAndGetCatalogVersion()); addDb(db); updatedObjects.first = db; } // Add a new uninitialized table to the table cache, effectively invalidating // any existing entry. The metadata for the table will be loaded lazily, on the // on the next access to the table. Table newTable = IncompleteTable.createUninitializedTable(getNextTableId(), db, tblName); newTable.setCatalogVersion(incrementAndGetCatalogVersion()); db.addTable(newTable); if (loadInBackground_) { tableLoadingMgr_.backgroundLoad( new TTableName(dbName.toLowerCase(), tblName.toLowerCase())); } updatedObjects.second = newTable; return false; } }