/**
* Loads a table asynchronously, returning a LoadRequest that can be used to get the result (a
* Table). If there is already a load in flight for this table name, the same underlying loading
* task (Future) will be used, helping to prevent duplicate loads of the same table. Can also be
* used to perform an incremental refresh of an existing table, by passing the previous Table
* value in previousTbl. This may speedup the loading process, but may return a stale object.
*/
public LoadRequest loadAsync(final TTableName tblName, final Table previousTbl)
throws DatabaseNotFoundException {
final Db parentDb = catalog_.getDb(tblName.getDb_name());
if (parentDb == null) {
throw new DatabaseNotFoundException("Database '" + tblName.getDb_name() + "' was not found.");
}
FutureTask<Table> tableLoadTask =
new FutureTask<Table>(
new Callable<Table>() {
@Override
public Table call() throws Exception {
return tblLoader_.load(parentDb, tblName.table_name, previousTbl);
}
});
FutureTask<Table> existingValue = loadingTables_.putIfAbsent(tblName, tableLoadTask);
if (existingValue == null) {
// There was no existing value, submit a new load request.
tblLoadingPool_.execute(tableLoadTask);
} else {
tableLoadTask = existingValue;
}
return new LoadRequest(tblName, tableLoadTask);
}
/**
* Updates the cached lastDdlTime for the given table. The lastDdlTime is used during the metadata
* refresh() operations to determine if there have been any external (outside of Impala)
* modifications to the table.
*/
public void updateLastDdlTime(TTableName tblName, long ddlTime) {
Db db = getDb(tblName.getDb_name());
if (db == null) return;
Table tbl = db.getTable(tblName.getTable_name());
if (tbl == null) return;
tbl.updateLastDdlTime(ddlTime);
}
/**
* Gets the next table name to load off the head of the table loading queue. If the queue is
* empty, this will block until a new table is added.
*/
private void loadNextTable() throws InterruptedException {
// Always get the next table from the head of the deque.
final TTableName tblName = tableLoadingDeque_.takeFirst();
tableLoadingSet_.remove(tblName);
LOG.debug("Loading next table. Remaining items in queue: " + tableLoadingDeque_.size());
try {
// TODO: Instead of calling "getOrLoad" here we could call "loadAsync". We would
// just need to add a mechanism for moving loaded tables into the Catalog.
catalog_.getOrLoadTable(tblName.getDb_name(), tblName.getTable_name());
} catch (CatalogException e) {
// Ignore.
}
}
/**
* Renames a table. Equivalent to an atomic drop + add of the table. Returns the new Table object
* with an incremented catalog version or null if operation was not successful.
*/
public Table renameTable(TTableName oldTableName, TTableName newTableName)
throws CatalogException {
// Ensure the removal of the old table and addition of the new table happen
// atomically.
catalogLock_.writeLock().lock();
try {
// Remove the old table name from the cache and add the new table.
Db db = getDb(oldTableName.getDb_name());
if (db != null) db.removeTable(oldTableName.getTable_name());
return addTable(newTableName.getDb_name(), newTableName.getTable_name());
} finally {
catalogLock_.writeLock().unlock();
}
}
/**
* Blocks until the table has finished loading and returns the result. If any errors were
* encountered while loading the table an IncompleteTable will be returned.
*/
public Table get() {
Table tbl;
try {
tbl = tblTask_.get();
} catch (Exception e) {
tbl =
IncompleteTable.createFailedMetadataLoadTable(
TableId.createInvalidId(),
catalog_.getDb(tblName_.getDb_name()),
tblName_.getTable_name(),
new TableLoadingException(e.getMessage(), e));
}
Preconditions.checkState(tbl.isLoaded());
return tbl;
}
/**
* Reloads a table's metadata, reusing any existing cached metadata to speed up the operation.
* Returns the updated Table object or null if no table with this name exists in the catalog. If
* the existing table is dropped or modified (indicated by the catalog version changing) while the
* reload is in progress, the loaded value will be discarded and the current cached value will be
* returned. This may mean that a missing table (not yet loaded table) will be returned.
*/
public Table reloadTable(TTableName tblName) throws CatalogException {
LOG.debug(
String.format(
"Refreshing table metadata: %s.%s", tblName.getDb_name(), tblName.getTable_name()));
long previousCatalogVersion;
TableLoadingMgr.LoadRequest loadReq;
catalogLock_.readLock().lock();
try {
Table tbl = getTable(tblName.getDb_name(), tblName.getTable_name());
if (tbl == null) return null;
previousCatalogVersion = tbl.getCatalogVersion();
loadReq = tableLoadingMgr_.loadAsync(tblName, tbl);
} finally {
catalogLock_.readLock().unlock();
}
Preconditions.checkNotNull(loadReq);
try {
return replaceTableIfUnchanged(loadReq.get(), previousCatalogVersion);
} finally {
loadReq.close();
}
}
/**
* Invalidates the table in the catalog cache, potentially adding/removing the table from the
* cache based on whether it exists in the Hive Metastore. The invalidation logic is: - If the
* table exists in the metastore, add it to the catalog as an uninitialized IncompleteTable
* (replacing any existing entry). The table metadata will be loaded lazily, on the next access.
* If the parent database for this table does not yet exist in Impala's cache it will also be
* added. - If the table does not exist in the metastore, remove it from the catalog cache. - If
* we are unable to determine whether the table exists in the metastore (there was an exception
* thrown making the RPC), invalidate any existing Table by replacing it with an uninitialized
* IncompleteTable.
*
* <p>The parameter updatedObjects is a Pair that contains details on what catalog objects were
* modified as a result of the invalidateTable() call. The first item in the Pair is a Db which
* will only be set if a new database was added as a result of this call, otherwise it will be
* null. The second item in the Pair is the Table that was modified/added/removed. Returns a flag
* that indicates whether the items in updatedObjects were removed (returns true) or
* added/modified (return false). Only Tables should ever be removed.
*/
public boolean invalidateTable(TTableName tableName, Pair<Db, Table> updatedObjects) {
Preconditions.checkNotNull(updatedObjects);
updatedObjects.first = null;
updatedObjects.second = null;
LOG.debug(
String.format(
"Invalidating table metadata: %s.%s",
tableName.getDb_name(), tableName.getTable_name()));
String dbName = tableName.getDb_name();
String tblName = tableName.getTable_name();
// Stores whether the table exists in the metastore. Can have three states:
// 1) true - Table exists in metastore.
// 2) false - Table does not exist in metastore.
// 3) unknown (null) - There was exception thrown by the metastore client.
Boolean tableExistsInMetaStore;
MetaStoreClient msClient = getMetaStoreClient();
try {
tableExistsInMetaStore = msClient.getHiveClient().tableExists(dbName, tblName);
} catch (UnknownDBException e) {
// The parent database does not exist in the metastore. Treat this the same
// as if the table does not exist.
tableExistsInMetaStore = false;
} catch (TException e) {
LOG.error("Error executing tableExists() metastore call: " + tblName, e);
tableExistsInMetaStore = null;
} finally {
msClient.release();
}
if (tableExistsInMetaStore != null && !tableExistsInMetaStore) {
updatedObjects.second = removeTable(dbName, tblName);
return true;
} else {
Db db = getDb(dbName);
if ((db == null || !db.containsTable(tblName)) && tableExistsInMetaStore == null) {
// The table does not exist in our cache AND it is unknown whether the table
// exists in the metastore. Do nothing.
return false;
} else if (db == null && tableExistsInMetaStore) {
// The table exists in the metastore, but our cache does not contain the parent
// database. A new db will be added to the cache along with the new table.
db = new Db(dbName, this);
db.setCatalogVersion(incrementAndGetCatalogVersion());
addDb(db);
updatedObjects.first = db;
}
// Add a new uninitialized table to the table cache, effectively invalidating
// any existing entry. The metadata for the table will be loaded lazily, on the
// on the next access to the table.
Table newTable = IncompleteTable.createUninitializedTable(getNextTableId(), db, tblName);
newTable.setCatalogVersion(incrementAndGetCatalogVersion());
db.addTable(newTable);
if (loadInBackground_) {
tableLoadingMgr_.backgroundLoad(
new TTableName(dbName.toLowerCase(), tblName.toLowerCase()));
}
updatedObjects.second = newTable;
return false;
}
}