/**
* Updates the cached lastDdlTime for the given table. The lastDdlTime is used during the metadata
* refresh() operations to determine if there have been any external (outside of Impala)
* modifications to the table.
*/
public void updateLastDdlTime(TTableName tblName, long ddlTime) {
Db db = getDb(tblName.getDb_name());
if (db == null) return;
Table tbl = db.getTable(tblName.getTable_name());
if (tbl == null) return;
tbl.updateLastDdlTime(ddlTime);
}
/**
* Adds a table with the given name to the catalog and returns the new table, loading the metadata
* if needed.
*/
public Table addTable(String dbName, String tblName) throws TableNotFoundException {
Db db = getDb(dbName);
if (db == null) return null;
Table incompleteTable = IncompleteTable.createUninitializedTable(getNextTableId(), db, tblName);
incompleteTable.setCatalogVersion(incrementAndGetCatalogVersion());
db.addTable(incompleteTable);
return db.getTable(tblName);
}
private void checkTableCols(
Db db, String tblName, int numClusteringCols, String[] colNames, ColumnType[] colTypes)
throws TableLoadingException {
Table tbl = db.getTable(tblName);
assertEquals(tbl.getName(), tblName);
assertEquals(tbl.getNumClusteringCols(), numClusteringCols);
List<Column> cols = tbl.getColumns();
assertEquals(colNames.length, colTypes.length);
assertEquals(cols.size(), colNames.length);
Iterator<Column> it = cols.iterator();
int i = 0;
while (it.hasNext()) {
Column col = it.next();
assertEquals(col.getName(), colNames[i]);
assertTrue(col.getType().equals(colTypes[i]));
++i;
}
}
/**
* Replaces an existing Table with a new value if it exists and has not changed (has the same
* catalog version as 'expectedCatalogVersion').
*/
private Table replaceTableIfUnchanged(Table updatedTbl, long expectedCatalogVersion)
throws DatabaseNotFoundException {
catalogLock_.writeLock().lock();
try {
Db db = getDb(updatedTbl.getDb().getName());
if (db == null) {
throw new DatabaseNotFoundException(
"Database does not exist: " + updatedTbl.getDb().getName());
}
Table existingTbl = db.getTable(updatedTbl.getName());
// The existing table does not exist or has been modified. Instead of
// adding the loaded value, return the existing table.
if (existingTbl == null || existingTbl.getCatalogVersion() != expectedCatalogVersion)
return existingTbl;
updatedTbl.setCatalogVersion(incrementAndGetCatalogVersion());
db.addTable(updatedTbl);
return updatedTbl;
} finally {
catalogLock_.writeLock().unlock();
}
}
private void checkHBaseTableCols(
Db db,
String hiveTableName,
String hbaseTableName,
String[] hiveColNames,
String[] colFamilies,
String[] colQualifiers,
ColumnType[] colTypes)
throws TableLoadingException {
checkTableCols(db, hiveTableName, 1, hiveColNames, colTypes);
HBaseTable tbl = (HBaseTable) db.getTable(hiveTableName);
assertEquals(tbl.getHBaseTableName(), hbaseTableName);
List<Column> cols = tbl.getColumns();
assertEquals(colFamilies.length, colTypes.length);
assertEquals(colQualifiers.length, colTypes.length);
Iterator<Column> it = cols.iterator();
int i = 0;
while (it.hasNext()) {
HBaseColumn col = (HBaseColumn) it.next();
assertEquals(col.getColumnFamily(), colFamilies[i]);
assertEquals(col.getColumnQualifier(), colQualifiers[i]);
++i;
}
}
@Test
public void TestColSchema() throws TableLoadingException {
Db functionalDb = catalog_.getDb("functional");
assertNotNull(functionalDb);
assertEquals(functionalDb.getName(), "functional");
assertNotNull(functionalDb.getTable("alltypes"));
assertNotNull(functionalDb.getTable("alltypes_view"));
assertNotNull(functionalDb.getTable("alltypes_view_sub"));
assertNotNull(functionalDb.getTable("alltypessmall"));
assertNotNull(functionalDb.getTable("alltypeserror"));
assertNotNull(functionalDb.getTable("alltypeserrornonulls"));
assertNotNull(functionalDb.getTable("alltypesagg"));
assertNotNull(functionalDb.getTable("alltypesaggnonulls"));
assertNotNull(functionalDb.getTable("alltypesnopart"));
assertNotNull(functionalDb.getTable("alltypesinsert"));
assertNotNull(functionalDb.getTable("complex_view"));
assertNotNull(functionalDb.getTable("testtbl"));
assertNotNull(functionalDb.getTable("dimtbl"));
assertNotNull(functionalDb.getTable("jointbl"));
assertNotNull(functionalDb.getTable("liketbl"));
assertNotNull(functionalDb.getTable("greptiny"));
assertNotNull(functionalDb.getTable("rankingssmall"));
assertNotNull(functionalDb.getTable("uservisitssmall"));
assertNotNull(functionalDb.getTable("view_view"));
// IMP-163 - table with string partition column does not load if there are partitions
assertNotNull(functionalDb.getTable("StringPartitionKey"));
// Test non-existent table
assertNull(functionalDb.getTable("nonexistenttable"));
// functional_seq contains the same tables as functional
Db testDb = catalog_.getDb("functional_seq");
assertNotNull(testDb);
assertEquals(testDb.getName(), "functional_seq");
assertNotNull(testDb.getTable("alltypes"));
assertNotNull(testDb.getTable("testtbl"));
Db hbaseDb = catalog_.getDb("functional_hbase");
assertNotNull(hbaseDb);
assertEquals(hbaseDb.getName(), "functional_hbase");
// Loading succeeds for an HBase table that has binary columns and an implicit key
// column mapping
assertNotNull(hbaseDb.getTable("alltypessmallbinary"));
assertNotNull(hbaseDb.getTable("alltypessmall"));
assertNotNull(hbaseDb.getTable("hbasealltypeserror"));
assertNotNull(hbaseDb.getTable("hbasealltypeserrornonulls"));
assertNotNull(hbaseDb.getTable("alltypesagg"));
assertNotNull(hbaseDb.getTable("stringids"));
checkTableCols(
functionalDb,
"alltypes",
2,
new String[] {
"year",
"month",
"id",
"bool_col",
"tinyint_col",
"smallint_col",
"int_col",
"bigint_col",
"float_col",
"double_col",
"date_string_col",
"string_col",
"timestamp_col"
},
new ColumnType[] {
ColumnType.INT, ColumnType.INT, ColumnType.INT,
ColumnType.BOOLEAN, ColumnType.TINYINT, ColumnType.SMALLINT,
ColumnType.INT, ColumnType.BIGINT, ColumnType.FLOAT,
ColumnType.DOUBLE, ColumnType.STRING, ColumnType.STRING,
ColumnType.TIMESTAMP
});
checkTableCols(
functionalDb,
"testtbl",
0,
new String[] {"id", "name", "zip"},
new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT});
checkTableCols(
testDb,
"testtbl",
0,
new String[] {"id", "name", "zip"},
new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT});
checkTableCols(
functionalDb,
"liketbl",
0,
new String[] {
"str_col", "match_like_col", "no_match_like_col", "match_regex_col", "no_match_regex_col"
},
new ColumnType[] {
ColumnType.STRING,
ColumnType.STRING,
ColumnType.STRING,
ColumnType.STRING,
ColumnType.STRING
});
checkTableCols(
functionalDb,
"dimtbl",
0,
new String[] {"id", "name", "zip"},
new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT});
checkTableCols(
functionalDb,
"jointbl",
0,
new String[] {"test_id", "test_name", "test_zip", "alltypes_id"},
new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT, ColumnType.INT});
checkHBaseTableCols(
hbaseDb,
"alltypessmall",
"functional_hbase.alltypessmall",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new ColumnType[] {
ColumnType.INT,
ColumnType.BIGINT,
ColumnType.BOOLEAN,
ColumnType.STRING,
ColumnType.DOUBLE,
ColumnType.FLOAT,
ColumnType.INT,
ColumnType.INT,
ColumnType.SMALLINT,
ColumnType.STRING,
ColumnType.TIMESTAMP,
ColumnType.TINYINT,
ColumnType.INT
});
checkHBaseTableCols(
hbaseDb,
"hbasealltypeserror",
"functional_hbase.hbasealltypeserror",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col"
},
new ColumnType[] {
ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN,
ColumnType.STRING, ColumnType.DOUBLE, ColumnType.FLOAT,
ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING,
ColumnType.TIMESTAMP, ColumnType.TINYINT
});
checkHBaseTableCols(
hbaseDb,
"hbasealltypeserrornonulls",
"functional_hbase.hbasealltypeserrornonulls",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col"
},
new ColumnType[] {
ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN,
ColumnType.STRING, ColumnType.DOUBLE, ColumnType.FLOAT,
ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING,
ColumnType.TIMESTAMP, ColumnType.TINYINT
});
checkHBaseTableCols(
hbaseDb,
"alltypesagg",
"functional_hbase.alltypesagg",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"day",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"day",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new ColumnType[] {
ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN,
ColumnType.STRING, ColumnType.INT, ColumnType.DOUBLE,
ColumnType.FLOAT, ColumnType.INT, ColumnType.INT,
ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP,
ColumnType.TINYINT, ColumnType.INT
});
checkHBaseTableCols(
hbaseDb,
"stringids",
"functional_hbase.alltypesagg",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"day",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"day",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new ColumnType[] {
ColumnType.STRING, ColumnType.BIGINT, ColumnType.BOOLEAN,
ColumnType.STRING, ColumnType.INT, ColumnType.DOUBLE,
ColumnType.FLOAT, ColumnType.INT, ColumnType.INT,
ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP,
ColumnType.TINYINT, ColumnType.INT
});
checkTableCols(
functionalDb, "greptiny", 0, new String[] {"field"}, new ColumnType[] {ColumnType.STRING});
checkTableCols(
functionalDb,
"rankingssmall",
0,
new String[] {"pagerank", "pageurl", "avgduration"},
new ColumnType[] {ColumnType.INT, ColumnType.STRING, ColumnType.INT});
checkTableCols(
functionalDb,
"uservisitssmall",
0,
new String[] {
"sourceip",
"desturl",
"visitdate",
"adrevenue",
"useragent",
"ccode",
"lcode",
"skeyword",
"avgtimeonsite"
},
new ColumnType[] {
ColumnType.STRING, ColumnType.STRING, ColumnType.STRING,
ColumnType.FLOAT, ColumnType.STRING, ColumnType.STRING,
ColumnType.STRING, ColumnType.STRING, ColumnType.INT
});
// case-insensitive lookup
assertEquals(functionalDb.getTable("alltypes"), functionalDb.getTable("AllTypes"));
}
/**
* Returns all known objects in the Catalog (Tables, Views, Databases, and Functions). Some
* metadata may be skipped for objects that have a catalog version < the specified "fromVersion".
* Skips builtins.
*/
public TGetAllCatalogObjectsResponse getCatalogObjects(long fromVersion) {
TGetAllCatalogObjectsResponse resp = new TGetAllCatalogObjectsResponse();
resp.setObjects(new ArrayList<TCatalogObject>());
resp.setMax_catalog_version(Catalog.INITIAL_CATALOG_VERSION);
// Take a lock on the catalog to ensure this update contains a consistent snapshot
// of all items in the catalog.
catalogLock_.readLock().lock();
try {
for (String dbName : getDbNames(null)) {
Db db = getDb(dbName);
if (db == null) {
LOG.error(
"Database: "
+ dbName
+ " was expected to be in the catalog "
+ "cache. Skipping database and all child objects for this update.");
continue;
}
if (db.isSystemDb()) continue;
TCatalogObject catalogDb =
new TCatalogObject(TCatalogObjectType.DATABASE, db.getCatalogVersion());
catalogDb.setDb(db.toThrift());
resp.addToObjects(catalogDb);
for (String tblName : db.getAllTableNames()) {
TCatalogObject catalogTbl =
new TCatalogObject(TCatalogObjectType.TABLE, Catalog.INITIAL_CATALOG_VERSION);
Table tbl = db.getTable(tblName);
if (tbl == null) {
LOG.error(
"Table: "
+ tblName
+ " was expected to be in the catalog "
+ "cache. Skipping table for this update.");
continue;
}
// Only add the extended metadata if this table's version is >=
// the fromVersion.
if (tbl.getCatalogVersion() >= fromVersion) {
try {
catalogTbl.setTable(tbl.toThrift());
} catch (Exception e) {
LOG.debug(
String.format(
"Error calling toThrift() on table %s.%s: %s",
dbName, tblName, e.getMessage()),
e);
continue;
}
catalogTbl.setCatalog_version(tbl.getCatalogVersion());
} else {
catalogTbl.setTable(new TTable(dbName, tblName));
}
resp.addToObjects(catalogTbl);
}
for (Function fn : db.getFunctions(null, new PatternMatcher())) {
TCatalogObject function =
new TCatalogObject(TCatalogObjectType.FUNCTION, fn.getCatalogVersion());
function.setFn(fn.toThrift());
resp.addToObjects(function);
}
}
for (DataSource dataSource : getDataSources()) {
TCatalogObject catalogObj =
new TCatalogObject(TCatalogObjectType.DATA_SOURCE, dataSource.getCatalogVersion());
catalogObj.setData_source(dataSource.toThrift());
resp.addToObjects(catalogObj);
}
for (HdfsCachePool cachePool : hdfsCachePools_) {
TCatalogObject pool =
new TCatalogObject(TCatalogObjectType.HDFS_CACHE_POOL, cachePool.getCatalogVersion());
pool.setCache_pool(cachePool.toThrift());
resp.addToObjects(pool);
}
// Get all roles
for (Role role : authPolicy_.getAllRoles()) {
TCatalogObject thriftRole = new TCatalogObject();
thriftRole.setRole(role.toThrift());
thriftRole.setCatalog_version(role.getCatalogVersion());
thriftRole.setType(role.getCatalogObjectType());
resp.addToObjects(thriftRole);
for (RolePrivilege p : role.getPrivileges()) {
TCatalogObject privilege = new TCatalogObject();
privilege.setPrivilege(p.toThrift());
privilege.setCatalog_version(p.getCatalogVersion());
privilege.setType(p.getCatalogObjectType());
resp.addToObjects(privilege);
}
}
// Each update should contain a single "TCatalog" object which is used to
// pass overall state on the catalog, such as the current version and the
// catalog service id.
TCatalogObject catalog = new TCatalogObject();
catalog.setType(TCatalogObjectType.CATALOG);
// By setting the catalog version to the latest catalog version at this point,
// it ensure impalads will always bump their versions, even in the case where
// an object has been dropped.
catalog.setCatalog_version(getCatalogVersion());
catalog.setCatalog(new TCatalog(catalogServiceId_));
resp.addToObjects(catalog);
// The max version is the max catalog version of all items in the update.
resp.setMax_catalog_version(getCatalogVersion());
return resp;
} finally {
catalogLock_.readLock().unlock();
}
}