@Test
public void TestColSchema() throws TableLoadingException {
Db functionalDb = catalog_.getDb("functional");
assertNotNull(functionalDb);
assertEquals(functionalDb.getName(), "functional");
assertNotNull(functionalDb.getTable("alltypes"));
assertNotNull(functionalDb.getTable("alltypes_view"));
assertNotNull(functionalDb.getTable("alltypes_view_sub"));
assertNotNull(functionalDb.getTable("alltypessmall"));
assertNotNull(functionalDb.getTable("alltypeserror"));
assertNotNull(functionalDb.getTable("alltypeserrornonulls"));
assertNotNull(functionalDb.getTable("alltypesagg"));
assertNotNull(functionalDb.getTable("alltypesaggnonulls"));
assertNotNull(functionalDb.getTable("alltypesnopart"));
assertNotNull(functionalDb.getTable("alltypesinsert"));
assertNotNull(functionalDb.getTable("complex_view"));
assertNotNull(functionalDb.getTable("testtbl"));
assertNotNull(functionalDb.getTable("dimtbl"));
assertNotNull(functionalDb.getTable("jointbl"));
assertNotNull(functionalDb.getTable("liketbl"));
assertNotNull(functionalDb.getTable("greptiny"));
assertNotNull(functionalDb.getTable("rankingssmall"));
assertNotNull(functionalDb.getTable("uservisitssmall"));
assertNotNull(functionalDb.getTable("view_view"));
// IMP-163 - table with string partition column does not load if there are partitions
assertNotNull(functionalDb.getTable("StringPartitionKey"));
// Test non-existent table
assertNull(functionalDb.getTable("nonexistenttable"));
// functional_seq contains the same tables as functional
Db testDb = catalog_.getDb("functional_seq");
assertNotNull(testDb);
assertEquals(testDb.getName(), "functional_seq");
assertNotNull(testDb.getTable("alltypes"));
assertNotNull(testDb.getTable("testtbl"));
Db hbaseDb = catalog_.getDb("functional_hbase");
assertNotNull(hbaseDb);
assertEquals(hbaseDb.getName(), "functional_hbase");
// Loading succeeds for an HBase table that has binary columns and an implicit key
// column mapping
assertNotNull(hbaseDb.getTable("alltypessmallbinary"));
assertNotNull(hbaseDb.getTable("alltypessmall"));
assertNotNull(hbaseDb.getTable("hbasealltypeserror"));
assertNotNull(hbaseDb.getTable("hbasealltypeserrornonulls"));
assertNotNull(hbaseDb.getTable("alltypesagg"));
assertNotNull(hbaseDb.getTable("stringids"));
checkTableCols(
functionalDb,
"alltypes",
2,
new String[] {
"year",
"month",
"id",
"bool_col",
"tinyint_col",
"smallint_col",
"int_col",
"bigint_col",
"float_col",
"double_col",
"date_string_col",
"string_col",
"timestamp_col"
},
new ColumnType[] {
ColumnType.INT, ColumnType.INT, ColumnType.INT,
ColumnType.BOOLEAN, ColumnType.TINYINT, ColumnType.SMALLINT,
ColumnType.INT, ColumnType.BIGINT, ColumnType.FLOAT,
ColumnType.DOUBLE, ColumnType.STRING, ColumnType.STRING,
ColumnType.TIMESTAMP
});
checkTableCols(
functionalDb,
"testtbl",
0,
new String[] {"id", "name", "zip"},
new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT});
checkTableCols(
testDb,
"testtbl",
0,
new String[] {"id", "name", "zip"},
new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT});
checkTableCols(
functionalDb,
"liketbl",
0,
new String[] {
"str_col", "match_like_col", "no_match_like_col", "match_regex_col", "no_match_regex_col"
},
new ColumnType[] {
ColumnType.STRING,
ColumnType.STRING,
ColumnType.STRING,
ColumnType.STRING,
ColumnType.STRING
});
checkTableCols(
functionalDb,
"dimtbl",
0,
new String[] {"id", "name", "zip"},
new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT});
checkTableCols(
functionalDb,
"jointbl",
0,
new String[] {"test_id", "test_name", "test_zip", "alltypes_id"},
new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT, ColumnType.INT});
checkHBaseTableCols(
hbaseDb,
"alltypessmall",
"functional_hbase.alltypessmall",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new ColumnType[] {
ColumnType.INT,
ColumnType.BIGINT,
ColumnType.BOOLEAN,
ColumnType.STRING,
ColumnType.DOUBLE,
ColumnType.FLOAT,
ColumnType.INT,
ColumnType.INT,
ColumnType.SMALLINT,
ColumnType.STRING,
ColumnType.TIMESTAMP,
ColumnType.TINYINT,
ColumnType.INT
});
checkHBaseTableCols(
hbaseDb,
"hbasealltypeserror",
"functional_hbase.hbasealltypeserror",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col"
},
new ColumnType[] {
ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN,
ColumnType.STRING, ColumnType.DOUBLE, ColumnType.FLOAT,
ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING,
ColumnType.TIMESTAMP, ColumnType.TINYINT
});
checkHBaseTableCols(
hbaseDb,
"hbasealltypeserrornonulls",
"functional_hbase.hbasealltypeserrornonulls",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"double_col",
"float_col",
"int_col",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col"
},
new ColumnType[] {
ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN,
ColumnType.STRING, ColumnType.DOUBLE, ColumnType.FLOAT,
ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING,
ColumnType.TIMESTAMP, ColumnType.TINYINT
});
checkHBaseTableCols(
hbaseDb,
"alltypesagg",
"functional_hbase.alltypesagg",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"day",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"day",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new ColumnType[] {
ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN,
ColumnType.STRING, ColumnType.INT, ColumnType.DOUBLE,
ColumnType.FLOAT, ColumnType.INT, ColumnType.INT,
ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP,
ColumnType.TINYINT, ColumnType.INT
});
checkHBaseTableCols(
hbaseDb,
"stringids",
"functional_hbase.alltypesagg",
new String[] {
"id",
"bigint_col",
"bool_col",
"date_string_col",
"day",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
new String[] {
null,
"bigint_col",
"bool_col",
"date_string_col",
"day",
"double_col",
"float_col",
"int_col",
"month",
"smallint_col",
"string_col",
"timestamp_col",
"tinyint_col",
"year"
},
new ColumnType[] {
ColumnType.STRING, ColumnType.BIGINT, ColumnType.BOOLEAN,
ColumnType.STRING, ColumnType.INT, ColumnType.DOUBLE,
ColumnType.FLOAT, ColumnType.INT, ColumnType.INT,
ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP,
ColumnType.TINYINT, ColumnType.INT
});
checkTableCols(
functionalDb, "greptiny", 0, new String[] {"field"}, new ColumnType[] {ColumnType.STRING});
checkTableCols(
functionalDb,
"rankingssmall",
0,
new String[] {"pagerank", "pageurl", "avgduration"},
new ColumnType[] {ColumnType.INT, ColumnType.STRING, ColumnType.INT});
checkTableCols(
functionalDb,
"uservisitssmall",
0,
new String[] {
"sourceip",
"desturl",
"visitdate",
"adrevenue",
"useragent",
"ccode",
"lcode",
"skeyword",
"avgtimeonsite"
},
new ColumnType[] {
ColumnType.STRING, ColumnType.STRING, ColumnType.STRING,
ColumnType.FLOAT, ColumnType.STRING, ColumnType.STRING,
ColumnType.STRING, ColumnType.STRING, ColumnType.INT
});
// case-insensitive lookup
assertEquals(functionalDb.getTable("alltypes"), functionalDb.getTable("AllTypes"));
}
/** Resets this catalog instance by clearing all cached table and database metadata. */
public void reset() throws CatalogException {
// First update the policy metadata.
if (sentryProxy_ != null) {
// Sentry Service is enabled.
try {
// Update the authorization policy, waiting for the result to complete.
sentryProxy_.refresh();
} catch (Exception e) {
throw new CatalogException("Error updating authorization policy: ", e);
}
}
catalogLock_.writeLock().lock();
try {
nextTableId_.set(0);
// Since UDFs/UDAs are not persisted in the metastore, we won't clear
// them across reset. To do this, we store all the functions before
// clearing and restore them after.
// TODO: Everything about this. Persist them.
List<Pair<String, HashMap<String, List<Function>>>> functions = Lists.newArrayList();
for (Db db : dbCache_.get().values()) {
if (db.numFunctions() == 0) continue;
functions.add(Pair.create(db.getName(), db.getAllFunctions()));
}
// Build a new DB cache, populate it, and replace the existing cache in one
// step.
ConcurrentHashMap<String, Db> newDbCache = new ConcurrentHashMap<String, Db>();
List<TTableName> tblsToBackgroundLoad = Lists.newArrayList();
MetaStoreClient msClient = metaStoreClientPool_.getClient();
try {
for (String dbName : msClient.getHiveClient().getAllDatabases()) {
Db db = new Db(dbName, this);
db.setCatalogVersion(incrementAndGetCatalogVersion());
newDbCache.put(db.getName().toLowerCase(), db);
for (String tableName : msClient.getHiveClient().getAllTables(dbName)) {
Table incompleteTbl =
IncompleteTable.createUninitializedTable(getNextTableId(), db, tableName);
incompleteTbl.setCatalogVersion(incrementAndGetCatalogVersion());
db.addTable(incompleteTbl);
if (loadInBackground_) {
tblsToBackgroundLoad.add(
new TTableName(dbName.toLowerCase(), tableName.toLowerCase()));
}
}
}
} finally {
msClient.release();
}
// Restore UDFs/UDAs.
for (Pair<String, HashMap<String, List<Function>>> dbFns : functions) {
Db db = null;
try {
db = newDbCache.get(dbFns.first);
} catch (Exception e) {
continue;
}
if (db == null) {
// DB no longer exists - it was probably dropped externally.
// TODO: We could restore this DB and then add the functions back?
continue;
}
for (List<Function> fns : dbFns.second.values()) {
for (Function fn : fns) {
if (fn.getBinaryType() == TFunctionBinaryType.BUILTIN) continue;
fn.setCatalogVersion(incrementAndGetCatalogVersion());
db.addFunction(fn);
}
}
}
dbCache_.set(newDbCache);
addDb(builtinsDb_);
// Submit tables for background loading.
for (TTableName tblName : tblsToBackgroundLoad) {
tableLoadingMgr_.backgroundLoad(tblName);
}
} catch (Exception e) {
LOG.error(e);
throw new CatalogException("Error initializing Catalog. Catalog may be empty.", e);
} finally {
catalogLock_.writeLock().unlock();
}
}
