@Test public void TestColSchema() throws TableLoadingException { Db functionalDb = catalog_.getDb("functional"); assertNotNull(functionalDb); assertEquals(functionalDb.getName(), "functional"); assertNotNull(functionalDb.getTable("alltypes")); assertNotNull(functionalDb.getTable("alltypes_view")); assertNotNull(functionalDb.getTable("alltypes_view_sub")); assertNotNull(functionalDb.getTable("alltypessmall")); assertNotNull(functionalDb.getTable("alltypeserror")); assertNotNull(functionalDb.getTable("alltypeserrornonulls")); assertNotNull(functionalDb.getTable("alltypesagg")); assertNotNull(functionalDb.getTable("alltypesaggnonulls")); assertNotNull(functionalDb.getTable("alltypesnopart")); assertNotNull(functionalDb.getTable("alltypesinsert")); assertNotNull(functionalDb.getTable("complex_view")); assertNotNull(functionalDb.getTable("testtbl")); assertNotNull(functionalDb.getTable("dimtbl")); assertNotNull(functionalDb.getTable("jointbl")); assertNotNull(functionalDb.getTable("liketbl")); assertNotNull(functionalDb.getTable("greptiny")); assertNotNull(functionalDb.getTable("rankingssmall")); assertNotNull(functionalDb.getTable("uservisitssmall")); assertNotNull(functionalDb.getTable("view_view")); // IMP-163 - table with string partition column does not load if there are partitions assertNotNull(functionalDb.getTable("StringPartitionKey")); // Test non-existent table assertNull(functionalDb.getTable("nonexistenttable")); // functional_seq contains the same tables as functional Db testDb = catalog_.getDb("functional_seq"); assertNotNull(testDb); assertEquals(testDb.getName(), "functional_seq"); assertNotNull(testDb.getTable("alltypes")); assertNotNull(testDb.getTable("testtbl")); Db hbaseDb = catalog_.getDb("functional_hbase"); assertNotNull(hbaseDb); assertEquals(hbaseDb.getName(), "functional_hbase"); // Loading succeeds for an HBase table that has binary columns and an implicit key // column mapping assertNotNull(hbaseDb.getTable("alltypessmallbinary")); assertNotNull(hbaseDb.getTable("alltypessmall")); assertNotNull(hbaseDb.getTable("hbasealltypeserror")); assertNotNull(hbaseDb.getTable("hbasealltypeserrornonulls")); assertNotNull(hbaseDb.getTable("alltypesagg")); assertNotNull(hbaseDb.getTable("stringids")); checkTableCols( functionalDb, "alltypes", 2, new String[] { "year", "month", "id", "bool_col", "tinyint_col", "smallint_col", "int_col", "bigint_col", "float_col", "double_col", "date_string_col", "string_col", "timestamp_col" }, new ColumnType[] { ColumnType.INT, ColumnType.INT, ColumnType.INT, ColumnType.BOOLEAN, ColumnType.TINYINT, ColumnType.SMALLINT, ColumnType.INT, ColumnType.BIGINT, ColumnType.FLOAT, ColumnType.DOUBLE, ColumnType.STRING, ColumnType.STRING, ColumnType.TIMESTAMP }); checkTableCols( functionalDb, "testtbl", 0, new String[] {"id", "name", "zip"}, new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT}); checkTableCols( testDb, "testtbl", 0, new String[] {"id", "name", "zip"}, new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT}); checkTableCols( functionalDb, "liketbl", 0, new String[] { "str_col", "match_like_col", "no_match_like_col", "match_regex_col", "no_match_regex_col" }, new ColumnType[] { ColumnType.STRING, ColumnType.STRING, ColumnType.STRING, ColumnType.STRING, ColumnType.STRING }); checkTableCols( functionalDb, "dimtbl", 0, new String[] {"id", "name", "zip"}, new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT}); checkTableCols( functionalDb, "jointbl", 0, new String[] {"test_id", "test_name", "test_zip", "alltypes_id"}, new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT, ColumnType.INT}); checkHBaseTableCols( hbaseDb, "alltypessmall", "functional_hbase.alltypessmall", new String[] { "id", "bigint_col", "bool_col", "date_string_col", "double_col", "float_col", "int_col", "month", "smallint_col", "string_col", "timestamp_col", "tinyint_col", "year" }, new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"}, new String[] { null, "bigint_col", "bool_col", "date_string_col", "double_col", "float_col", "int_col", "month", "smallint_col", "string_col", "timestamp_col", "tinyint_col", "year" }, new ColumnType[] { ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN, ColumnType.STRING, ColumnType.DOUBLE, ColumnType.FLOAT, ColumnType.INT, ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP, ColumnType.TINYINT, ColumnType.INT }); checkHBaseTableCols( hbaseDb, "hbasealltypeserror", "functional_hbase.hbasealltypeserror", new String[] { "id", "bigint_col", "bool_col", "date_string_col", "double_col", "float_col", "int_col", "smallint_col", "string_col", "timestamp_col", "tinyint_col" }, new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"}, new String[] { null, "bigint_col", "bool_col", "date_string_col", "double_col", "float_col", "int_col", "smallint_col", "string_col", "timestamp_col", "tinyint_col" }, new ColumnType[] { ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN, ColumnType.STRING, ColumnType.DOUBLE, ColumnType.FLOAT, ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP, ColumnType.TINYINT }); checkHBaseTableCols( hbaseDb, "hbasealltypeserrornonulls", "functional_hbase.hbasealltypeserrornonulls", new String[] { "id", "bigint_col", "bool_col", "date_string_col", "double_col", "float_col", "int_col", "smallint_col", "string_col", "timestamp_col", "tinyint_col" }, new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"}, new String[] { null, "bigint_col", "bool_col", "date_string_col", "double_col", "float_col", "int_col", "smallint_col", "string_col", "timestamp_col", "tinyint_col" }, new ColumnType[] { ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN, ColumnType.STRING, ColumnType.DOUBLE, ColumnType.FLOAT, ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP, ColumnType.TINYINT }); checkHBaseTableCols( hbaseDb, "alltypesagg", "functional_hbase.alltypesagg", new String[] { "id", "bigint_col", "bool_col", "date_string_col", "day", "double_col", "float_col", "int_col", "month", "smallint_col", "string_col", "timestamp_col", "tinyint_col", "year" }, new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"}, new String[] { null, "bigint_col", "bool_col", "date_string_col", "day", "double_col", "float_col", "int_col", "month", "smallint_col", "string_col", "timestamp_col", "tinyint_col", "year" }, new ColumnType[] { ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN, ColumnType.STRING, ColumnType.INT, ColumnType.DOUBLE, ColumnType.FLOAT, ColumnType.INT, ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP, ColumnType.TINYINT, ColumnType.INT }); checkHBaseTableCols( hbaseDb, "stringids", "functional_hbase.alltypesagg", new String[] { "id", "bigint_col", "bool_col", "date_string_col", "day", "double_col", "float_col", "int_col", "month", "smallint_col", "string_col", "timestamp_col", "tinyint_col", "year" }, new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"}, new String[] { null, "bigint_col", "bool_col", "date_string_col", "day", "double_col", "float_col", "int_col", "month", "smallint_col", "string_col", "timestamp_col", "tinyint_col", "year" }, new ColumnType[] { ColumnType.STRING, ColumnType.BIGINT, ColumnType.BOOLEAN, ColumnType.STRING, ColumnType.INT, ColumnType.DOUBLE, ColumnType.FLOAT, ColumnType.INT, ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP, ColumnType.TINYINT, ColumnType.INT }); checkTableCols( functionalDb, "greptiny", 0, new String[] {"field"}, new ColumnType[] {ColumnType.STRING}); checkTableCols( functionalDb, "rankingssmall", 0, new String[] {"pagerank", "pageurl", "avgduration"}, new ColumnType[] {ColumnType.INT, ColumnType.STRING, ColumnType.INT}); checkTableCols( functionalDb, "uservisitssmall", 0, new String[] { "sourceip", "desturl", "visitdate", "adrevenue", "useragent", "ccode", "lcode", "skeyword", "avgtimeonsite" }, new ColumnType[] { ColumnType.STRING, ColumnType.STRING, ColumnType.STRING, ColumnType.FLOAT, ColumnType.STRING, ColumnType.STRING, ColumnType.STRING, ColumnType.STRING, ColumnType.INT }); // case-insensitive lookup assertEquals(functionalDb.getTable("alltypes"), functionalDb.getTable("AllTypes")); }
/** Resets this catalog instance by clearing all cached table and database metadata. */ public void reset() throws CatalogException { // First update the policy metadata. if (sentryProxy_ != null) { // Sentry Service is enabled. try { // Update the authorization policy, waiting for the result to complete. sentryProxy_.refresh(); } catch (Exception e) { throw new CatalogException("Error updating authorization policy: ", e); } } catalogLock_.writeLock().lock(); try { nextTableId_.set(0); // Since UDFs/UDAs are not persisted in the metastore, we won't clear // them across reset. To do this, we store all the functions before // clearing and restore them after. // TODO: Everything about this. Persist them. List<Pair<String, HashMap<String, List<Function>>>> functions = Lists.newArrayList(); for (Db db : dbCache_.get().values()) { if (db.numFunctions() == 0) continue; functions.add(Pair.create(db.getName(), db.getAllFunctions())); } // Build a new DB cache, populate it, and replace the existing cache in one // step. ConcurrentHashMap<String, Db> newDbCache = new ConcurrentHashMap<String, Db>(); List<TTableName> tblsToBackgroundLoad = Lists.newArrayList(); MetaStoreClient msClient = metaStoreClientPool_.getClient(); try { for (String dbName : msClient.getHiveClient().getAllDatabases()) { Db db = new Db(dbName, this); db.setCatalogVersion(incrementAndGetCatalogVersion()); newDbCache.put(db.getName().toLowerCase(), db); for (String tableName : msClient.getHiveClient().getAllTables(dbName)) { Table incompleteTbl = IncompleteTable.createUninitializedTable(getNextTableId(), db, tableName); incompleteTbl.setCatalogVersion(incrementAndGetCatalogVersion()); db.addTable(incompleteTbl); if (loadInBackground_) { tblsToBackgroundLoad.add( new TTableName(dbName.toLowerCase(), tableName.toLowerCase())); } } } } finally { msClient.release(); } // Restore UDFs/UDAs. for (Pair<String, HashMap<String, List<Function>>> dbFns : functions) { Db db = null; try { db = newDbCache.get(dbFns.first); } catch (Exception e) { continue; } if (db == null) { // DB no longer exists - it was probably dropped externally. // TODO: We could restore this DB and then add the functions back? continue; } for (List<Function> fns : dbFns.second.values()) { for (Function fn : fns) { if (fn.getBinaryType() == TFunctionBinaryType.BUILTIN) continue; fn.setCatalogVersion(incrementAndGetCatalogVersion()); db.addFunction(fn); } } } dbCache_.set(newDbCache); addDb(builtinsDb_); // Submit tables for background loading. for (TTableName tblName : tblsToBackgroundLoad) { tableLoadingMgr_.backgroundLoad(tblName); } } catch (Exception e) { LOG.error(e); throw new CatalogException("Error initializing Catalog. Catalog may be empty.", e); } finally { catalogLock_.writeLock().unlock(); } }