/** * Creates the tables used by the graph store. * * @param config Hadoop configuration * @param vertexDataHandler vertex storage handler * @param edgeDataHandler edge storage handler * @param graphDataHandler graph storage handler * @param vertexDataTableName vertex data table name * @param edgeTableName edge data table name * @param graphDataTableName graph data table name * @throws IOException */ private static void createTablesIfNotExists( final Configuration config, final VertexDataHandler vertexDataHandler, final EdgeDataHandler edgeDataHandler, final GraphDataHandler graphDataHandler, final String vertexDataTableName, final String edgeTableName, final String graphDataTableName) throws IOException { HTableDescriptor vertexDataTableDescriptor = new HTableDescriptor(TableName.valueOf(vertexDataTableName)); HTableDescriptor edgeDataTableDescriptor = new HTableDescriptor(TableName.valueOf(edgeTableName)); HTableDescriptor graphDataTableDescriptor = new HTableDescriptor(TableName.valueOf(graphDataTableName)); HBaseAdmin admin = new HBaseAdmin(config); if (!admin.tableExists(vertexDataTableDescriptor.getName())) { vertexDataHandler.createTable(admin, vertexDataTableDescriptor); } if (!admin.tableExists(edgeDataTableDescriptor.getName())) { edgeDataHandler.createTable(admin, edgeDataTableDescriptor); } if (!admin.tableExists(graphDataTableDescriptor.getName())) { graphDataHandler.createTable(admin, graphDataTableDescriptor); } admin.close(); }
/** * This test assumes a master crash/failure during the meta migration process and attempts to * continue the meta migration process when a new master takes over. When a master dies during the * meta migration we will have some rows of META.CatalogFamily updated with PB serialization and * some still hanging with writable serialization. When the backup master/ or fresh start of * master attempts the migration it will encounter some rows of META already updated with new HRI * and some still legacy. This test will simulate this scenario and validates that the migration * process can safely skip the updated rows and migrate any pending rows at startup. * * @throws Exception */ @Test public void testMasterCrashDuringMetaMigration() throws Exception { final byte[] FAMILY = Bytes.toBytes("family"); HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration"); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); htd.addFamily(hcd); Configuration conf = TEST_UTIL.getConfiguration(); // Create 10 New regions. createMultiRegionsWithPBSerialization(conf, htd.getName(), 10); // Create 10 Legacy regions. createMultiRegionsWithWritableSerialization(conf, htd.getName(), 10); CatalogTracker ct = TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker(); // Erase the current version of root meta for this test. undoVersionInRoot(ct); MetaReader.fullScanMetaAndPrint(ct); LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI"); long numMigratedRows = MetaMigrationConvertingToPB.updateMetaIfNecessary(TEST_UTIL.getHBaseCluster().getMaster()); assertEquals(numMigratedRows, 10); // Assert that the flag in ROOT is updated to reflect the correct status boolean metaUpdated = MetaMigrationConvertingToPB.isMetaTableUpdated( TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker()); assertEquals(true, metaUpdated); verifyMetaRowsAreUpdated(ct); LOG.info("END testMasterCrashDuringMetaMigration"); }
/** * Deletes the tables given tables. * * @param config cluster configuration * @param vertexDataTableName vertex data table name * @param edgeDataTableName edge data table name * @param graphDataTableName graph data table name * @throws IOException */ private static void deleteTablesIfExists( final Configuration config, final String vertexDataTableName, final String edgeDataTableName, final String graphDataTableName) throws IOException { HTableDescriptor vertexDataTableDescriptor = new HTableDescriptor(TableName.valueOf(vertexDataTableName)); HTableDescriptor edgeDataTableDescriptor = new HTableDescriptor(TableName.valueOf(edgeDataTableName)); HTableDescriptor graphsTableDescriptor = new HTableDescriptor(TableName.valueOf(graphDataTableName)); HBaseAdmin admin = new HBaseAdmin(config); if (admin.tableExists(vertexDataTableDescriptor.getName())) { deleteTable(admin, vertexDataTableDescriptor); } if (admin.tableExists(edgeDataTableDescriptor.getName())) { deleteTable(admin, edgeDataTableDescriptor); } if (admin.tableExists(graphsTableDescriptor.getName())) { deleteTable(admin, graphsTableDescriptor); } admin.close(); }
@Override public void add(HTableDescriptor htd) throws IOException { if (Bytes.equals(HConstants.ROOT_TABLE_NAME, htd.getName())) { throw new NotImplementedException(); } if (Bytes.equals(HConstants.META_TABLE_NAME, htd.getName())) { throw new NotImplementedException(); } if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(htd.getNameAsString())) { throw new NotImplementedException(); } if (!this.fsreadonly) updateHTableDescriptor(this.fs, this.rootdir, htd); long modtime = getTableInfoModtime(this.fs, this.rootdir, htd.getNameAsString()); this.cache.put(htd.getNameAsString(), new TableDescriptorModtime(modtime, htd)); }
@Test public void testMetaMigration() throws Exception { LOG.info("Starting testMetaMigration"); final byte[] FAMILY = Bytes.toBytes("family"); HTableDescriptor htd = new HTableDescriptor("testMetaMigration"); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); htd.addFamily(hcd); Configuration conf = TEST_UTIL.getConfiguration(); byte[][] regionNames = new byte[][] { HConstants.EMPTY_START_ROW, Bytes.toBytes("region_a"), Bytes.toBytes("region_b") }; createMultiRegionsWithWritableSerialization(conf, htd.getName(), regionNames); CatalogTracker ct = TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker(); // Erase the current version of root meta for this test. undoVersionInRoot(ct); MetaReader.fullScanMetaAndPrint(ct); LOG.info("Meta Print completed.testMetaMigration"); long numMigratedRows = MetaMigrationConvertingToPB.updateMeta(TEST_UTIL.getHBaseCluster().getMaster()); MetaReader.fullScanMetaAndPrint(ct); // Should be one entry only and it should be for the table we just added. assertEquals(regionNames.length, numMigratedRows); // Assert that the flag in ROOT is updated to reflect the correct status boolean metaUpdated = MetaMigrationConvertingToPB.isMetaTableUpdated( TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker()); assertEquals(true, metaUpdated); verifyMetaRowsAreUpdated(ct); }
/** * Create a new {@link HRegionInfo} from the snapshot region info. Keep the same startKey, endKey, * regionId and split information but change the table name. * * @param snapshotRegionInfo Info for region to clone. * @return the new HRegion instance */ public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) { return new HRegionInfo( tableDesc.getName(), snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(), snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId()); }
/** * Update table descriptor * * @param fs * @param conf * @param hTableDescriptor * @return New tableinfo or null if we failed update. * @throws IOException Thrown if failed update. */ static Path updateHTableDescriptor(FileSystem fs, Path rootdir, HTableDescriptor hTableDescriptor) throws IOException { Path tableDir = FSUtils.getTablePath(rootdir, hTableDescriptor.getName()); Path p = writeTableDescriptor(fs, hTableDescriptor, tableDir, getTableInfoPath(fs, tableDir)); if (p == null) throw new IOException("Failed update"); LOG.info("Updated tableinfo=" + p); return p; }
@Override public void setUp() throws Exception { // setup config values necessary for store this.conf = TEST_UTIL.getConfiguration(); this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0); this.conf.setInt("hbase.hstore.compaction.min", minFiles); this.conf.setInt("hbase.hstore.compaction.max", maxFiles); this.conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, minSize); this.conf.setLong("hbase.hstore.compaction.max.size", maxSize); this.conf.setFloat("hbase.hstore.compaction.ratio", 1.0F); // Setting up a Store Path basedir = new Path(DIR); String logName = "logs"; Path logdir = new Path(DIR, logName); Path oldLogDir = new Path(basedir, HConstants.HREGION_OLDLOGDIR_NAME); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family")); FileSystem fs = FileSystem.get(conf); fs.delete(logdir, true); HTableDescriptor htd = new HTableDescriptor(Bytes.toBytes("table")); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); hlog = HLogFactory.createHLog(fs, basedir, logName, conf); region = HRegion.createHRegion(info, basedir, conf, htd); HRegion.closeHRegion(region); Path tableDir = new Path(basedir, Bytes.toString(htd.getName())); region = new HRegion(tableDir, hlog, fs, conf, info, htd, null); store = new HStore(basedir, region, hcd, fs, conf); manager = store.compactionPolicy; TEST_FILE = StoreFile.getRandomFilename(fs, store.getHomedir()); fs.create(TEST_FILE); }
private void initHRegion( byte[] tableName, String callingMethod, Configuration conf, byte[]... families) throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); for (byte[] family : families) { htd.addFamily(new HColumnDescriptor(family)); } HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); Path path = new Path(DIR + callingMethod); if (fs.exists(path)) { if (!fs.delete(path, true)) { throw new IOException("Failed delete of " + path); } } region = HRegion.createHRegion(info, path, conf, htd); }
@Test public void testCreateAndUpdate() throws IOException { Path testdir = UTIL.getDataTestDir("testCreateAndUpdate"); HTableDescriptor htd = new HTableDescriptor("testCreate"); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); assertFalse(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); FileStatus[] statuses = fs.listStatus(testdir); assertTrue("statuses.length=" + statuses.length, statuses.length == 1); for (int i = 0; i < 10; i++) { FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); } statuses = fs.listStatus(testdir); assertTrue(statuses.length == 1); Path tmpTableDir = new Path(FSUtils.getTablePath(testdir, htd.getName()), ".tmp"); statuses = fs.listStatus(tmpTableDir); assertTrue(statuses.length == 0); }
/** * Callers must afterward call {@link HRegion#closeHRegion(HRegion)} * * @param tableName * @param callingMethod * @param conf * @param families * @throws IOException * @return created and initialized region. */ private HRegion initHRegion( byte[] tableName, String callingMethod, HBaseConfiguration conf, String family) throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor familyDesc; for (int i = 0; i < BLOOM_TYPE.length; i++) { BloomType bloomType = BLOOM_TYPE[i]; familyDesc = new HColumnDescriptor(family + "_" + bloomType) .setBlocksize(1) .setBloomFilterType(BLOOM_TYPE[i]); htd.addFamily(familyDesc); } HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); Path path = new Path(DIR + callingMethod); HRegion r = HRegion.createHRegion(info, path, conf, htd); blockCache = new CacheConfig(conf).getBlockCache(); return r; }
// Initialize and set up tables public static void initialize() throws Exception { setupLog4j(); Configuration config = HBaseConfiguration.create(); HTableDescriptor desc = new HTableDescriptor(TABLE_NAME); desc.addFamily(new HColumnDescriptor(FAMILY)); admin = new HBaseAdmin(config); try { System.out.println("Creating the table " + TABLE_NAME); admin.createTable(desc); } catch (TableExistsException e) { System.out.println("Table " + TABLE_NAME + " already exists"); } table = new TransactionalTable(config, desc.getName()); try { transactionManager = new TransactionManager(config); } catch (Exception e) { throw new RuntimeException(e); } }
/** * Deletes a HBase table. * * @param admin HBase admin * @param tableDescriptor descriptor for the table to delete * @throws IOException */ private static void deleteTable(final HBaseAdmin admin, final HTableDescriptor tableDescriptor) throws IOException { LOG.info("deleting table: " + tableDescriptor.getNameAsString()); admin.disableTable(tableDescriptor.getName()); admin.deleteTable(tableDescriptor.getName()); }
@After public void tearDown() throws Exception { for (HTableDescriptor htd : this.admin.listTables()) { TEST_UTIL.deleteTable(htd.getName()); } }