protected void createTable(String tableName) throws Exception { HTableDescriptor td = new HTableDescriptor(tableName.getBytes()); HColumnDescriptor cd = new HColumnDescriptor(TEST_TABLE_CF.getBytes()); td.addFamily(cd); admin.createTable(td); LOG.info(tableName + " table is successfully created."); }
/** Test default value handling for memStoreFlushSize */ @Test public void testGetMemStoreFlushSize() { HTableDescriptor desc = new HTableDescriptor("table"); assertEquals(-1, desc.getMemStoreFlushSize()); desc.setMemStoreFlushSize(1111L); assertEquals(1111L, desc.getMemStoreFlushSize()); }
@Override void perform() throws IOException { Admin admin = connection.getAdmin(); try { HTableDescriptor htd = createTableDesc(); TableName tableName = htd.getTableName(); if (admin.tableExists(tableName)) { return; } String numRegionKey = String.format(NUM_REGIONS_KEY, this.getClass().getSimpleName()); numRegions = getConf().getInt(numRegionKey, DEFAULT_NUM_REGIONS); byte[] startKey = Bytes.toBytes("row-0000000000"); byte[] endKey = Bytes.toBytes("row-" + Integer.MAX_VALUE); LOG.info("Creating table:" + htd); admin.createTable(htd, startKey, endKey, numRegions); Assert.assertTrue("Table: " + htd + " was not created", admin.tableExists(tableName)); HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName); enabledTables.put(tableName, freshTableDesc); LOG.info("Created table:" + freshTableDesc); } catch (Exception e) { LOG.warn("Caught exception in action: " + this.getClass()); throw e; } finally { admin.close(); } verifyTables(); }
public void createTable(String tableName, List<String> ColumnFamilies) { HBaseAdmin admin = null; try { admin = new HBaseAdmin(conf); HTableDescriptor tableDescriptor = new HTableDescriptor(Bytes.toBytes(tableName)); for (String columnFamily : ColumnFamilies) { HColumnDescriptor columnDescriptor = new HColumnDescriptor(columnFamily); tableDescriptor.addFamily(columnDescriptor); } admin.createTable(tableDescriptor); admin.close(); } catch (TableExistsException e) { System.out.println("Table already exist:" + tableName); try { admin.close(); } catch (IOException e1) { System.out.println("Error occurred while cloing the HBaseAdmin conneciton:" + e1); } } catch (MasterNotRunningException e) { throw new RuntimeException("HBase master not running, table creation failed."); } catch (ZooKeeperConnectionException e) { throw new RuntimeException("Zookeeper not running, table creation failed."); } catch (IOException e) { throw new RuntimeException("IO error, table creation failed."); } }
public static void CreateTable(HBaseConfiguration conf) throws IOException { HBaseAdmin admin = new HBaseAdmin(conf); HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf("people")); tableDescriptor.addFamily(new HColumnDescriptor("personal")); tableDescriptor.addFamily(new HColumnDescriptor("contactinfo")); tableDescriptor.addFamily(new HColumnDescriptor("creditcard")); admin.createTable(tableDescriptor); }
@Override public HTableDescriptor createHtd(final String tableName) { HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY); hcd.setMobEnabled(true); hcd.setMobThreshold(0L); htd.addFamily(hcd); return htd; }
private HTableDescriptor createTableDesc() { String tableName = "ittable-" + String.format("%010d", RandomUtils.nextInt(Integer.MAX_VALUE)); String familyName = "cf-" + Math.abs(RandomUtils.nextInt()); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); // add random column family htd.addFamily(new HColumnDescriptor(familyName)); return htd; }
@Test public void createTableInDefaultNamespace() throws Exception { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("default_table")); HColumnDescriptor colDesc = new HColumnDescriptor("cf1"); desc.addFamily(colDesc); admin.createTable(desc); assertTrue(admin.listTables().length == 1); admin.disableTable(desc.getTableName()); admin.deleteTable(desc.getTableName()); }
/** Test that we add and remove strings from configuration properly. */ @Test public void testAddGetRemoveConfiguration() throws Exception { HTableDescriptor desc = new HTableDescriptor("table"); String key = "Some"; String value = "value"; desc.setConfiguration(key, value); assertEquals(value, desc.getConfigurationValue(key)); desc.removeConfiguration(key); assertEquals(null, desc.getConfigurationValue(key)); }
@Test public void testReadingHTDFromFS() throws IOException { final String name = "testReadingHTDFromFS"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); HTableDescriptor htd = new HTableDescriptor(name); Path rootdir = UTIL.getDataTestDir(name); createHTDInFS(fs, rootdir, htd); HTableDescriptor htd2 = FSTableDescriptors.getTableDescriptor(fs, rootdir, htd.getNameAsString()); assertTrue(htd.equals(htd2)); }
@Test public void testRemoves() throws IOException { final String name = "testRemoves"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(fs, rootdir); HTableDescriptor htd = new HTableDescriptor(name); htds.add(htd); assertNotNull(htds.remove(htd.getNameAsString())); assertNull(htds.remove(htd.getNameAsString())); }
/** * Test cps in the table description * * @throws Exception */ @Test public void testGetSetRemoveCP() throws Exception { HTableDescriptor desc = new HTableDescriptor("table"); // simple CP String className = BaseRegionObserver.class.getName(); // add and check that it is present desc.addCoprocessor(className); assertTrue(desc.hasCoprocessor(className)); // remove it and check that it is gone desc.removeCoprocessor(className); assertFalse(desc.hasCoprocessor(className)); }
@Test public void createTableInSystemNamespace() throws Exception { TableName tableName = TableName.valueOf("hbase:createTableInSystemNamespace"); HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor colDesc = new HColumnDescriptor("cf1"); desc.addFamily(colDesc); admin.createTable(desc); assertEquals(0, admin.listTables().length); assertTrue(admin.tableExists(tableName)); admin.disableTable(desc.getTableName()); admin.deleteTable(desc.getTableName()); }
@Before public void beforeMethod() throws IOException { for (HTableDescriptor desc : admin.listTables(prefix + ".*")) { admin.disableTable(desc.getTableName()); admin.deleteTable(desc.getTableName()); } for (NamespaceDescriptor ns : admin.listNamespaceDescriptors()) { if (ns.getName().startsWith(prefix)) { admin.deleteNamespace(ns.getName()); } } }
public void createTable(String tableName, List<String> columnFamilies) { try { Admin admin = connection.getAdmin(); HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(tableName)); for (String family : columnFamilies) { descriptor.addFamily(new HColumnDescriptor(family)); } admin.createTable(descriptor); } catch (IOException e) { e.printStackTrace(); } }
@Override void perform() throws IOException { HTableDescriptor selected = selectTable(disabledTables); if (selected == null) { return; } HColumnDescriptor columnDesc = selectFamily(selected); if (columnDesc == null) { return; } Admin admin = connection.getAdmin(); int versions = RandomUtils.nextInt(10) + 3; try { TableName tableName = selected.getTableName(); LOG.info( "Altering versions of column family: " + columnDesc + " to: " + versions + " in table: " + tableName); columnDesc.setMinVersions(versions); columnDesc.setMaxVersions(versions); admin.modifyTable(tableName, selected); // assertion HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName); HColumnDescriptor freshColumnDesc = freshTableDesc.getFamily(columnDesc.getName()); Assert.assertEquals( "Column family: " + columnDesc + " was not altered", freshColumnDesc.getMaxVersions(), versions); Assert.assertEquals( "Column family: " + freshColumnDesc + " was not altered", freshColumnDesc.getMinVersions(), versions); LOG.info( "Altered versions of column family: " + columnDesc + " to: " + versions + " in table: " + tableName); disabledTables.put(tableName, freshTableDesc); } catch (Exception e) { LOG.warn("Caught exception in action: " + this.getClass()); throw e; } finally { admin.close(); } verifyTables(); }
@Test public void testPb() throws DeserializationException, IOException { HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC); final int v = 123; htd.setMaxFileSize(v); htd.setDurability(Durability.ASYNC_WAL); htd.setReadOnly(true); byte[] bytes = htd.toByteArray(); HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(bytes); assertEquals(htd, deserializedHtd); assertEquals(v, deserializedHtd.getMaxFileSize()); assertTrue(deserializedHtd.isReadOnly()); assertEquals(Durability.ASYNC_WAL, deserializedHtd.getDurability()); }
@Test public void testHTableDescriptors() throws IOException, InterruptedException { final String name = "testHTableDescriptors"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any debris laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); final int count = 10; // Write out table infos. for (int i = 0; i < count; i++) { HTableDescriptor htd = new HTableDescriptor(name + i); createHTDInFS(fs, rootdir, htd); } FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) { @Override public HTableDescriptor get(byte[] tablename) throws TableExistsException, FileNotFoundException, IOException { LOG.info(Bytes.toString(tablename) + ", cachehits=" + this.cachehits); return super.get(tablename); } }; for (int i = 0; i < count; i++) { assertTrue(htds.get(Bytes.toBytes(name + i)) != null); } for (int i = 0; i < count; i++) { assertTrue(htds.get(Bytes.toBytes(name + i)) != null); } // Update the table infos for (int i = 0; i < count; i++) { HTableDescriptor htd = new HTableDescriptor(name + i); htd.addFamily(new HColumnDescriptor("" + i)); FSTableDescriptors.updateHTableDescriptor(fs, rootdir, htd); } // Wait a while so mod time we write is for sure different. Thread.sleep(100); for (int i = 0; i < count; i++) { assertTrue(htds.get(Bytes.toBytes(name + i)) != null); } for (int i = 0; i < count; i++) { assertTrue(htds.get(Bytes.toBytes(name + i)) != null); } assertEquals(count * 4, htds.invocations); assertTrue( "expected=" + (count * 2) + ", actual=" + htds.cachehits, htds.cachehits >= (count * 2)); assertTrue(htds.get(HConstants.ROOT_TABLE_NAME) != null); assertEquals(htds.invocations, count * 4 + 1); assertTrue( "expected=" + ((count * 2) + 1) + ", actual=" + htds.cachehits, htds.cachehits >= ((count * 2) + 1)); }
@Test public void verifyReservedNS() throws IOException { // verify existence of reserved namespaces NamespaceDescriptor ns = admin.getNamespaceDescriptor(NamespaceDescriptor.DEFAULT_NAMESPACE.getName()); assertNotNull(ns); assertEquals(ns.getName(), NamespaceDescriptor.DEFAULT_NAMESPACE.getName()); assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR)); ns = admin.getNamespaceDescriptor(NamespaceDescriptor.SYSTEM_NAMESPACE.getName()); assertNotNull(ns); assertEquals(ns.getName(), NamespaceDescriptor.SYSTEM_NAMESPACE.getName()); assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)); assertEquals(2, admin.listNamespaceDescriptors().length); // verify existence of system tables Set<TableName> systemTables = Sets.newHashSet(TableName.META_TABLE_NAME, TableName.NAMESPACE_TABLE_NAME); HTableDescriptor[] descs = admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE.getName()); assertEquals(systemTables.size(), descs.length); for (HTableDescriptor desc : descs) { assertTrue(systemTables.contains(desc.getTableName())); } // verify system tables aren't listed assertEquals(0, admin.listTables().length); // Try creating default and system namespaces. boolean exceptionCaught = false; try { admin.createNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE); } catch (IOException exp) { LOG.warn(exp); exceptionCaught = true; } finally { assertTrue(exceptionCaught); } exceptionCaught = false; try { admin.createNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE); } catch (IOException exp) { LOG.warn(exp); exceptionCaught = true; } finally { assertTrue(exceptionCaught); } }
@Override void perform() throws IOException { HTableDescriptor selected = selectTable(disabledTables); if (selected == null) { return; } HColumnDescriptor columnDesc = selectFamily(selected); if (columnDesc == null) { return; } Admin admin = connection.getAdmin(); try { TableName tableName = selected.getTableName(); // possible DataBlockEncoding ids int[] possibleIds = {0, 2, 3, 4, 6}; short id = (short) possibleIds[RandomUtils.nextInt(possibleIds.length)]; LOG.info( "Altering encoding of column family: " + columnDesc + " to: " + id + " in table: " + tableName); columnDesc.setDataBlockEncoding(DataBlockEncoding.getEncodingById(id)); admin.modifyTable(tableName, selected); // assertion HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName); HColumnDescriptor freshColumnDesc = freshTableDesc.getFamily(columnDesc.getName()); Assert.assertEquals( "Encoding of column family: " + columnDesc + " was not altered", freshColumnDesc.getDataBlockEncoding().getId(), id); LOG.info( "Altered encoding of column family: " + freshColumnDesc + " to: " + id + " in table: " + tableName); disabledTables.put(tableName, freshTableDesc); } catch (Exception e) { LOG.warn("Caught exception in action: " + this.getClass()); throw e; } finally { admin.close(); } verifyTables(); }
private void initHRegion( byte[] tableName, String callingMethod, Configuration conf, byte[]... families) throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); for (byte[] family : families) { htd.addFamily(new HColumnDescriptor(family)); } HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); Path path = new Path(DIR + callingMethod); if (fs.exists(path)) { if (!fs.delete(path, true)) { throw new IOException("Failed delete of " + path); } } region = HRegion.createHRegion(info, path, conf, htd); }
@Test public void testCreateAndUpdate() throws IOException { Path testdir = UTIL.getDataTestDir("testCreateAndUpdate"); HTableDescriptor htd = new HTableDescriptor("testCreate"); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); assertFalse(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); FileStatus[] statuses = fs.listStatus(testdir); assertTrue("statuses.length=" + statuses.length, statuses.length == 1); for (int i = 0; i < 10; i++) { FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); } statuses = fs.listStatus(testdir); assertTrue(statuses.length == 1); Path tmpTableDir = new Path(FSUtils.getTablePath(testdir, htd.getName()), ".tmp"); statuses = fs.listStatus(tmpTableDir); assertTrue(statuses.length == 0); }
// populate tables @Override void perform() throws IOException { HTableDescriptor selected = selectTable(enabledTables); if (selected == null) { return; } Admin admin = connection.getAdmin(); TableName tableName = selected.getTableName(); try (Table table = connection.getTable(tableName)) { ArrayList<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>(admin.getTableRegions(selected.getTableName())); int numRegions = regionInfos.size(); // average number of rows to be added per action to each region int average_rows = 1; int numRows = average_rows * numRegions; LOG.info("Adding " + numRows + " rows to table: " + selected); for (int i = 0; i < numRows; i++) { // nextInt(Integer.MAX_VALUE)) to return positive numbers only byte[] rowKey = Bytes.toBytes( "row-" + String.format("%010d", RandomUtils.nextInt(Integer.MAX_VALUE))); HColumnDescriptor cfd = selectFamily(selected); if (cfd == null) { return; } byte[] family = cfd.getName(); byte[] qualifier = Bytes.toBytes("col-" + RandomUtils.nextInt(Integer.MAX_VALUE) % 10); byte[] value = Bytes.toBytes("val-" + RandomStringUtils.randomAlphanumeric(10)); Put put = new Put(rowKey); put.addColumn(family, qualifier, value); table.put(put); } HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName); enabledTables.put(tableName, freshTableDesc); LOG.info("Added " + numRows + " rows to table: " + selected); } catch (Exception e) { LOG.warn("Caught exception in action: " + this.getClass()); throw e; } finally { admin.close(); } verifyTables(); }
@BeforeClass public static void setUpBeforeClass() throws Exception { conf1 = HBaseConfiguration.create(); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); // smaller block size and capacity to trigger more operations // and test them conf1.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20); conf1.setInt("replication.source.size.capacity", 1024); conf1.setLong("replication.source.sleepforretries", 100); conf1.setInt("hbase.regionserver.maxlogs", 10); conf1.setLong("hbase.master.logcleaner.ttl", 10); conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); conf1.setBoolean("dfs.support.append", true); conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); conf1.setStrings( CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, "org.apache.hadoop.hbase.replication.TestMasterReplication$CoprocessorCounter"); utility1 = new HBaseTestingUtility(conf1); utility1.startMiniZKCluster(); MiniZooKeeperCluster miniZK = utility1.getZkCluster(); new ZooKeeperWatcher(conf1, "cluster1", null, true); conf2 = new Configuration(conf1); conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); conf3 = new Configuration(conf1); conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3"); utility2 = new HBaseTestingUtility(conf2); utility2.setZkCluster(miniZK); new ZooKeeperWatcher(conf2, "cluster3", null, true); utility3 = new HBaseTestingUtility(conf3); utility3.setZkCluster(miniZK); new ZooKeeperWatcher(conf3, "cluster3", null, true); table = new HTableDescriptor(tableName); HColumnDescriptor fam = new HColumnDescriptor(famName); fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); table.addFamily(fam); fam = new HColumnDescriptor(noRepfamName); table.addFamily(fam); }
@Override void perform() throws IOException { HTableDescriptor selected = selectTable(disabledTables); if (selected == null) { return; } Admin admin = connection.getAdmin(); try { TableName tableName = selected.getTableName(); LOG.info("Enabling table :" + selected); admin.enableTable(tableName); Assert.assertTrue( "Table: " + selected + " was not enabled", admin.isTableEnabled(tableName)); HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName); enabledTables.put(tableName, freshTableDesc); LOG.info("Enabled table :" + freshTableDesc); } catch (Exception e) { LOG.warn("Caught exception in action: " + this.getClass()); // TODO workaround // loose restriction for TableNotDisabledException/TableNotEnabledException thrown in sync // operations 1) when enable/disable starts, the table state is changed to // ENABLING/DISABLING (ZK node in 1.x), which will be further changed to ENABLED/DISABLED // once the operation completes 2) if master failover happens in the middle of the // enable/disable operation, the new master will try to recover the tables in // ENABLING/DISABLING state, as programmed in // AssignmentManager#recoverTableInEnablingState() and // AssignmentManager#recoverTableInDisablingState() // 3) after the new master initialization completes, the procedure tries to re-do the // enable/disable operation, which was already done. Ignore those exceptions before // change of behaviors of AssignmentManager in presence of PV2 if (e instanceof TableNotDisabledException) { LOG.warn("Caught TableNotDisabledException in action: " + this.getClass()); e.printStackTrace(); } else { throw e; } } finally { admin.close(); } verifyTables(); }
@Test public void testIllegalHTableNames() { for (String tn : illegalTableNames) { try { HTableDescriptor.isLegalTableName(Bytes.toBytes(tn)); fail("invalid tablename " + tn + " should have failed"); } catch (Exception e) { // expected } } }
// ColumnAction has implemented selectFamily() shared by multiple family Actions protected HColumnDescriptor selectFamily(HTableDescriptor htd) { if (htd == null) { return null; } HColumnDescriptor[] families = htd.getColumnFamilies(); if (families.length == 0) { LOG.info("No column families in table: " + htd); return null; } HColumnDescriptor randomCfd = families[RandomUtils.nextInt(families.length)]; return randomCfd; }
/** Create the Mob Table. */ public static void createMobTable( final HBaseTestingUtility util, final TableName tableName, int regionReplication, final byte[]... families) throws IOException, InterruptedException { HTableDescriptor htd = new HTableDescriptor(tableName); htd.setRegionReplication(regionReplication); for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family); hcd.setMobEnabled(true); hcd.setMobThreshold(0L); htd.addFamily(hcd); } byte[][] splitKeys = SnapshotTestingUtils.getSplitKeys(); util.getHBaseAdmin().createTable(htd, splitKeys); SnapshotTestingUtils.waitForTableToBeOnline(util, tableName); assertEquals( (splitKeys.length + 1) * regionReplication, util.getHBaseAdmin().getTableRegions(tableName).size()); }
@Override void perform() throws IOException { HTableDescriptor selected = selectTable(disabledTables); HColumnDescriptor cfd = selectFamily(selected); if (selected == null || cfd == null) { return; } Admin admin = connection.getAdmin(); try { if (selected.getColumnFamilies().length < 2) { LOG.info("No enough column families to delete in table " + selected.getTableName()); return; } TableName tableName = selected.getTableName(); LOG.info("Deleting column family: " + cfd + " from table: " + tableName); admin.deleteColumnFamily(tableName, cfd.getName()); // assertion HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName); Assert.assertFalse( "Column family: " + cfd + " was not added", freshTableDesc.hasFamily(cfd.getName())); LOG.info("Deleted column family: " + cfd + " from table: " + tableName); disabledTables.put(tableName, freshTableDesc); } catch (Exception e) { LOG.warn("Caught exception in action: " + this.getClass()); throw e; } finally { admin.close(); } verifyTables(); }
@Override void perform() throws IOException { HTableDescriptor selected = selectTable(disabledTables); if (selected == null) { return; } Admin admin = connection.getAdmin(); try { HColumnDescriptor cfd = createFamilyDesc(); if (selected.hasFamily(cfd.getName())) { LOG.info( new String(cfd.getName()) + " already exists in table " + selected.getTableName()); return; } TableName tableName = selected.getTableName(); LOG.info("Adding column family: " + cfd + " to table: " + tableName); admin.addColumn(tableName, cfd); // assertion HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName); Assert.assertTrue( "Column family: " + cfd + " was not added", freshTableDesc.hasFamily(cfd.getName())); LOG.info("Added column family: " + cfd + " to table: " + tableName); disabledTables.put(tableName, freshTableDesc); } catch (Exception e) { LOG.warn("Caught exception in action: " + this.getClass()); throw e; } finally { admin.close(); } verifyTables(); }