@Test(timeout = 300000) public void testCreateTableNumberOfRegions() throws IOException, InterruptedException { TableName tableName = TableName.valueOf("testCreateTableNumberOfRegions"); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc); HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); Map<HRegionInfo, ServerName> regions = ht.getRegionLocations(); assertEquals("Table should have only 1 region", 1, regions.size()); ht.close(); TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2"); desc = new HTableDescriptor(TABLE_2); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, new byte[][] {new byte[] {42}}); HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2); regions = ht2.getRegionLocations(); assertEquals("Table should have only 2 region", 2, regions.size()); ht2.close(); TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3"); desc = new HTableDescriptor(TABLE_3); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, "a".getBytes(), "z".getBytes(), 3); HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3); regions = ht3.getRegionLocations(); assertEquals("Table should have only 3 region", 3, regions.size()); ht3.close(); TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4"); desc = new HTableDescriptor(TABLE_4); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); try { admin.createTable(desc, "a".getBytes(), "z".getBytes(), 2); fail("Should not be able to create a table with only 2 regions using this API."); } catch (IllegalArgumentException eae) { // Expected } TableName TABLE_5 = TableName.valueOf(tableName.getNameAsString() + "_5"); desc = new HTableDescriptor(TABLE_5); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, new byte[] {1}, new byte[] {127}, 16); HTable ht5 = new HTable(TEST_UTIL.getConfiguration(), TABLE_5); regions = ht5.getRegionLocations(); assertEquals("Table should have 16 region", 16, regions.size()); ht5.close(); }
/** * Test retain assignment on enableTable. * * @throws IOException */ @Test(timeout = 300000) public void testEnableTableRetainAssignment() throws IOException { final TableName tableName = TableName.valueOf("testEnableTableAssignment"); byte[][] splitKeys = { new byte[] {1, 1, 1}, new byte[] {2, 2, 2}, new byte[] {3, 3, 3}, new byte[] {4, 4, 4}, new byte[] {5, 5, 5}, new byte[] {6, 6, 6}, new byte[] {7, 7, 7}, new byte[] {8, 8, 8}, new byte[] {9, 9, 9} }; int expectedRegions = splitKeys.length + 1; HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, splitKeys); HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); Map<HRegionInfo, ServerName> regions = ht.getRegionLocations(); assertEquals( "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size()); // Disable table. admin.disableTable(tableName); // Enable table, use retain assignment to assign regions. admin.enableTable(tableName); Map<HRegionInfo, ServerName> regions2 = ht.getRegionLocations(); // Check the assignment. assertEquals(regions.size(), regions2.size()); for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { assertEquals(regions2.get(entry.getKey()), entry.getValue()); } }
@SuppressWarnings("deprecation") protected void verifyRoundRobinDistribution(HTable ht, int expectedRegions) throws IOException { int numRS = ht.getConnection().getCurrentNrHRS(); Map<HRegionInfo, ServerName> regions = ht.getRegionLocations(); Map<ServerName, List<HRegionInfo>> server2Regions = new HashMap<ServerName, List<HRegionInfo>>(); for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { ServerName server = entry.getValue(); List<HRegionInfo> regs = server2Regions.get(server); if (regs == null) { regs = new ArrayList<HRegionInfo>(); server2Regions.put(server, regs); } regs.add(entry.getKey()); } float average = (float) expectedRegions / numRS; int min = (int) Math.floor(average); int max = (int) Math.ceil(average); for (List<HRegionInfo> regionList : server2Regions.values()) { assertTrue(regionList.size() == min || regionList.size() == max); } }
void splitTest( byte[] splitPoint, byte[][] familyNames, int[] rowCounts, int numVersions, int blockSize) throws Exception { TableName tableName = TableName.valueOf("testForceSplit"); StringBuilder sb = new StringBuilder(); // Add tail to String so can see better in logs where a test is running. for (int i = 0; i < rowCounts.length; i++) { sb.append("_").append(Integer.toString(rowCounts[i])); } assertFalse(admin.tableExists(tableName)); final HTable table = TEST_UTIL.createTable(tableName, familyNames, numVersions, blockSize); int rowCount = 0; byte[] q = new byte[0]; // insert rows into column families. The number of rows that have values // in a specific column family is decided by rowCounts[familyIndex] for (int index = 0; index < familyNames.length; index++) { ArrayList<Put> puts = new ArrayList<Put>(rowCounts[index]); for (int i = 0; i < rowCounts[index]; i++) { byte[] k = Bytes.toBytes(i); Put put = new Put(k); put.add(familyNames[index], q, k); puts.add(put); } table.put(puts); if (rowCount < rowCounts[index]) { rowCount = rowCounts[index]; } } // get the initial layout (should just be one region) Map<HRegionInfo, ServerName> m = table.getRegionLocations(); LOG.info("Initial regions (" + m.size() + "): " + m); assertTrue(m.size() == 1); // Verify row count Scan scan = new Scan(); ResultScanner scanner = table.getScanner(scan); int rows = 0; for (@SuppressWarnings("unused") Result result : scanner) { rows++; } scanner.close(); assertEquals(rowCount, rows); // Have an outstanding scan going on to make sure we can scan over splits. scan = new Scan(); scanner = table.getScanner(scan); // Scan first row so we are into first region before split happens. scanner.next(); // Split the table this.admin.split(tableName, splitPoint); final AtomicInteger count = new AtomicInteger(0); Thread t = new Thread("CheckForSplit") { @Override public void run() { for (int i = 0; i < 45; i++) { try { sleep(1000); } catch (InterruptedException e) { continue; } // check again table = new HTable(conf, tableName); Map<HRegionInfo, ServerName> regions = null; try { regions = table.getRegionLocations(); } catch (IOException e) { e.printStackTrace(); } if (regions == null) continue; count.set(regions.size()); if (count.get() >= 2) { LOG.info("Found: " + regions); break; } LOG.debug("Cycle waiting on split"); } LOG.debug("CheckForSplit thread exited, current region count: " + count.get()); } }; t.setPriority(Thread.NORM_PRIORITY - 2); t.start(); t.join(); // Verify row count rows = 1; // We counted one row above. for (@SuppressWarnings("unused") Result result : scanner) { rows++; if (rows > rowCount) { scanner.close(); assertTrue("Scanned more than expected (" + rowCount + ")", false); } } scanner.close(); assertEquals(rowCount, rows); Map<HRegionInfo, ServerName> regions = null; try { regions = table.getRegionLocations(); } catch (IOException e) { e.printStackTrace(); } assertEquals(2, regions.size()); Set<HRegionInfo> hRegionInfos = regions.keySet(); HRegionInfo[] r = hRegionInfos.toArray(new HRegionInfo[hRegionInfos.size()]); if (splitPoint != null) { // make sure the split point matches our explicit configuration assertEquals(Bytes.toString(splitPoint), Bytes.toString(r[0].getEndKey())); assertEquals(Bytes.toString(splitPoint), Bytes.toString(r[1].getStartKey())); LOG.debug("Properly split on " + Bytes.toString(splitPoint)); } else { if (familyNames.length > 1) { int splitKey = Bytes.toInt(r[0].getEndKey()); // check if splitKey is based on the largest column family // in terms of it store size int deltaForLargestFamily = Math.abs(rowCount / 2 - splitKey); LOG.debug( "SplitKey=" + splitKey + "&deltaForLargestFamily=" + deltaForLargestFamily + ", r=" + r[0]); for (int index = 0; index < familyNames.length; index++) { int delta = Math.abs(rowCounts[index] / 2 - splitKey); if (delta < deltaForLargestFamily) { assertTrue( "Delta " + delta + " for family " + index + " should be at least deltaForLargestFamily " + deltaForLargestFamily, false); } } } } TEST_UTIL.deleteTable(tableName); table.close(); }
@Test(timeout = 300000) public void testCreateTableWithRegions() throws IOException, InterruptedException { TableName tableName = TableName.valueOf("testCreateTableWithRegions"); byte[][] splitKeys = { new byte[] {1, 1, 1}, new byte[] {2, 2, 2}, new byte[] {3, 3, 3}, new byte[] {4, 4, 4}, new byte[] {5, 5, 5}, new byte[] {6, 6, 6}, new byte[] {7, 7, 7}, new byte[] {8, 8, 8}, new byte[] {9, 9, 9}, }; int expectedRegions = splitKeys.length + 1; HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, splitKeys); boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys); assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable); HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); Map<HRegionInfo, ServerName> regions = ht.getRegionLocations(); assertEquals( "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size()); System.err.println("Found " + regions.size() + " regions"); Iterator<HRegionInfo> hris = regions.keySet().iterator(); HRegionInfo hri = hris.next(); assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0); assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0])); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0])); assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1])); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1])); assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2])); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2])); assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3])); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3])); assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4])); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4])); assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5])); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5])); assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6])); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6])); assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7])); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7])); assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8])); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8])); assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); verifyRoundRobinDistribution(ht, expectedRegions); ht.close(); // Now test using start/end with a number of regions // Use 80 bit numbers to make sure we aren't limited byte[] startKey = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; byte[] endKey = {9, 9, 9, 9, 9, 9, 9, 9, 9, 9}; // Splitting into 10 regions, we expect (null,1) ... (9, null) // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle expectedRegions = 10; TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2"); desc = new HTableDescriptor(TABLE_2); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); admin.createTable(desc, startKey, endKey, expectedRegions); HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2); regions = ht2.getRegionLocations(); assertEquals( "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size()); System.err.println("Found " + regions.size() + " regions"); hris = regions.keySet().iterator(); hri = hris.next(); assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0); assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1})); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1})); assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {2, 2, 2, 2, 2, 2, 2, 2, 2, 2})); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {2, 2, 2, 2, 2, 2, 2, 2, 2, 2})); assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {3, 3, 3, 3, 3, 3, 3, 3, 3, 3})); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {3, 3, 3, 3, 3, 3, 3, 3, 3, 3})); assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {4, 4, 4, 4, 4, 4, 4, 4, 4, 4})); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {4, 4, 4, 4, 4, 4, 4, 4, 4, 4})); assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5, 5})); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5, 5})); assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {6, 6, 6, 6, 6, 6, 6, 6, 6, 6})); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {6, 6, 6, 6, 6, 6, 6, 6, 6, 6})); assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7, 7})); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7, 7})); assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {8, 8, 8, 8, 8, 8, 8, 8, 8, 8})); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {8, 8, 8, 8, 8, 8, 8, 8, 8, 8})); assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {9, 9, 9, 9, 9, 9, 9, 9, 9, 9})); hri = hris.next(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {9, 9, 9, 9, 9, 9, 9, 9, 9, 9})); assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); verifyRoundRobinDistribution(ht2, expectedRegions); ht2.close(); // Try once more with something that divides into something infinite startKey = new byte[] {0, 0, 0, 0, 0, 0}; endKey = new byte[] {1, 0, 0, 0, 0, 0}; expectedRegions = 5; TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3"); desc = new HTableDescriptor(TABLE_3); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); admin.createTable(desc, startKey, endKey, expectedRegions); HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3); regions = ht3.getRegionLocations(); assertEquals( "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size()); System.err.println("Found " + regions.size() + " regions"); verifyRoundRobinDistribution(ht3, expectedRegions); ht3.close(); // Try an invalid case where there are duplicate split keys splitKeys = new byte[][] { new byte[] {1, 1, 1}, new byte[] {2, 2, 2}, new byte[] {3, 3, 3}, new byte[] {2, 2, 2} }; TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4"); desc = new HTableDescriptor(TABLE_4); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); Admin ladmin = new HBaseAdmin(TEST_UTIL.getConfiguration()); try { ladmin.createTable(desc, splitKeys); assertTrue( "Should not be able to create this table because of " + "duplicate split keys", false); } catch (IllegalArgumentException iae) { // Expected } ladmin.close(); }