@Test(timeout = 300000)
public void testCreateTableNumberOfRegions() throws IOException, InterruptedException {
TableName tableName = TableName.valueOf("testCreateTableNumberOfRegions");
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc);
HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
assertEquals("Table should have only 1 region", 1, regions.size());
ht.close();
TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2");
desc = new HTableDescriptor(TABLE_2);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc, new byte[][] {new byte[] {42}});
HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2);
regions = ht2.getRegionLocations();
assertEquals("Table should have only 2 region", 2, regions.size());
ht2.close();
TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3");
desc = new HTableDescriptor(TABLE_3);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc, "a".getBytes(), "z".getBytes(), 3);
HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3);
regions = ht3.getRegionLocations();
assertEquals("Table should have only 3 region", 3, regions.size());
ht3.close();
TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4");
desc = new HTableDescriptor(TABLE_4);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
try {
admin.createTable(desc, "a".getBytes(), "z".getBytes(), 2);
fail("Should not be able to create a table with only 2 regions using this API.");
} catch (IllegalArgumentException eae) {
// Expected
}
TableName TABLE_5 = TableName.valueOf(tableName.getNameAsString() + "_5");
desc = new HTableDescriptor(TABLE_5);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc, new byte[] {1}, new byte[] {127}, 16);
HTable ht5 = new HTable(TEST_UTIL.getConfiguration(), TABLE_5);
regions = ht5.getRegionLocations();
assertEquals("Table should have 16 region", 16, regions.size());
ht5.close();
}
/**
* Test retain assignment on enableTable.
*
* @throws IOException
*/
@Test(timeout = 300000)
public void testEnableTableRetainAssignment() throws IOException {
final TableName tableName = TableName.valueOf("testEnableTableAssignment");
byte[][] splitKeys = {
new byte[] {1, 1, 1},
new byte[] {2, 2, 2},
new byte[] {3, 3, 3},
new byte[] {4, 4, 4},
new byte[] {5, 5, 5},
new byte[] {6, 6, 6},
new byte[] {7, 7, 7},
new byte[] {8, 8, 8},
new byte[] {9, 9, 9}
};
int expectedRegions = splitKeys.length + 1;
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc, splitKeys);
HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
assertEquals(
"Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
expectedRegions,
regions.size());
// Disable table.
admin.disableTable(tableName);
// Enable table, use retain assignment to assign regions.
admin.enableTable(tableName);
Map<HRegionInfo, ServerName> regions2 = ht.getRegionLocations();
// Check the assignment.
assertEquals(regions.size(), regions2.size());
for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
assertEquals(regions2.get(entry.getKey()), entry.getValue());
}
}
@SuppressWarnings("deprecation")
protected void verifyRoundRobinDistribution(HTable ht, int expectedRegions) throws IOException {
int numRS = ht.getConnection().getCurrentNrHRS();
Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
Map<ServerName, List<HRegionInfo>> server2Regions =
new HashMap<ServerName, List<HRegionInfo>>();
for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
ServerName server = entry.getValue();
List<HRegionInfo> regs = server2Regions.get(server);
if (regs == null) {
regs = new ArrayList<HRegionInfo>();
server2Regions.put(server, regs);
}
regs.add(entry.getKey());
}
float average = (float) expectedRegions / numRS;
int min = (int) Math.floor(average);
int max = (int) Math.ceil(average);
for (List<HRegionInfo> regionList : server2Regions.values()) {
assertTrue(regionList.size() == min || regionList.size() == max);
}
}
void splitTest(
byte[] splitPoint, byte[][] familyNames, int[] rowCounts, int numVersions, int blockSize)
throws Exception {
TableName tableName = TableName.valueOf("testForceSplit");
StringBuilder sb = new StringBuilder();
// Add tail to String so can see better in logs where a test is running.
for (int i = 0; i < rowCounts.length; i++) {
sb.append("_").append(Integer.toString(rowCounts[i]));
}
assertFalse(admin.tableExists(tableName));
final HTable table = TEST_UTIL.createTable(tableName, familyNames, numVersions, blockSize);
int rowCount = 0;
byte[] q = new byte[0];
// insert rows into column families. The number of rows that have values
// in a specific column family is decided by rowCounts[familyIndex]
for (int index = 0; index < familyNames.length; index++) {
ArrayList<Put> puts = new ArrayList<Put>(rowCounts[index]);
for (int i = 0; i < rowCounts[index]; i++) {
byte[] k = Bytes.toBytes(i);
Put put = new Put(k);
put.add(familyNames[index], q, k);
puts.add(put);
}
table.put(puts);
if (rowCount < rowCounts[index]) {
rowCount = rowCounts[index];
}
}
// get the initial layout (should just be one region)
Map<HRegionInfo, ServerName> m = table.getRegionLocations();
LOG.info("Initial regions (" + m.size() + "): " + m);
assertTrue(m.size() == 1);
// Verify row count
Scan scan = new Scan();
ResultScanner scanner = table.getScanner(scan);
int rows = 0;
for (@SuppressWarnings("unused") Result result : scanner) {
rows++;
}
scanner.close();
assertEquals(rowCount, rows);
// Have an outstanding scan going on to make sure we can scan over splits.
scan = new Scan();
scanner = table.getScanner(scan);
// Scan first row so we are into first region before split happens.
scanner.next();
// Split the table
this.admin.split(tableName, splitPoint);
final AtomicInteger count = new AtomicInteger(0);
Thread t =
new Thread("CheckForSplit") {
@Override
public void run() {
for (int i = 0; i < 45; i++) {
try {
sleep(1000);
} catch (InterruptedException e) {
continue;
}
// check again table = new HTable(conf, tableName);
Map<HRegionInfo, ServerName> regions = null;
try {
regions = table.getRegionLocations();
} catch (IOException e) {
e.printStackTrace();
}
if (regions == null) continue;
count.set(regions.size());
if (count.get() >= 2) {
LOG.info("Found: " + regions);
break;
}
LOG.debug("Cycle waiting on split");
}
LOG.debug("CheckForSplit thread exited, current region count: " + count.get());
}
};
t.setPriority(Thread.NORM_PRIORITY - 2);
t.start();
t.join();
// Verify row count
rows = 1; // We counted one row above.
for (@SuppressWarnings("unused") Result result : scanner) {
rows++;
if (rows > rowCount) {
scanner.close();
assertTrue("Scanned more than expected (" + rowCount + ")", false);
}
}
scanner.close();
assertEquals(rowCount, rows);
Map<HRegionInfo, ServerName> regions = null;
try {
regions = table.getRegionLocations();
} catch (IOException e) {
e.printStackTrace();
}
assertEquals(2, regions.size());
Set<HRegionInfo> hRegionInfos = regions.keySet();
HRegionInfo[] r = hRegionInfos.toArray(new HRegionInfo[hRegionInfos.size()]);
if (splitPoint != null) {
// make sure the split point matches our explicit configuration
assertEquals(Bytes.toString(splitPoint), Bytes.toString(r[0].getEndKey()));
assertEquals(Bytes.toString(splitPoint), Bytes.toString(r[1].getStartKey()));
LOG.debug("Properly split on " + Bytes.toString(splitPoint));
} else {
if (familyNames.length > 1) {
int splitKey = Bytes.toInt(r[0].getEndKey());
// check if splitKey is based on the largest column family
// in terms of it store size
int deltaForLargestFamily = Math.abs(rowCount / 2 - splitKey);
LOG.debug(
"SplitKey="
+ splitKey
+ "&deltaForLargestFamily="
+ deltaForLargestFamily
+ ", r="
+ r[0]);
for (int index = 0; index < familyNames.length; index++) {
int delta = Math.abs(rowCounts[index] / 2 - splitKey);
if (delta < deltaForLargestFamily) {
assertTrue(
"Delta "
+ delta
+ " for family "
+ index
+ " should be at least deltaForLargestFamily "
+ deltaForLargestFamily,
false);
}
}
}
}
TEST_UTIL.deleteTable(tableName);
table.close();
}
@Test(timeout = 300000)
public void testCreateTableWithRegions() throws IOException, InterruptedException {
TableName tableName = TableName.valueOf("testCreateTableWithRegions");
byte[][] splitKeys = {
new byte[] {1, 1, 1},
new byte[] {2, 2, 2},
new byte[] {3, 3, 3},
new byte[] {4, 4, 4},
new byte[] {5, 5, 5},
new byte[] {6, 6, 6},
new byte[] {7, 7, 7},
new byte[] {8, 8, 8},
new byte[] {9, 9, 9},
};
int expectedRegions = splitKeys.length + 1;
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc, splitKeys);
boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys);
assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable);
HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
assertEquals(
"Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
expectedRegions,
regions.size());
System.err.println("Found " + regions.size() + " regions");
Iterator<HRegionInfo> hris = regions.keySet().iterator();
HRegionInfo hri = hris.next();
assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
verifyRoundRobinDistribution(ht, expectedRegions);
ht.close();
// Now test using start/end with a number of regions
// Use 80 bit numbers to make sure we aren't limited
byte[] startKey = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
byte[] endKey = {9, 9, 9, 9, 9, 9, 9, 9, 9, 9};
// Splitting into 10 regions, we expect (null,1) ... (9, null)
// with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle
expectedRegions = 10;
TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2");
desc = new HTableDescriptor(TABLE_2);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
admin.createTable(desc, startKey, endKey, expectedRegions);
HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2);
regions = ht2.getRegionLocations();
assertEquals(
"Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
expectedRegions,
regions.size());
System.err.println("Found " + regions.size() + " regions");
hris = regions.keySet().iterator();
hri = hris.next();
assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {2, 2, 2, 2, 2, 2, 2, 2, 2, 2}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {2, 2, 2, 2, 2, 2, 2, 2, 2, 2}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {3, 3, 3, 3, 3, 3, 3, 3, 3, 3}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {3, 3, 3, 3, 3, 3, 3, 3, 3, 3}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {4, 4, 4, 4, 4, 4, 4, 4, 4, 4}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {4, 4, 4, 4, 4, 4, 4, 4, 4, 4}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5, 5}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5, 5}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {6, 6, 6, 6, 6, 6, 6, 6, 6, 6}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {6, 6, 6, 6, 6, 6, 6, 6, 6, 6}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7, 7}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7, 7}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {8, 8, 8, 8, 8, 8, 8, 8, 8, 8}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {8, 8, 8, 8, 8, 8, 8, 8, 8, 8}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte[] {9, 9, 9, 9, 9, 9, 9, 9, 9, 9}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] {9, 9, 9, 9, 9, 9, 9, 9, 9, 9}));
assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
verifyRoundRobinDistribution(ht2, expectedRegions);
ht2.close();
// Try once more with something that divides into something infinite
startKey = new byte[] {0, 0, 0, 0, 0, 0};
endKey = new byte[] {1, 0, 0, 0, 0, 0};
expectedRegions = 5;
TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3");
desc = new HTableDescriptor(TABLE_3);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
admin.createTable(desc, startKey, endKey, expectedRegions);
HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3);
regions = ht3.getRegionLocations();
assertEquals(
"Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
expectedRegions,
regions.size());
System.err.println("Found " + regions.size() + " regions");
verifyRoundRobinDistribution(ht3, expectedRegions);
ht3.close();
// Try an invalid case where there are duplicate split keys
splitKeys =
new byte[][] {
new byte[] {1, 1, 1},
new byte[] {2, 2, 2},
new byte[] {3, 3, 3},
new byte[] {2, 2, 2}
};
TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4");
desc = new HTableDescriptor(TABLE_4);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
Admin ladmin = new HBaseAdmin(TEST_UTIL.getConfiguration());
try {
ladmin.createTable(desc, splitKeys);
assertTrue(
"Should not be able to create this table because of " + "duplicate split keys", false);
} catch (IllegalArgumentException iae) {
// Expected
}
ladmin.close();
}
