@Override
public void postMove(
ObserverContext<MasterCoprocessorEnvironment> ctx,
HRegionInfo region,
ServerName srcServer,
ServerName destServer)
throws IOException {
if (balancer != null && balancer.isTableColocated(region.getTable())) {
AssignmentManager am = ctx.getEnvironment().getMasterServices().getAssignmentManager();
RegionStates regionStates = am.getRegionStates();
String tableName = region.getTable().getNameAsString();
String correspondingTable =
region.getTable().getNameAsString().startsWith(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX)
? MetaDataUtil.getUserTableName(tableName)
: MetaDataUtil.getLocalIndexTableName(tableName);
List<HRegionInfo> regions =
regionStates.getRegionsOfTable(TableName.valueOf(correspondingTable));
for (HRegionInfo hri : regions) {
if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) == 0 && destServer != null) {
balancer.regionOnline(hri, destServer);
am.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, destServer));
am.unassign(hri);
}
}
}
super.postMove(ctx, region, srcServer, destServer);
}
protected void waitRegionInTransition(final List<HRegionInfo> regions)
throws IOException, CoordinatedStateException {
AssignmentManager am = this.masterServices.getAssignmentManager();
RegionStates states = am.getRegionStates();
long waitTime = server.getConfiguration().getLong("hbase.master.wait.on.region", 5 * 60 * 1000);
for (HRegionInfo region : regions) {
long done = System.currentTimeMillis() + waitTime;
while (System.currentTimeMillis() < done) {
if (states.isRegionInState(region, State.FAILED_OPEN)) {
am.regionOffline(region);
}
if (!states.isRegionInTransition(region)) break;
try {
Thread.sleep(waitingTimeForEvents);
} catch (InterruptedException e) {
LOG.warn("Interrupted while sleeping");
throw (InterruptedIOException) new InterruptedIOException().initCause(e);
}
LOG.debug(
"Waiting on region to clear regions in transition; "
+ am.getRegionStates().getRegionTransitionState(region));
}
if (states.isRegionInTransition(region)) {
throw new IOException(
"Waited hbase.master.wait.on.region ("
+ waitTime
+ "ms) for region to leave region "
+ region.getRegionNameAsString()
+ " in transitions");
}
}
}
/**
* This test tests 1, merging region not online; 2, merging same two regions; 3, merging unknown
* regions. They are in one test case so that we don't have to create many tables, and these tests
* are simple.
*/
@Test
public void testMerge() throws Exception {
LOG.info("Starting testMerge");
final TableName tableName = TableName.valueOf("testMerge");
try {
// Create table and load data.
Table table = createTableAndLoadData(master, tableName);
RegionStates regionStates = master.getAssignmentManager().getRegionStates();
List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName);
// Fake offline one region
HRegionInfo a = regions.get(0);
HRegionInfo b = regions.get(1);
regionStates.regionOffline(a);
try {
// Merge offline region. Region a is offline here
admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
fail("Offline regions should not be able to merge");
} catch (IOException ie) {
System.out.println(ie);
assertTrue(
"Exception should mention regions not online",
StringUtils.stringifyException(ie).contains("regions not online")
&& ie instanceof MergeRegionException);
}
try {
// Merge the same region: b and b.
admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true);
fail("A region should not be able to merge with itself, even forcifully");
} catch (IOException ie) {
assertTrue(
"Exception should mention regions not online",
StringUtils.stringifyException(ie).contains("region to itself")
&& ie instanceof MergeRegionException);
}
try {
// Merge unknown regions
admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
fail("Unknown region could not be merged");
} catch (IOException ie) {
assertTrue("UnknownRegionException should be thrown", ie instanceof UnknownRegionException);
}
table.close();
} finally {
TEST_UTIL.deleteTable(tableName);
}
}
@Override
public ReportRegionStateTransitionResponse reportRegionStateTransition(
RpcController c, ReportRegionStateTransitionRequest req) throws ServiceException {
ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
if (enabled.get()
&& req.getTransition(0).getTransitionCode() == TransitionCode.READY_TO_MERGE
&& !resp.hasErrorMessage()) {
RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
for (RegionState regionState : regionStates.getRegionsInTransition().values()) {
// Find the merging_new region and remove it
if (regionState.isMergingNew()) {
regionStates.deleteRegion(regionState.getRegion());
}
}
}
return resp;
}
/**
* Unassign the list of regions. Configuration knobs: hbase.bulk.waitbetween.reopen indicates the
* number of milliseconds to wait before unassigning another region from this region server
*
* @param regions
* @throws InterruptedException
*/
private void unassign(List<HRegionInfo> regions) throws InterruptedException {
int waitTime = this.server.getConfiguration().getInt("hbase.bulk.waitbetween.reopen", 0);
RegionStates regionStates = assignmentManager.getRegionStates();
for (HRegionInfo region : regions) {
if (server.isStopped()) {
return;
}
if (regionStates.isRegionInTransition(region)) {
continue;
}
assignmentManager.unassign(region);
while (regionStates.isRegionInTransition(region) && !server.isStopped()) {
regionStates.waitForUpdate(100);
}
if (waitTime > 0 && !server.isStopped()) {
Thread.sleep(waitTime);
}
}
}
@Test
@SuppressWarnings("deprecation")
public void testMasterOpsWhileSplitting() throws Exception {
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
HMaster m = cluster.getMaster();
try (Table ht = TEST_UTIL.createTable(TABLENAME, FAMILYNAME)) {
assertTrue(m.getTableStateManager().isTableState(TABLENAME, TableState.State.ENABLED));
TEST_UTIL.loadTable(ht, FAMILYNAME, false);
}
List<Pair<HRegionInfo, ServerName>> tableRegions =
MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME);
LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
assertEquals(1, tableRegions.size());
assertArrayEquals(HConstants.EMPTY_START_ROW, tableRegions.get(0).getFirst().getStartKey());
assertArrayEquals(HConstants.EMPTY_END_ROW, tableRegions.get(0).getFirst().getEndKey());
// Now trigger a split and stop when the split is in progress
LOG.info("Splitting table");
TEST_UTIL.getAdmin().split(TABLENAME);
LOG.info("Waiting for split result to be about to open");
RegionStates regionStates = m.getAssignmentManager().getRegionStates();
while (regionStates.getRegionsOfTable(TABLENAME).size() <= 1) {
Thread.sleep(100);
}
LOG.info("Making sure we can call getTableRegions while opening");
tableRegions =
MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME, false);
LOG.info("Regions: " + Joiner.on(',').join(tableRegions));
// We have three regions because one is split-in-progress
assertEquals(3, tableRegions.size());
LOG.info("Making sure we can call getTableRegionClosest while opening");
Pair<HRegionInfo, ServerName> pair = m.getTableRegionForRow(TABLENAME, Bytes.toBytes("cde"));
LOG.info("Result is: " + pair);
Pair<HRegionInfo, ServerName> tableRegionFromName =
MetaTableAccessor.getRegion(m.getConnection(), pair.getFirst().getRegionName());
assertEquals(tableRegionFromName.getFirst(), pair.getFirst());
}
@Test
public void testWholesomeMerge() throws Exception {
LOG.info("Starting testWholesomeMerge");
final TableName tableName = TableName.valueOf("testWholesomeMerge");
// Create table and load data.
Table table = createTableAndLoadData(master, tableName);
// Merge 1st and 2nd region
mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1, INITIAL_REGION_NUM - 1);
// Merge 2nd and 3th region
PairOfSameType<HRegionInfo> mergedRegions =
mergeRegionsAndVerifyRegionNum(master, tableName, 1, 2, INITIAL_REGION_NUM - 2);
verifyRowCount(table, ROWSIZE);
// Randomly choose one of the two merged regions
HRegionInfo hri =
RandomUtils.nextBoolean() ? mergedRegions.getFirst() : mergedRegions.getSecond();
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
AssignmentManager am = cluster.getMaster().getAssignmentManager();
RegionStates regionStates = am.getRegionStates();
long start = EnvironmentEdgeManager.currentTime();
while (!regionStates.isRegionInState(hri, State.MERGED)) {
assertFalse(
"Timed out in waiting one merged region to be in state MERGED",
EnvironmentEdgeManager.currentTime() - start > 60000);
Thread.sleep(500);
}
// We should not be able to assign it again
am.assign(hri, true);
assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri));
assertTrue(regionStates.isRegionInState(hri, State.MERGED));
// We should not be able to unassign it either
am.unassign(hri, null);
assertFalse("Merged region can't be unassigned", regionStates.isRegionInTransition(hri));
assertTrue(regionStates.isRegionInState(hri, State.MERGED));
table.close();
}
/** This tests retaining assignments on a cluster restart */
@Test(timeout = 300000)
public void testRetainAssignmentOnRestart() throws Exception {
UTIL.startMiniCluster(2);
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
Threads.sleep(1);
}
// Turn off balancer
UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().synchronousBalanceSwitch(false);
LOG.info("\n\nCreating tables");
for (byte[] TABLE : TABLES) {
UTIL.createTable(TABLE, FAMILY);
}
for (byte[] TABLE : TABLES) {
UTIL.waitTableEnabled(TABLE);
}
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
UTIL.waitUntilNoRegionsInTransition(120000);
// We don't have to use SnapshotOfRegionAssignmentFromMeta.
// We use it here because AM used to use it to load all user region placements
SnapshotOfRegionAssignmentFromMeta snapshot =
new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection());
snapshot.initialize();
Map<HRegionInfo, ServerName> regionToRegionServerMap = snapshot.getRegionToRegionServerMap();
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads();
assertEquals(2, threads.size());
int[] rsPorts = new int[3];
for (int i = 0; i < 2; i++) {
rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort();
}
rsPorts[2] = cluster.getMaster().getServerName().getPort();
for (ServerName serverName : regionToRegionServerMap.values()) {
boolean found = false; // Test only, no need to optimize
for (int k = 0; k < 3 && !found; k++) {
found = serverName.getPort() == rsPorts[k];
}
assertTrue(found);
}
LOG.info("\n\nShutting down HBase cluster");
cluster.shutdown();
cluster.waitUntilShutDown();
LOG.info("\n\nSleeping a bit");
Thread.sleep(2000);
LOG.info("\n\nStarting cluster the second time with the same ports");
try {
cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 4);
master = cluster.startMaster().getMaster();
for (int i = 0; i < 3; i++) {
cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]);
cluster.startRegionServer();
}
} finally {
// Reset region server port so as not to conflict with other tests
cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0);
cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2);
}
// Make sure live regionservers are on the same host/port
List<ServerName> localServers = master.getServerManager().getOnlineServersList();
assertEquals(4, localServers.size());
for (int i = 0; i < 3; i++) {
boolean found = false;
for (ServerName serverName : localServers) {
if (serverName.getPort() == rsPorts[i]) {
found = true;
break;
}
}
assertTrue(found);
}
// Wait till master is initialized and all regions are assigned
RegionStates regionStates = master.getAssignmentManager().getRegionStates();
int expectedRegions = regionToRegionServerMap.size() + 1;
while (!master.isInitialized()
|| regionStates.getRegionAssignments().size() != expectedRegions) {
Threads.sleep(100);
}
snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection());
snapshot.initialize();
Map<HRegionInfo, ServerName> newRegionToRegionServerMap = snapshot.getRegionToRegionServerMap();
assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size());
for (Map.Entry<HRegionInfo, ServerName> entry : newRegionToRegionServerMap.entrySet()) {
if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue;
ServerName oldServer = regionToRegionServerMap.get(entry.getKey());
ServerName currentServer = entry.getValue();
assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort());
assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode());
}
}