/**
* Tests an on-the-fly RPC that was scheduled for the earlier RS on the same port for openRegion.
* The region server should reject this RPC. (HBASE-9721)
*/
@Test
public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception {
Assert.assertTrue(getRS().getRegion(regionName).isAvailable());
ServerName sn = getRS().getServerName();
ServerName earlierServerName = ServerName.valueOf(sn.getHostname(), sn.getPort(), 1);
try {
CloseRegionRequest request =
RequestConverter.buildCloseRegionRequest(earlierServerName, regionName);
getRS().getRSRpcServices().closeRegion(null, request);
Assert.fail("The closeRegion should have been rejected");
} catch (ServiceException se) {
Assert.assertTrue(se.getCause() instanceof IOException);
Assert.assertTrue(
se.getCause().getMessage().contains("This RPC was intended for a different server"));
}
// actual close
closeRegionNoZK();
try {
AdminProtos.OpenRegionRequest orr =
RequestConverter.buildOpenRegionRequest(earlierServerName, hri, null, null);
getRS().getRSRpcServices().openRegion(null, orr);
Assert.fail("The openRegion should have been rejected");
} catch (ServiceException se) {
Assert.assertTrue(se.getCause() instanceof IOException);
Assert.assertTrue(
se.getCause().getMessage().contains("This RPC was intended for a different server"));
} finally {
openRegion(HTU, getRS(), hri);
}
}
/** Test that if we do a close while opening it stops the opening. */
@Test(timeout = 60000)
public void testCancelOpeningWithoutZK() throws Exception {
// We close
closeRegionNoZK();
checkRegionIsClosed(HTU, getRS(), hri);
// Let do the initial steps, without having a handler
getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE);
// That's a close without ZK.
AdminProtos.CloseRegionRequest crr =
RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName);
try {
getRS().rpcServices.closeRegion(null, crr);
Assert.assertTrue(false);
} catch (ServiceException expected) {
}
// The state in RIT should have changed to close
Assert.assertEquals(
Boolean.FALSE, getRS().getRegionsInTransitionInRS().get(hri.getEncodedNameAsBytes()));
// Let's start the open handler
HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable());
getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd));
// The open handler should have removed the region from RIT but kept the region closed
checkRegionIsClosed(HTU, getRS(), hri);
openRegion(HTU, getRS(), hri);
}
public static void closeRegion(HBaseTestingUtility HTU, HRegionServer rs, HRegionInfo hri)
throws Exception {
AdminProtos.CloseRegionRequest crr =
RequestConverter.buildCloseRegionRequest(rs.getServerName(), hri.getEncodedName());
AdminProtos.CloseRegionResponse responseClose = rs.rpcServices.closeRegion(null, crr);
Assert.assertTrue(responseClose.getClosed());
checkRegionIsClosed(HTU, rs, hri);
}
/** Close the region without using ZK */
private void closeRegionNoZK() throws Exception {
// no transition in ZK
AdminProtos.CloseRegionRequest crr =
RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName);
AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
Assert.assertTrue(responseClose.getClosed());
// now waiting & checking. After a while, the transition should be done and the region closed
checkRegionIsClosed(HTU, getRS(), hri);
}
public static void openRegion(HBaseTestingUtility HTU, HRegionServer rs, HRegionInfo hri)
throws Exception {
AdminProtos.OpenRegionRequest orr =
RequestConverter.buildOpenRegionRequest(rs.getServerName(), hri, null, null);
AdminProtos.OpenRegionResponse responseOpen = rs.rpcServices.openRegion(null, orr);
Assert.assertTrue(responseOpen.getOpeningStateCount() == 1);
Assert.assertTrue(
responseOpen
.getOpeningState(0)
.equals(AdminProtos.OpenRegionResponse.RegionOpeningState.OPENED));
checkRegionIsOpened(HTU, rs, hri);
}
/**
* Test adding server to draining servers and then move regions off it. Make sure that no regions
* are moved back to the draining server.
*
* @throws IOException
* @throws KeeperException
*/
@Test // (timeout=30000)
public void testDrainingServerOffloading() throws Exception {
// I need master in the below.
HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
HRegionInfo hriToMoveBack = null;
// Set first server as draining server.
HRegionServer drainingServer =
setDrainingServer(TEST_UTIL.getMiniHBaseCluster().getRegionServer(0));
try {
final int regionsOnDrainingServer = drainingServer.getNumberOfOnlineRegions();
Assert.assertTrue(regionsOnDrainingServer > 0);
List<HRegionInfo> hris = ProtobufUtil.getOnlineRegions(drainingServer);
for (HRegionInfo hri : hris) {
// Pass null and AssignmentManager will chose a random server BUT it
// should exclude draining servers.
master.moveRegion(
null, RequestConverter.buildMoveRegionRequest(hri.getEncodedNameAsBytes(), null));
// Save off region to move back.
hriToMoveBack = hri;
}
// Wait for regions to come back on line again.
waitForAllRegionsOnline();
Assert.assertEquals(0, drainingServer.getNumberOfOnlineRegions());
} finally {
unsetDrainingServer(drainingServer);
}
// Now we've unset the draining server, we should be able to move a region
// to what was the draining server.
master.moveRegion(
null,
RequestConverter.buildMoveRegionRequest(
hriToMoveBack.getEncodedNameAsBytes(),
Bytes.toBytes(drainingServer.getServerName().toString())));
// Wait for regions to come back on line again.
waitForAllRegionsOnline();
Assert.assertEquals(1, drainingServer.getNumberOfOnlineRegions());
}
@Test(timeout = 60000)
public void testMultipleCloseFromMaster() throws Exception {
for (int i = 0; i < 10; i++) {
AdminProtos.CloseRegionRequest crr =
RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, null);
try {
AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
Assert.assertTrue(
"request " + i + " failed", responseClose.getClosed() || responseClose.hasClosed());
} catch (ServiceException se) {
Assert.assertTrue("The next queries may throw an exception.", i > 0);
}
}
checkRegionIsClosed(HTU, getRS(), hri);
openRegion(HTU, getRS(), hri);
}
@Override
public Result call(int callTimeout) throws Exception {
if (controller.isCanceled()) return null;
if (Thread.interrupted()) {
throw new InterruptedIOException();
}
byte[] reg = location.getRegionInfo().getRegionName();
ClientProtos.GetRequest request = RequestConverter.buildGetRequest(reg, get);
controller.setCallTimeout(callTimeout);
try {
ClientProtos.GetResponse response = getStub().get(controller, request);
if (response == null) {
return null;
}
return ProtobufUtil.toResult(response.getResult());
} catch (ServiceException se) {
throw ProtobufUtil.getRemoteException(se);
}
}
/**
* Update the assignment plan to all the region servers
*
* @param plan
* @throws IOException
*/
private void updateAssignmentPlanToRegionServers(FavoredNodesPlan plan) throws IOException {
LOG.info("Start to update the region servers with the new assignment plan");
// Get the region to region server map
Map<ServerName, List<HRegionInfo>> currentAssignment =
this.getRegionAssignmentSnapshot().getRegionServerToRegionMap();
// track of the failed and succeeded updates
int succeededNum = 0;
Map<ServerName, Exception> failedUpdateMap = new HashMap<ServerName, Exception>();
for (Map.Entry<ServerName, List<HRegionInfo>> entry : currentAssignment.entrySet()) {
List<Pair<HRegionInfo, List<ServerName>>> regionUpdateInfos =
new ArrayList<Pair<HRegionInfo, List<ServerName>>>();
try {
// Keep track of the favored updates for the current region server
FavoredNodesPlan singleServerPlan = null;
// Find out all the updates for the current region server
for (HRegionInfo region : entry.getValue()) {
List<ServerName> favoredServerList = plan.getFavoredNodes(region);
if (favoredServerList != null
&& favoredServerList.size() == FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
// Create the single server plan if necessary
if (singleServerPlan == null) {
singleServerPlan = new FavoredNodesPlan();
}
// Update the single server update
singleServerPlan.updateAssignmentPlan(region, favoredServerList);
regionUpdateInfos.add(
new Pair<HRegionInfo, List<ServerName>>(region, favoredServerList));
}
}
if (singleServerPlan != null) {
// Update the current region server with its updated favored nodes
BlockingInterface currentRegionServer =
((ClusterConnection) this.connection).getAdmin(entry.getKey());
UpdateFavoredNodesRequest request =
RequestConverter.buildUpdateFavoredNodesRequest(regionUpdateInfos);
UpdateFavoredNodesResponse updateFavoredNodesResponse =
currentRegionServer.updateFavoredNodes(null, request);
LOG.info(
"Region server "
+ ProtobufUtil.getServerInfo(null, currentRegionServer).getServerName()
+ " has updated "
+ updateFavoredNodesResponse.getResponse()
+ " / "
+ singleServerPlan.getAssignmentMap().size()
+ " regions with the assignment plan");
succeededNum++;
}
} catch (Exception e) {
failedUpdateMap.put(entry.getKey(), e);
}
}
// log the succeeded updates
LOG.info("Updated " + succeededNum + " region servers with " + "the new assignment plan");
// log the failed updates
int failedNum = failedUpdateMap.size();
if (failedNum != 0) {
LOG.error(
"Failed to update the following + "
+ failedNum
+ " region servers with its corresponding favored nodes");
for (Map.Entry<ServerName, Exception> entry : failedUpdateMap.entrySet()) {
LOG.error(
"Failed to update "
+ entry.getKey().getHostAndPort()
+ " because of "
+ entry.getValue().getMessage());
}
}
}