public static void waitUntilIndexTableCreated(HMaster master, String tableName) throws IOException, InterruptedException { boolean isEnabled = false; boolean isExist = false; do { isExist = MetaReader.tableExists(master.getCatalogTracker(), TableName.valueOf(tableName)); isEnabled = master.getAssignmentManager().getZKTable().isEnabledTable(TableName.valueOf(tableName)); Thread.sleep(1000); } while ((false == isExist) && (false == isEnabled)); }
/** * This test tests 1, merging region not online; 2, merging same two regions; 3, merging unknown * regions. They are in one test case so that we don't have to create many tables, and these tests * are simple. */ @Test public void testMerge() throws Exception { LOG.info("Starting testMerge"); final TableName tableName = TableName.valueOf("testMerge"); try { // Create table and load data. Table table = createTableAndLoadData(master, tableName); RegionStates regionStates = master.getAssignmentManager().getRegionStates(); List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName); // Fake offline one region HRegionInfo a = regions.get(0); HRegionInfo b = regions.get(1); regionStates.regionOffline(a); try { // Merge offline region. Region a is offline here admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false); fail("Offline regions should not be able to merge"); } catch (IOException ie) { System.out.println(ie); assertTrue( "Exception should mention regions not online", StringUtils.stringifyException(ie).contains("regions not online") && ie instanceof MergeRegionException); } try { // Merge the same region: b and b. admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true); fail("A region should not be able to merge with itself, even forcifully"); } catch (IOException ie) { assertTrue( "Exception should mention regions not online", StringUtils.stringifyException(ie).contains("region to itself") && ie instanceof MergeRegionException); } try { // Merge unknown regions admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true); fail("Unknown region could not be merged"); } catch (IOException ie) { assertTrue("UnknownRegionException should be thrown", ie instanceof UnknownRegionException); } table.close(); } finally { TEST_UTIL.deleteTable(tableName); } }
@Override public ReportRegionStateTransitionResponse reportRegionStateTransition( RpcController c, ReportRegionStateTransitionRequest req) throws ServiceException { ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req); if (enabled.get() && req.getTransition(0).getTransitionCode() == TransitionCode.READY_TO_MERGE && !resp.hasErrorMessage()) { RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates(); for (RegionState regionState : regionStates.getRegionsInTransition().values()) { // Find the merging_new region and remove it if (regionState.isMergingNew()) { regionStates.deleteRegion(regionState.getRegion()); } } } return resp; }
private void waitAndVerifyRegionNum(HMaster master, TableName tablename, int expectedRegionNum) throws Exception { List<Pair<HRegionInfo, ServerName>> tableRegionsInMeta; List<HRegionInfo> tableRegionsInMaster; long timeout = System.currentTimeMillis() + waitTime; while (System.currentTimeMillis() < timeout) { tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename); tableRegionsInMaster = master.getAssignmentManager().getRegionStates().getRegionsOfTable(tablename); if (tableRegionsInMeta.size() == expectedRegionNum && tableRegionsInMaster.size() == expectedRegionNum) { break; } Thread.sleep(250); } tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename); LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta)); assertEquals(expectedRegionNum, tableRegionsInMeta.size()); }
@Test public void testAssignmentManagerDoesntUseDrainingServer() throws Exception { AssignmentManager am; Configuration conf = TEST_UTIL.getConfiguration(); final HMaster master = Mockito.mock(HMaster.class); final Server server = Mockito.mock(Server.class); final ServerManager serverManager = Mockito.mock(ServerManager.class); final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000); final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000); LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf); final HRegionInfo REGIONINFO = new HRegionInfo( TableName.valueOf("table_test"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "zkWatcher-Test", abortable, true); Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>(); onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD); Mockito.when(server.getConfiguration()).thenReturn(conf); Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1")); Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher); CoordinatedStateManager cp = new ZkCoordinatedStateManager(); cp.initialize(server); cp.start(); Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp); Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers); Mockito.when(serverManager.getOnlineServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList(null)) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); for (ServerName sn : onlineServers.keySet()) { Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true); Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true); Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true); Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>())) .thenReturn(RegionOpeningState.OPENED); Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null)) .thenReturn(RegionOpeningState.OPENED); Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true); } Mockito.when(master.getServerManager()).thenReturn(serverManager); am = new AssignmentManager( server, serverManager, balancer, startupMasterExecutor("mockExecutorService"), null, null); Mockito.when(master.getAssignmentManager()).thenReturn(am); Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher); am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A)); zkWatcher.registerListenerFirst(am); addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager); am.assign(REGIONINFO, true); setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO); setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO); am.waitForAssignment(REGIONINFO); assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO)); assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A); }
@Test public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf); AssignmentManager am; final HMaster master = Mockito.mock(HMaster.class); final Server server = Mockito.mock(Server.class); final ServerManager serverManager = Mockito.mock(ServerManager.class); final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000); final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000); final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000); final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000); final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000); final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>(); Set<ServerName> bunchServersAssigned = new HashSet<ServerName>(); HRegionInfo REGIONINFO_A = new HRegionInfo( TableName.valueOf("table_A"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_B = new HRegionInfo( TableName.valueOf("table_B"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_C = new HRegionInfo( TableName.valueOf("table_C"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_D = new HRegionInfo( TableName.valueOf("table_D"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_E = new HRegionInfo( TableName.valueOf("table_E"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>(); List<ServerName> drainedServers = new ArrayList<ServerName>(); onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD); bulk.put(REGIONINFO_A, SERVERNAME_A); bulk.put(REGIONINFO_B, SERVERNAME_B); bulk.put(REGIONINFO_C, SERVERNAME_C); bulk.put(REGIONINFO_D, SERVERNAME_D); bulk.put(REGIONINFO_E, SERVERNAME_E); ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher( TEST_UTIL.getConfiguration(), "zkWatcher-BulkAssignTest", abortable, true); Mockito.when(server.getConfiguration()).thenReturn(conf); Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1")); Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher); CoordinatedStateManager cp = new ZkCoordinatedStateManager(); cp.initialize(server); cp.start(); Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp); Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers); Mockito.when(serverManager.getOnlineServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList(null)) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) { Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true); Mockito.when(serverManager.sendRegionClose(entry.getValue(), entry.getKey(), -1)) .thenReturn(true); Mockito.when(serverManager.sendRegionOpen(entry.getValue(), entry.getKey(), -1, null)) .thenReturn(RegionOpeningState.OPENED); Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true); } Mockito.when(master.getServerManager()).thenReturn(serverManager); drainedServers.add(SERVERNAME_A); drainedServers.add(SERVERNAME_B); drainedServers.add(SERVERNAME_C); drainedServers.add(SERVERNAME_D); am = new AssignmentManager( server, serverManager, balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null); Mockito.when(master.getAssignmentManager()).thenReturn(am); zkWatcher.registerListener(am); for (ServerName drained : drainedServers) { addServerToDrainedList(drained, onlineServers, serverManager); } am.assign(bulk); Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition(); for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) { setRegionOpenedOnZK( zkWatcher, entry.getValue().getServerName(), entry.getValue().getRegion()); } am.waitForAssignment(REGIONINFO_A); am.waitForAssignment(REGIONINFO_B); am.waitForAssignment(REGIONINFO_C); am.waitForAssignment(REGIONINFO_D); am.waitForAssignment(REGIONINFO_E); Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments(); for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) { LOG.info( "Region Assignment: " + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue()); bunchServersAssigned.add(entry.getValue()); } for (ServerName sn : drainedServers) { assertFalse(bunchServersAssigned.contains(sn)); } }
/** * Test that draining servers are ignored even after killing regionserver(s). Verify that the * draining server is not given any of the dead servers regions. * * @throws KeeperException * @throws IOException */ @Test(timeout = 30000) public void testDrainingServerWithAbort() throws KeeperException, Exception { HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); waitForAllRegionsOnline(); final long regionCount = TEST_UTIL.getMiniHBaseCluster().countServedRegions(); // Let's get a copy of the regions today. Collection<HRegion> regions = new ArrayList<HRegion>(); for (int i = 0; i < NB_SLAVES; i++) { HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i); regions.addAll(hrs.getCopyOfOnlineRegionsSortedBySize().values()); } // Choose the draining server HRegionServer drainingServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); final int regionsOnDrainingServer = drainingServer.getNumberOfOnlineRegions(); Assert.assertTrue(regionsOnDrainingServer > 0); ServerManager sm = master.getServerManager(); Collection<HRegion> regionsBefore = drainingServer.getCopyOfOnlineRegionsSortedBySize().values(); LOG.info("Regions of drained server are: " + regionsBefore); try { // Add first server to draining servers up in zk. setDrainingServer(drainingServer); // wait for the master to receive and manage the event while (sm.createDestinationServersList().contains(drainingServer.getServerName())) { Thread.sleep(1); } LOG.info("The available servers are: " + sm.createDestinationServersList()); Assert.assertEquals( "Nothing should have happened here.", regionsOnDrainingServer, drainingServer.getNumberOfOnlineRegions()); Assert.assertFalse( "We should not have regions in transition here. List is: " + master.getAssignmentManager().getRegionStates().getRegionsInTransition(), master.getAssignmentManager().getRegionStates().isRegionsInTransition()); // Kill a few regionservers. for (int aborted = 0; aborted <= 2; aborted++) { HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(aborted + 1); hrs.abort("Aborting"); } // Wait for regions to come back online again. waitForAllRegionsOnline(); Collection<HRegion> regionsAfter = drainingServer.getCopyOfOnlineRegionsSortedBySize().values(); LOG.info("Regions of drained server are: " + regionsAfter); Assert.assertEquals( "Test conditions are not met: regions were" + " created/deleted during the test. ", regionCount, TEST_UTIL.getMiniHBaseCluster().countServedRegions()); // Assert the draining server still has the same regions. StringBuilder result = new StringBuilder(); for (HRegion r : regionsAfter) { if (!regionsBefore.contains(r)) { result.append(r).append(" was added after the drain"); if (regions.contains(r)) { result.append("(existing region"); } else { result.append("(new region)"); } result.append("; "); } } for (HRegion r : regionsBefore) { if (!regionsAfter.contains(r)) { result.append(r).append(" was removed after the drain; "); } } Assert.assertTrue("Errors are: " + result.toString(), result.length() == 0); } finally { unsetDrainingServer(drainingServer); } }