@Test public void testAssignmentManagerDoesntUseDrainingServer() throws Exception { AssignmentManager am; Configuration conf = TEST_UTIL.getConfiguration(); final HMaster master = Mockito.mock(HMaster.class); final Server server = Mockito.mock(Server.class); final ServerManager serverManager = Mockito.mock(ServerManager.class); final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000); final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000); LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf); final HRegionInfo REGIONINFO = new HRegionInfo( TableName.valueOf("table_test"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "zkWatcher-Test", abortable, true); Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>(); onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD); Mockito.when(server.getConfiguration()).thenReturn(conf); Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1")); Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher); CoordinatedStateManager cp = new ZkCoordinatedStateManager(); cp.initialize(server); cp.start(); Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp); Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers); Mockito.when(serverManager.getOnlineServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList(null)) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); for (ServerName sn : onlineServers.keySet()) { Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true); Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true); Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true); Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>())) .thenReturn(RegionOpeningState.OPENED); Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null)) .thenReturn(RegionOpeningState.OPENED); Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true); } Mockito.when(master.getServerManager()).thenReturn(serverManager); am = new AssignmentManager( server, serverManager, balancer, startupMasterExecutor("mockExecutorService"), null, null); Mockito.when(master.getAssignmentManager()).thenReturn(am); Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher); am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A)); zkWatcher.registerListenerFirst(am); addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager); am.assign(REGIONINFO, true); setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO); setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO); am.waitForAssignment(REGIONINFO); assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO)); assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A); }
/** * Test that draining servers are ignored even after killing regionserver(s). Verify that the * draining server is not given any of the dead servers regions. * * @throws KeeperException * @throws IOException */ @Test(timeout = 30000) public void testDrainingServerWithAbort() throws KeeperException, Exception { HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); waitForAllRegionsOnline(); final long regionCount = TEST_UTIL.getMiniHBaseCluster().countServedRegions(); // Let's get a copy of the regions today. Collection<HRegion> regions = new ArrayList<HRegion>(); for (int i = 0; i < NB_SLAVES; i++) { HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i); regions.addAll(hrs.getCopyOfOnlineRegionsSortedBySize().values()); } // Choose the draining server HRegionServer drainingServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); final int regionsOnDrainingServer = drainingServer.getNumberOfOnlineRegions(); Assert.assertTrue(regionsOnDrainingServer > 0); ServerManager sm = master.getServerManager(); Collection<HRegion> regionsBefore = drainingServer.getCopyOfOnlineRegionsSortedBySize().values(); LOG.info("Regions of drained server are: " + regionsBefore); try { // Add first server to draining servers up in zk. setDrainingServer(drainingServer); // wait for the master to receive and manage the event while (sm.createDestinationServersList().contains(drainingServer.getServerName())) { Thread.sleep(1); } LOG.info("The available servers are: " + sm.createDestinationServersList()); Assert.assertEquals( "Nothing should have happened here.", regionsOnDrainingServer, drainingServer.getNumberOfOnlineRegions()); Assert.assertFalse( "We should not have regions in transition here. List is: " + master.getAssignmentManager().getRegionStates().getRegionsInTransition(), master.getAssignmentManager().getRegionStates().isRegionsInTransition()); // Kill a few regionservers. for (int aborted = 0; aborted <= 2; aborted++) { HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(aborted + 1); hrs.abort("Aborting"); } // Wait for regions to come back online again. waitForAllRegionsOnline(); Collection<HRegion> regionsAfter = drainingServer.getCopyOfOnlineRegionsSortedBySize().values(); LOG.info("Regions of drained server are: " + regionsAfter); Assert.assertEquals( "Test conditions are not met: regions were" + " created/deleted during the test. ", regionCount, TEST_UTIL.getMiniHBaseCluster().countServedRegions()); // Assert the draining server still has the same regions. StringBuilder result = new StringBuilder(); for (HRegion r : regionsAfter) { if (!regionsBefore.contains(r)) { result.append(r).append(" was added after the drain"); if (regions.contains(r)) { result.append("(existing region"); } else { result.append("(new region)"); } result.append("; "); } } for (HRegion r : regionsBefore) { if (!regionsAfter.contains(r)) { result.append(r).append(" was removed after the drain; "); } } Assert.assertTrue("Errors are: " + result.toString(), result.length() == 0); } finally { unsetDrainingServer(drainingServer); } }
@Test public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf); AssignmentManager am; final HMaster master = Mockito.mock(HMaster.class); final Server server = Mockito.mock(Server.class); final ServerManager serverManager = Mockito.mock(ServerManager.class); final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000); final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000); final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000); final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000); final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000); final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>(); Set<ServerName> bunchServersAssigned = new HashSet<ServerName>(); HRegionInfo REGIONINFO_A = new HRegionInfo( TableName.valueOf("table_A"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_B = new HRegionInfo( TableName.valueOf("table_B"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_C = new HRegionInfo( TableName.valueOf("table_C"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_D = new HRegionInfo( TableName.valueOf("table_D"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_E = new HRegionInfo( TableName.valueOf("table_E"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>(); List<ServerName> drainedServers = new ArrayList<ServerName>(); onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD); bulk.put(REGIONINFO_A, SERVERNAME_A); bulk.put(REGIONINFO_B, SERVERNAME_B); bulk.put(REGIONINFO_C, SERVERNAME_C); bulk.put(REGIONINFO_D, SERVERNAME_D); bulk.put(REGIONINFO_E, SERVERNAME_E); ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher( TEST_UTIL.getConfiguration(), "zkWatcher-BulkAssignTest", abortable, true); Mockito.when(server.getConfiguration()).thenReturn(conf); Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1")); Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher); CoordinatedStateManager cp = new ZkCoordinatedStateManager(); cp.initialize(server); cp.start(); Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp); Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers); Mockito.when(serverManager.getOnlineServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList(null)) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) { Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true); Mockito.when(serverManager.sendRegionClose(entry.getValue(), entry.getKey(), -1)) .thenReturn(true); Mockito.when(serverManager.sendRegionOpen(entry.getValue(), entry.getKey(), -1, null)) .thenReturn(RegionOpeningState.OPENED); Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true); } Mockito.when(master.getServerManager()).thenReturn(serverManager); drainedServers.add(SERVERNAME_A); drainedServers.add(SERVERNAME_B); drainedServers.add(SERVERNAME_C); drainedServers.add(SERVERNAME_D); am = new AssignmentManager( server, serverManager, balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null); Mockito.when(master.getAssignmentManager()).thenReturn(am); zkWatcher.registerListener(am); for (ServerName drained : drainedServers) { addServerToDrainedList(drained, onlineServers, serverManager); } am.assign(bulk); Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition(); for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) { setRegionOpenedOnZK( zkWatcher, entry.getValue().getServerName(), entry.getValue().getRegion()); } am.waitForAssignment(REGIONINFO_A); am.waitForAssignment(REGIONINFO_B); am.waitForAssignment(REGIONINFO_C); am.waitForAssignment(REGIONINFO_D); am.waitForAssignment(REGIONINFO_E); Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments(); for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) { LOG.info( "Region Assignment: " + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue()); bunchServersAssigned.add(entry.getValue()); } for (ServerName sn : drainedServers) { assertFalse(bunchServersAssigned.contains(sn)); } }
@Test(timeout = 300000) public void testDataCorrectnessReplayingRecoveredEdits() throws Exception { final int NUM_MASTERS = 1; final int NUM_RS = 3; TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); try { final byte[] TABLENAME = Bytes.toBytes("testDataCorrectnessReplayingRecoveredEdits"); final byte[] FAMILY = Bytes.toBytes("family"); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HMaster master = cluster.getMaster(); // Create table HTableDescriptor desc = new HTableDescriptor(TABLENAME); desc.addFamily(new HColumnDescriptor(FAMILY)); HBaseAdmin hbaseAdmin = TEST_UTIL.getHBaseAdmin(); hbaseAdmin.createTable(desc); assertTrue(hbaseAdmin.isTableAvailable(TABLENAME)); // Put data: r1->v1 Log.info("Loading r1 to v1 into " + Bytes.toString(TABLENAME)); HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); putDataAndVerify(table, "r1", FAMILY, "v1", 1); // Move region to target server HRegionInfo regionInfo = table.getRegionLocation("r1").getRegionInfo(); int originServerNum = cluster.getServerWith(regionInfo.getRegionName()); HRegionServer originServer = cluster.getRegionServer(originServerNum); int targetServerNum = (originServerNum + 1) % NUM_RS; HRegionServer targetServer = cluster.getRegionServer(targetServerNum); assertFalse(originServer.equals(targetServer)); Log.info("Moving " + regionInfo.getEncodedName() + " to " + targetServer.getServerName()); hbaseAdmin.move( regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(targetServer.getServerName().getServerName())); do { Thread.sleep(1); } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum); // Put data: r2->v2 Log.info("Loading r2 to v2 into " + Bytes.toString(TABLENAME)); putDataAndVerify(table, "r2", FAMILY, "v2", 2); // Move region to origin server Log.info("Moving " + regionInfo.getEncodedName() + " to " + originServer.getServerName()); hbaseAdmin.move( regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(originServer.getServerName().getServerName())); do { Thread.sleep(1); } while (cluster.getServerWith(regionInfo.getRegionName()) == targetServerNum); // Put data: r3->v3 Log.info("Loading r3 to v3 into " + Bytes.toString(TABLENAME)); putDataAndVerify(table, "r3", FAMILY, "v3", 3); // Kill target server Log.info("Killing target server " + targetServer.getServerName()); targetServer.kill(); cluster.getRegionServerThreads().get(targetServerNum).join(); // Wait until finish processing of shutdown while (master.getServerManager().areDeadServersInProgress()) { Thread.sleep(5); } // Kill origin server Log.info("Killing origin server " + targetServer.getServerName()); originServer.kill(); cluster.getRegionServerThreads().get(originServerNum).join(); // Put data: r4->v4 Log.info("Loading r4 to v4 into " + Bytes.toString(TABLENAME)); putDataAndVerify(table, "r4", FAMILY, "v4", 4); } finally { TEST_UTIL.shutdownMiniCluster(); } }