public static void waitUntilIndexTableCreated(HMaster master, String tableName)
     throws IOException, InterruptedException {
   boolean isEnabled = false;
   boolean isExist = false;
   do {
     isExist = MetaReader.tableExists(master.getCatalogTracker(), TableName.valueOf(tableName));
     isEnabled =
         master.getAssignmentManager().getZKTable().isEnabledTable(TableName.valueOf(tableName));
     Thread.sleep(1000);
   } while ((false == isExist) && (false == isEnabled));
 }
  /**
   * This test tests 1, merging region not online; 2, merging same two regions; 3, merging unknown
   * regions. They are in one test case so that we don't have to create many tables, and these tests
   * are simple.
   */
  @Test
  public void testMerge() throws Exception {
    LOG.info("Starting testMerge");
    final TableName tableName = TableName.valueOf("testMerge");

    try {
      // Create table and load data.
      Table table = createTableAndLoadData(master, tableName);
      RegionStates regionStates = master.getAssignmentManager().getRegionStates();
      List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName);
      // Fake offline one region
      HRegionInfo a = regions.get(0);
      HRegionInfo b = regions.get(1);
      regionStates.regionOffline(a);
      try {
        // Merge offline region. Region a is offline here
        admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
        fail("Offline regions should not be able to merge");
      } catch (IOException ie) {
        System.out.println(ie);
        assertTrue(
            "Exception should mention regions not online",
            StringUtils.stringifyException(ie).contains("regions not online")
                && ie instanceof MergeRegionException);
      }
      try {
        // Merge the same region: b and b.
        admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true);
        fail("A region should not be able to merge with itself, even forcifully");
      } catch (IOException ie) {
        assertTrue(
            "Exception should mention regions not online",
            StringUtils.stringifyException(ie).contains("region to itself")
                && ie instanceof MergeRegionException);
      }
      try {
        // Merge unknown regions
        admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
        fail("Unknown region could not be merged");
      } catch (IOException ie) {
        assertTrue("UnknownRegionException should be thrown", ie instanceof UnknownRegionException);
      }
      table.close();
    } finally {
      TEST_UTIL.deleteTable(tableName);
    }
  }
 @Override
 public ReportRegionStateTransitionResponse reportRegionStateTransition(
     RpcController c, ReportRegionStateTransitionRequest req) throws ServiceException {
   ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
   if (enabled.get()
       && req.getTransition(0).getTransitionCode() == TransitionCode.READY_TO_MERGE
       && !resp.hasErrorMessage()) {
     RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
     for (RegionState regionState : regionStates.getRegionsInTransition().values()) {
       // Find the merging_new region and remove it
       if (regionState.isMergingNew()) {
         regionStates.deleteRegion(regionState.getRegion());
       }
     }
   }
   return resp;
 }
  private void waitAndVerifyRegionNum(HMaster master, TableName tablename, int expectedRegionNum)
      throws Exception {
    List<Pair<HRegionInfo, ServerName>> tableRegionsInMeta;
    List<HRegionInfo> tableRegionsInMaster;
    long timeout = System.currentTimeMillis() + waitTime;
    while (System.currentTimeMillis() < timeout) {
      tableRegionsInMeta =
          MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
      tableRegionsInMaster =
          master.getAssignmentManager().getRegionStates().getRegionsOfTable(tablename);
      if (tableRegionsInMeta.size() == expectedRegionNum
          && tableRegionsInMaster.size() == expectedRegionNum) {
        break;
      }
      Thread.sleep(250);
    }

    tableRegionsInMeta =
        MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
    LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
    assertEquals(expectedRegionNum, tableRegionsInMeta.size());
  }
  @Test
  public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
    AssignmentManager am;
    Configuration conf = TEST_UTIL.getConfiguration();
    final HMaster master = Mockito.mock(HMaster.class);
    final Server server = Mockito.mock(Server.class);
    final ServerManager serverManager = Mockito.mock(ServerManager.class);
    final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
    final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
    final HRegionInfo REGIONINFO =
        new HRegionInfo(
            TableName.valueOf("table_test"),
            HConstants.EMPTY_START_ROW,
            HConstants.EMPTY_START_ROW);

    ZooKeeperWatcher zkWatcher =
        new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "zkWatcher-Test", abortable, true);

    Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();

    onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);

    Mockito.when(server.getConfiguration()).thenReturn(conf);
    Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
    Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);

    CoordinatedStateManager cp = new ZkCoordinatedStateManager();
    cp.initialize(server);
    cp.start();

    Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);

    Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
    Mockito.when(serverManager.getOnlineServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    Mockito.when(serverManager.createDestinationServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
    Mockito.when(serverManager.createDestinationServersList(null))
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    for (ServerName sn : onlineServers.keySet()) {
      Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
      Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
      Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
      Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
          .thenReturn(RegionOpeningState.OPENED);
      Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
          .thenReturn(RegionOpeningState.OPENED);
      Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
    }

    Mockito.when(master.getServerManager()).thenReturn(serverManager);

    am =
        new AssignmentManager(
            server,
            serverManager,
            balancer,
            startupMasterExecutor("mockExecutorService"),
            null,
            null);

    Mockito.when(master.getAssignmentManager()).thenReturn(am);
    Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);

    am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));

    zkWatcher.registerListenerFirst(am);

    addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);

    am.assign(REGIONINFO, true);

    setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
    setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);

    am.waitForAssignment(REGIONINFO);

    assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
    assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
  }
  @Test
  public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
    AssignmentManager am;
    final HMaster master = Mockito.mock(HMaster.class);
    final Server server = Mockito.mock(Server.class);
    final ServerManager serverManager = Mockito.mock(ServerManager.class);
    final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
    final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
    final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
    final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
    final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
    final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();

    Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();

    HRegionInfo REGIONINFO_A =
        new HRegionInfo(
            TableName.valueOf("table_A"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_B =
        new HRegionInfo(
            TableName.valueOf("table_B"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_C =
        new HRegionInfo(
            TableName.valueOf("table_C"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_D =
        new HRegionInfo(
            TableName.valueOf("table_D"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_E =
        new HRegionInfo(
            TableName.valueOf("table_E"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);

    Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
    List<ServerName> drainedServers = new ArrayList<ServerName>();

    onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);

    bulk.put(REGIONINFO_A, SERVERNAME_A);
    bulk.put(REGIONINFO_B, SERVERNAME_B);
    bulk.put(REGIONINFO_C, SERVERNAME_C);
    bulk.put(REGIONINFO_D, SERVERNAME_D);
    bulk.put(REGIONINFO_E, SERVERNAME_E);

    ZooKeeperWatcher zkWatcher =
        new ZooKeeperWatcher(
            TEST_UTIL.getConfiguration(), "zkWatcher-BulkAssignTest", abortable, true);

    Mockito.when(server.getConfiguration()).thenReturn(conf);
    Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
    Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);

    CoordinatedStateManager cp = new ZkCoordinatedStateManager();
    cp.initialize(server);
    cp.start();

    Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);

    Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
    Mockito.when(serverManager.getOnlineServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    Mockito.when(serverManager.createDestinationServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
    Mockito.when(serverManager.createDestinationServersList(null))
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
      Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
      Mockito.when(serverManager.sendRegionClose(entry.getValue(), entry.getKey(), -1))
          .thenReturn(true);
      Mockito.when(serverManager.sendRegionOpen(entry.getValue(), entry.getKey(), -1, null))
          .thenReturn(RegionOpeningState.OPENED);
      Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
    }

    Mockito.when(master.getServerManager()).thenReturn(serverManager);

    drainedServers.add(SERVERNAME_A);
    drainedServers.add(SERVERNAME_B);
    drainedServers.add(SERVERNAME_C);
    drainedServers.add(SERVERNAME_D);

    am =
        new AssignmentManager(
            server,
            serverManager,
            balancer,
            startupMasterExecutor("mockExecutorServiceBulk"),
            null,
            null);

    Mockito.when(master.getAssignmentManager()).thenReturn(am);

    zkWatcher.registerListener(am);

    for (ServerName drained : drainedServers) {
      addServerToDrainedList(drained, onlineServers, serverManager);
    }

    am.assign(bulk);

    Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
    for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
      setRegionOpenedOnZK(
          zkWatcher, entry.getValue().getServerName(), entry.getValue().getRegion());
    }

    am.waitForAssignment(REGIONINFO_A);
    am.waitForAssignment(REGIONINFO_B);
    am.waitForAssignment(REGIONINFO_C);
    am.waitForAssignment(REGIONINFO_D);
    am.waitForAssignment(REGIONINFO_E);

    Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
    for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
      LOG.info(
          "Region Assignment: "
              + entry.getKey().getRegionNameAsString()
              + " Server: "
              + entry.getValue());
      bunchServersAssigned.add(entry.getValue());
    }

    for (ServerName sn : drainedServers) {
      assertFalse(bunchServersAssigned.contains(sn));
    }
  }
  /**
   * Test that draining servers are ignored even after killing regionserver(s). Verify that the
   * draining server is not given any of the dead servers regions.
   *
   * @throws KeeperException
   * @throws IOException
   */
  @Test(timeout = 30000)
  public void testDrainingServerWithAbort() throws KeeperException, Exception {
    HMaster master = TEST_UTIL.getHBaseCluster().getMaster();

    waitForAllRegionsOnline();

    final long regionCount = TEST_UTIL.getMiniHBaseCluster().countServedRegions();

    // Let's get a copy of the regions today.
    Collection<HRegion> regions = new ArrayList<HRegion>();
    for (int i = 0; i < NB_SLAVES; i++) {
      HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i);
      regions.addAll(hrs.getCopyOfOnlineRegionsSortedBySize().values());
    }

    // Choose the draining server
    HRegionServer drainingServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
    final int regionsOnDrainingServer = drainingServer.getNumberOfOnlineRegions();
    Assert.assertTrue(regionsOnDrainingServer > 0);

    ServerManager sm = master.getServerManager();

    Collection<HRegion> regionsBefore =
        drainingServer.getCopyOfOnlineRegionsSortedBySize().values();
    LOG.info("Regions of drained server are: " + regionsBefore);

    try {
      // Add first server to draining servers up in zk.
      setDrainingServer(drainingServer);

      // wait for the master to receive and manage the event
      while (sm.createDestinationServersList().contains(drainingServer.getServerName())) {
        Thread.sleep(1);
      }

      LOG.info("The available servers are: " + sm.createDestinationServersList());

      Assert.assertEquals(
          "Nothing should have happened here.",
          regionsOnDrainingServer,
          drainingServer.getNumberOfOnlineRegions());
      Assert.assertFalse(
          "We should not have regions in transition here. List is: "
              + master.getAssignmentManager().getRegionStates().getRegionsInTransition(),
          master.getAssignmentManager().getRegionStates().isRegionsInTransition());

      // Kill a few regionservers.
      for (int aborted = 0; aborted <= 2; aborted++) {
        HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(aborted + 1);
        hrs.abort("Aborting");
      }

      // Wait for regions to come back online again.
      waitForAllRegionsOnline();

      Collection<HRegion> regionsAfter =
          drainingServer.getCopyOfOnlineRegionsSortedBySize().values();
      LOG.info("Regions of drained server are: " + regionsAfter);

      Assert.assertEquals(
          "Test conditions are not met: regions were" + " created/deleted during the test. ",
          regionCount,
          TEST_UTIL.getMiniHBaseCluster().countServedRegions());

      // Assert the draining server still has the same regions.
      StringBuilder result = new StringBuilder();
      for (HRegion r : regionsAfter) {
        if (!regionsBefore.contains(r)) {
          result.append(r).append(" was added after the drain");
          if (regions.contains(r)) {
            result.append("(existing region");
          } else {
            result.append("(new region)");
          }
          result.append("; ");
        }
      }
      for (HRegion r : regionsBefore) {
        if (!regionsAfter.contains(r)) {
          result.append(r).append(" was removed after the drain; ");
        }
      }
      Assert.assertTrue("Errors are: " + result.toString(), result.length() == 0);

    } finally {
      unsetDrainingServer(drainingServer);
    }
  }