private Table createTableAndLoadData(
      HMaster master, TableName tablename, int numRegions, int replication) throws Exception {
    assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
    byte[][] splitRows = new byte[numRegions - 1][];
    for (int i = 0; i < splitRows.length; i++) {
      splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
    }

    Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
    if (replication > 1) {
      HBaseTestingUtility.setReplicas(admin, tablename, replication);
    }
    loadData(table);
    verifyRowCount(table, ROWSIZE);

    // sleep here is an ugly hack to allow region transitions to finish
    long timeout = System.currentTimeMillis() + waitTime;
    List<Pair<HRegionInfo, ServerName>> tableRegions;
    while (System.currentTimeMillis() < timeout) {
      tableRegions =
          MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
      if (tableRegions.size() == numRegions * replication) break;
      Thread.sleep(250);
    }

    tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
    LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
    assertEquals(numRegions * replication, tableRegions.size());
    return table;
  }
 @Test
 public void testMetaUpdatedFlagInROOT() throws Exception {
   HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
   boolean metaUpdated =
       MetaMigrationConvertingToPB.isMetaTableUpdated(master.getCatalogTracker());
   assertEquals(true, metaUpdated);
   verifyMetaRowsAreUpdated(master.getCatalogTracker());
 }
Ejemplo n.º 3
0
 public HBaseTcpServer(HMaster master) {
   ZooKeeperAdmin.createBaseZNodes();
   initConf(master.getConfiguration());
   tcpPort = getMasterTcpPort(master.getConfiguration());
   serverName = master.getServerName();
   this.master = master;
   init(master.getConfiguration());
 }
  public void renderNoFlush(
      @SuppressWarnings({"unused", "hiding"}) final java.io.Writer jamonWriter)
      throws java.io.IOException {
    // 35, 1

    Collection<ServerName> backupMasters = null;
    if (master.isActiveMaster()) {
      ClusterStatus status = master.getClusterStatus();
      backupMasters = status.getBackupMasters();
    }

    // 43, 1
    jamonWriter.write("<table class=\"table table-striped\">\n");
    // 44, 1
    if ((backupMasters != null && backupMasters.size() > 0)) {
      // 44, 59
      jamonWriter.write(
          "\n<tr>\n    <th>ServerName</th>\n    <th>Port</th>\n    <th>Start Time</th>\n</tr>\n");
      // 50, 1

      ServerName[] serverNames = backupMasters.toArray(new ServerName[backupMasters.size()]);
      Arrays.sort(serverNames);
      for (ServerName serverName : serverNames) {

        // 55, 1
        jamonWriter.write("<tr>\n    <td>");
        // 56, 9
        org.jamon.escaping.Escaping.HTML.write(
            org.jamon.emit.StandardEmitter.valueOf(serverName.getHostname()), jamonWriter);
        // 56, 39
        jamonWriter.write("</td>\n    <td>");
        // 57, 9
        org.jamon.escaping.Escaping.HTML.write(
            org.jamon.emit.StandardEmitter.valueOf(serverName.getPort()), jamonWriter);
        // 57, 35
        jamonWriter.write("</td>\n    <td>");
        // 58, 9
        org.jamon.escaping.Escaping.HTML.write(
            org.jamon.emit.StandardEmitter.valueOf(new Date(serverName.getStartcode())),
            jamonWriter);
        // 58, 50
        jamonWriter.write("</td>\n</tr>\n");
        // 60, 1

      }
    }
    // 63, 7
    jamonWriter.write("\n<tr><td>Total:");
    // 64, 15
    org.jamon.escaping.Escaping.HTML.write(
        org.jamon.emit.StandardEmitter.valueOf((backupMasters != null) ? backupMasters.size() : 0),
        jamonWriter);
    // 64, 71
    jamonWriter.write("</td>\n</table>\n\n\n\n\n");
  }
Ejemplo n.º 5
0
 public static void waitUntilIndexTableCreated(HMaster master, String tableName)
     throws IOException, InterruptedException {
   boolean isEnabled = false;
   boolean isExist = false;
   do {
     isExist = MetaReader.tableExists(master.getCatalogTracker(), TableName.valueOf(tableName));
     isEnabled =
         master.getAssignmentManager().getZKTable().isEnabledTable(TableName.valueOf(tableName));
     Thread.sleep(1000);
   } while ((false == isExist) && (false == isEnabled));
 }
 @Test
 public void testMergeWithReplicas() throws Exception {
   final TableName tableName = TableName.valueOf("testMergeWithReplicas");
   // Create table and load data.
   createTableAndLoadData(master, tableName, 5, 2);
   List<Pair<HRegionInfo, ServerName>> initialRegionToServers =
       MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tableName);
   // Merge 1st and 2nd region
   PairOfSameType<HRegionInfo> mergedRegions =
       mergeRegionsAndVerifyRegionNum(master, tableName, 0, 2, 5 * 2 - 2);
   List<Pair<HRegionInfo, ServerName>> currentRegionToServers =
       MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tableName);
   List<HRegionInfo> initialRegions = new ArrayList<HRegionInfo>();
   for (Pair<HRegionInfo, ServerName> p : initialRegionToServers) {
     initialRegions.add(p.getFirst());
   }
   List<HRegionInfo> currentRegions = new ArrayList<HRegionInfo>();
   for (Pair<HRegionInfo, ServerName> p : currentRegionToServers) {
     currentRegions.add(p.getFirst());
   }
   assertTrue(initialRegions.contains(mergedRegions.getFirst())); // this is the first region
   assertTrue(
       initialRegions.contains(
           RegionReplicaUtil.getRegionInfoForReplica(
               mergedRegions.getFirst(), 1))); // this is the replica of the first region
   assertTrue(initialRegions.contains(mergedRegions.getSecond())); // this is the second region
   assertTrue(
       initialRegions.contains(
           RegionReplicaUtil.getRegionInfoForReplica(
               mergedRegions.getSecond(), 1))); // this is the replica of the second region
   assertTrue(!initialRegions.contains(currentRegions.get(0))); // this is the new region
   assertTrue(
       !initialRegions.contains(
           RegionReplicaUtil.getRegionInfoForReplica(
               currentRegions.get(0), 1))); // replica of the new region
   assertTrue(
       currentRegions.contains(
           RegionReplicaUtil.getRegionInfoForReplica(
               currentRegions.get(0), 1))); // replica of the new region
   assertTrue(
       !currentRegions.contains(
           RegionReplicaUtil.getRegionInfoForReplica(
               mergedRegions.getFirst(), 1))); // replica of the merged region
   assertTrue(
       !currentRegions.contains(
           RegionReplicaUtil.getRegionInfoForReplica(
               mergedRegions.getSecond(), 1))); // replica of the merged region
 }
 @BeforeClass
 public static void beforeAllTests() throws Exception {
   // Start a cluster
   TEST_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
   MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
   master = cluster.getMaster();
   master.balanceSwitch(false);
   admin = TEST_UTIL.getHBaseAdmin();
 }
 private PairOfSameType<HRegionInfo> requestMergeRegion(
     HMaster master, TableName tablename, int regionAnum, int regionBnum) throws Exception {
   List<Pair<HRegionInfo, ServerName>> tableRegions =
       MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
   HRegionInfo regionA = tableRegions.get(regionAnum).getFirst();
   HRegionInfo regionB = tableRegions.get(regionBnum).getFirst();
   TEST_UTIL
       .getHBaseAdmin()
       .mergeRegions(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false);
   return new PairOfSameType<HRegionInfo>(regionA, regionB);
 }
Ejemplo n.º 9
0
 @BeforeClass
 public static void setUp() throws Exception {
   TEST_UTIL = new HBaseTestingUtility();
   TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE);
   admin = TEST_UTIL.getHBaseAdmin();
   cluster = TEST_UTIL.getHBaseCluster();
   master = ((MiniHBaseCluster) cluster).getMaster();
   zkNamespaceManager = new ZKNamespaceManager(master.getZooKeeper());
   zkNamespaceManager.start();
   LOG.info("Done initializing cluster");
 }
  private void waitAndVerifyRegionNum(HMaster master, TableName tablename, int expectedRegionNum)
      throws Exception {
    List<Pair<HRegionInfo, ServerName>> tableRegionsInMeta;
    List<HRegionInfo> tableRegionsInMaster;
    long timeout = System.currentTimeMillis() + waitTime;
    while (System.currentTimeMillis() < timeout) {
      tableRegionsInMeta =
          MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
      tableRegionsInMaster =
          master.getAssignmentManager().getRegionStates().getRegionsOfTable(tablename);
      if (tableRegionsInMeta.size() == expectedRegionNum
          && tableRegionsInMaster.size() == expectedRegionNum) {
        break;
      }
      Thread.sleep(250);
    }

    tableRegionsInMeta =
        MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
    LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
    assertEquals(expectedRegionNum, tableRegionsInMeta.size());
  }
Ejemplo n.º 11
0
  @Test
  public void createTableTest() throws IOException, InterruptedException {
    String testName = "createTableTest";
    String nsName = prefix + "_" + testName;
    LOG.info(testName);

    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(nsName + ":my_table"));
    HColumnDescriptor colDesc = new HColumnDescriptor("my_cf");
    desc.addFamily(colDesc);
    try {
      admin.createTable(desc);
      fail("Expected no namespace exists exception");
    } catch (NamespaceNotFoundException ex) {
    }
    // create table and in new namespace
    admin.createNamespace(NamespaceDescriptor.create(nsName).build());
    admin.createTable(desc);
    TEST_UTIL.waitTableAvailable(desc.getTableName().getName(), 10000);
    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
    assertTrue(
        fs.exists(
            new Path(
                master.getMasterFileSystem().getRootDir(),
                new Path(
                    HConstants.BASE_NAMESPACE_DIR,
                    new Path(nsName, desc.getTableName().getQualifierAsString())))));
    assertEquals(1, admin.listTables().length);

    // verify non-empty namespace can't be removed
    try {
      admin.deleteNamespace(nsName);
      fail("Expected non-empty namespace constraint exception");
    } catch (Exception ex) {
      LOG.info("Caught expected exception: " + ex);
    }

    // sanity check try to write and read from table
    Table table = TEST_UTIL.getConnection().getTable(desc.getTableName());
    Put p = new Put(Bytes.toBytes("row1"));
    p.add(Bytes.toBytes("my_cf"), Bytes.toBytes("my_col"), Bytes.toBytes("value1"));
    table.put(p);
    // flush and read from disk to make sure directory changes are working
    admin.flush(desc.getTableName());
    Get g = new Get(Bytes.toBytes("row1"));
    assertTrue(table.exists(g));

    // normal case of removing namespace
    TEST_UTIL.deleteTable(desc.getTableName());
    admin.deleteNamespace(nsName);
  }
 /**
  * Test adding server to draining servers and then move regions off it. Make sure that no regions
  * are moved back to the draining server.
  *
  * @throws IOException
  * @throws KeeperException
  */
 @Test // (timeout=30000)
 public void testDrainingServerOffloading() throws Exception {
   // I need master in the below.
   HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
   HRegionInfo hriToMoveBack = null;
   // Set first server as draining server.
   HRegionServer drainingServer =
       setDrainingServer(TEST_UTIL.getMiniHBaseCluster().getRegionServer(0));
   try {
     final int regionsOnDrainingServer = drainingServer.getNumberOfOnlineRegions();
     Assert.assertTrue(regionsOnDrainingServer > 0);
     List<HRegionInfo> hris = ProtobufUtil.getOnlineRegions(drainingServer);
     for (HRegionInfo hri : hris) {
       // Pass null and AssignmentManager will chose a random server BUT it
       // should exclude draining servers.
       master.moveRegion(
           null, RequestConverter.buildMoveRegionRequest(hri.getEncodedNameAsBytes(), null));
       // Save off region to move back.
       hriToMoveBack = hri;
     }
     // Wait for regions to come back on line again.
     waitForAllRegionsOnline();
     Assert.assertEquals(0, drainingServer.getNumberOfOnlineRegions());
   } finally {
     unsetDrainingServer(drainingServer);
   }
   // Now we've unset the draining server, we should be able to move a region
   // to what was the draining server.
   master.moveRegion(
       null,
       RequestConverter.buildMoveRegionRequest(
           hriToMoveBack.getEncodedNameAsBytes(),
           Bytes.toBytes(drainingServer.getServerName().toString())));
   // Wait for regions to come back on line again.
   waitForAllRegionsOnline();
   Assert.assertEquals(1, drainingServer.getNumberOfOnlineRegions());
 }
Ejemplo n.º 13
0
  public static void stopMasterAndAssignMeta(HBaseTestingUtility HTU)
      throws IOException, InterruptedException {
    // Stop master
    HMaster master = HTU.getHBaseCluster().getMaster();
    ServerName masterAddr = master.getServerName();
    master.stopMaster();

    Log.info("Waiting until master thread exits");
    while (HTU.getHBaseCluster().getMasterThread() != null
        && HTU.getHBaseCluster().getMasterThread().isAlive()) {
      Threads.sleep(100);
    }

    HRegionServer.TEST_SKIP_REPORTING_TRANSITION = true;
    // Master is down, so is the meta. We need to assign it somewhere
    // so that regions can be assigned during the mocking phase.
    HRegionServer hrs = HTU.getHBaseCluster().getLiveRegionServerThreads().get(0).getRegionServer();
    ZooKeeperWatcher zkw = hrs.getZooKeeper();
    MetaTableLocator mtl = new MetaTableLocator();
    ServerName sn = mtl.getMetaRegionLocation(zkw);
    if (sn != null && !masterAddr.equals(sn)) {
      return;
    }

    ProtobufUtil.openRegion(
        hrs.getRSRpcServices(), hrs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
    while (true) {
      sn = mtl.getMetaRegionLocation(zkw);
      if (sn != null
          && sn.equals(hrs.getServerName())
          && hrs.onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
        break;
      }
      Thread.sleep(100);
    }
  }
  /**
   * This test tests 1, merging region not online; 2, merging same two regions; 3, merging unknown
   * regions. They are in one test case so that we don't have to create many tables, and these tests
   * are simple.
   */
  @Test
  public void testMerge() throws Exception {
    LOG.info("Starting testMerge");
    final TableName tableName = TableName.valueOf("testMerge");

    try {
      // Create table and load data.
      Table table = createTableAndLoadData(master, tableName);
      RegionStates regionStates = master.getAssignmentManager().getRegionStates();
      List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName);
      // Fake offline one region
      HRegionInfo a = regions.get(0);
      HRegionInfo b = regions.get(1);
      regionStates.regionOffline(a);
      try {
        // Merge offline region. Region a is offline here
        admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
        fail("Offline regions should not be able to merge");
      } catch (IOException ie) {
        System.out.println(ie);
        assertTrue(
            "Exception should mention regions not online",
            StringUtils.stringifyException(ie).contains("regions not online")
                && ie instanceof MergeRegionException);
      }
      try {
        // Merge the same region: b and b.
        admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true);
        fail("A region should not be able to merge with itself, even forcifully");
      } catch (IOException ie) {
        assertTrue(
            "Exception should mention regions not online",
            StringUtils.stringifyException(ie).contains("region to itself")
                && ie instanceof MergeRegionException);
      }
      try {
        // Merge unknown regions
        admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
        fail("Unknown region could not be merged");
      } catch (IOException ie) {
        assertTrue("UnknownRegionException should be thrown", ie instanceof UnknownRegionException);
      }
      table.close();
    } finally {
      TEST_UTIL.deleteTable(tableName);
    }
  }
Ejemplo n.º 15
0
  public static List<Pair<byte[], ServerName>> getStartKeysAndLocations(
      HMaster master, String tableName) throws IOException, InterruptedException {

    List<Pair<HRegionInfo, ServerName>> tableRegionsAndLocations =
        MetaReader.getTableRegionsAndLocations(
            master.getCatalogTracker(), TableName.valueOf(tableName));
    List<Pair<byte[], ServerName>> startKeyAndLocationPairs =
        new ArrayList<Pair<byte[], ServerName>>(tableRegionsAndLocations.size());
    Pair<byte[], ServerName> startKeyAndLocation = null;
    for (Pair<HRegionInfo, ServerName> regionAndLocation : tableRegionsAndLocations) {
      startKeyAndLocation =
          new Pair<byte[], ServerName>(
              regionAndLocation.getFirst().getStartKey(), regionAndLocation.getSecond());
      startKeyAndLocationPairs.add(startKeyAndLocation);
    }
    return startKeyAndLocationPairs;
  }
 @Override
 public ReportRegionStateTransitionResponse reportRegionStateTransition(
     RpcController c, ReportRegionStateTransitionRequest req) throws ServiceException {
   ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
   if (enabled.get()
       && req.getTransition(0).getTransitionCode() == TransitionCode.READY_TO_MERGE
       && !resp.hasErrorMessage()) {
     RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
     for (RegionState regionState : regionStates.getRegionsInTransition().values()) {
       // Find the merging_new region and remove it
       if (regionState.isMergingNew()) {
         regionStates.deleteRegion(regionState.getRegion());
       }
     }
   }
   return resp;
 }
Ejemplo n.º 17
0
  static {
    conf = HBaseConfiguration.create();
    conf.set(
        ConfigProperties.CONFIG_NAME_HBASE_MASTER,
        config.getProperty(ConfigProperties.CONFIG_NAME_HBASE_MASTER));
    conf.set(
        ConfigProperties.CONFIG_NAME_HBASE_ZOOKEEPER_PROPRERTY_CLIENTPORT,
        config.getProperty(ConfigProperties.CONFIG_NAME_HBASE_ZOOKEEPER_PROPRERTY_CLIENTPORT));
    conf.set(
        ConfigProperties.CONFIG_NAME_HBASE_ZOOKEEPER_QUORUM,
        config.getProperty(ConfigProperties.CONFIG_NAME_HBASE_ZOOKEEPER_QUORUM));
    try {
      master = new HMaster(conf);
      watcher = master.getZooKeeperWatcher();

    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Ejemplo n.º 18
0
  @Test
  public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
    AssignmentManager am;
    final HMaster master = Mockito.mock(HMaster.class);
    final Server server = Mockito.mock(Server.class);
    final ServerManager serverManager = Mockito.mock(ServerManager.class);
    final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
    final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
    final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
    final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
    final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
    final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();

    Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();

    HRegionInfo REGIONINFO_A =
        new HRegionInfo(
            TableName.valueOf("table_A"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_B =
        new HRegionInfo(
            TableName.valueOf("table_B"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_C =
        new HRegionInfo(
            TableName.valueOf("table_C"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_D =
        new HRegionInfo(
            TableName.valueOf("table_D"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_E =
        new HRegionInfo(
            TableName.valueOf("table_E"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);

    Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
    List<ServerName> drainedServers = new ArrayList<ServerName>();

    onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);

    bulk.put(REGIONINFO_A, SERVERNAME_A);
    bulk.put(REGIONINFO_B, SERVERNAME_B);
    bulk.put(REGIONINFO_C, SERVERNAME_C);
    bulk.put(REGIONINFO_D, SERVERNAME_D);
    bulk.put(REGIONINFO_E, SERVERNAME_E);

    ZooKeeperWatcher zkWatcher =
        new ZooKeeperWatcher(
            TEST_UTIL.getConfiguration(), "zkWatcher-BulkAssignTest", abortable, true);

    Mockito.when(server.getConfiguration()).thenReturn(conf);
    Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
    Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);

    CoordinatedStateManager cp = new ZkCoordinatedStateManager();
    cp.initialize(server);
    cp.start();

    Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);

    Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
    Mockito.when(serverManager.getOnlineServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    Mockito.when(serverManager.createDestinationServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
    Mockito.when(serverManager.createDestinationServersList(null))
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
      Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
      Mockito.when(serverManager.sendRegionClose(entry.getValue(), entry.getKey(), -1))
          .thenReturn(true);
      Mockito.when(serverManager.sendRegionOpen(entry.getValue(), entry.getKey(), -1, null))
          .thenReturn(RegionOpeningState.OPENED);
      Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
    }

    Mockito.when(master.getServerManager()).thenReturn(serverManager);

    drainedServers.add(SERVERNAME_A);
    drainedServers.add(SERVERNAME_B);
    drainedServers.add(SERVERNAME_C);
    drainedServers.add(SERVERNAME_D);

    am =
        new AssignmentManager(
            server,
            serverManager,
            balancer,
            startupMasterExecutor("mockExecutorServiceBulk"),
            null,
            null);

    Mockito.when(master.getAssignmentManager()).thenReturn(am);

    zkWatcher.registerListener(am);

    for (ServerName drained : drainedServers) {
      addServerToDrainedList(drained, onlineServers, serverManager);
    }

    am.assign(bulk);

    Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
    for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
      setRegionOpenedOnZK(
          zkWatcher, entry.getValue().getServerName(), entry.getValue().getRegion());
    }

    am.waitForAssignment(REGIONINFO_A);
    am.waitForAssignment(REGIONINFO_B);
    am.waitForAssignment(REGIONINFO_C);
    am.waitForAssignment(REGIONINFO_D);
    am.waitForAssignment(REGIONINFO_E);

    Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
    for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
      LOG.info(
          "Region Assignment: "
              + entry.getKey().getRegionNameAsString()
              + " Server: "
              + entry.getValue());
      bunchServersAssigned.add(entry.getValue());
    }

    for (ServerName sn : drainedServers) {
      assertFalse(bunchServersAssigned.contains(sn));
    }
  }
Ejemplo n.º 19
0
  @Test(timeout = 300000)
  public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
    final int NUM_MASTERS = 1;
    final int NUM_RS = 3;
    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);

    try {
      final byte[] TABLENAME = Bytes.toBytes("testDataCorrectnessReplayingRecoveredEdits");
      final byte[] FAMILY = Bytes.toBytes("family");
      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
      HMaster master = cluster.getMaster();

      // Create table
      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
      desc.addFamily(new HColumnDescriptor(FAMILY));
      HBaseAdmin hbaseAdmin = TEST_UTIL.getHBaseAdmin();
      hbaseAdmin.createTable(desc);

      assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));

      // Put data: r1->v1
      Log.info("Loading r1 to v1 into " + Bytes.toString(TABLENAME));
      HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
      putDataAndVerify(table, "r1", FAMILY, "v1", 1);

      // Move region to target server
      HRegionInfo regionInfo = table.getRegionLocation("r1").getRegionInfo();
      int originServerNum = cluster.getServerWith(regionInfo.getRegionName());
      HRegionServer originServer = cluster.getRegionServer(originServerNum);
      int targetServerNum = (originServerNum + 1) % NUM_RS;
      HRegionServer targetServer = cluster.getRegionServer(targetServerNum);
      assertFalse(originServer.equals(targetServer));
      Log.info("Moving " + regionInfo.getEncodedName() + " to " + targetServer.getServerName());
      hbaseAdmin.move(
          regionInfo.getEncodedNameAsBytes(),
          Bytes.toBytes(targetServer.getServerName().getServerName()));
      do {
        Thread.sleep(1);
      } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum);

      // Put data: r2->v2
      Log.info("Loading r2 to v2 into " + Bytes.toString(TABLENAME));
      putDataAndVerify(table, "r2", FAMILY, "v2", 2);

      // Move region to origin server
      Log.info("Moving " + regionInfo.getEncodedName() + " to " + originServer.getServerName());
      hbaseAdmin.move(
          regionInfo.getEncodedNameAsBytes(),
          Bytes.toBytes(originServer.getServerName().getServerName()));
      do {
        Thread.sleep(1);
      } while (cluster.getServerWith(regionInfo.getRegionName()) == targetServerNum);

      // Put data: r3->v3
      Log.info("Loading r3 to v3 into " + Bytes.toString(TABLENAME));
      putDataAndVerify(table, "r3", FAMILY, "v3", 3);

      // Kill target server
      Log.info("Killing target server " + targetServer.getServerName());
      targetServer.kill();
      cluster.getRegionServerThreads().get(targetServerNum).join();
      // Wait until finish processing of shutdown
      while (master.getServerManager().areDeadServersInProgress()) {
        Thread.sleep(5);
      }
      // Kill origin server
      Log.info("Killing origin server " + targetServer.getServerName());
      originServer.kill();
      cluster.getRegionServerThreads().get(originServerNum).join();

      // Put data: r4->v4
      Log.info("Loading r4 to v4 into " + Bytes.toString(TABLENAME));
      putDataAndVerify(table, "r4", FAMILY, "v4", 4);

    } finally {
      TEST_UTIL.shutdownMiniCluster();
    }
  }
  /**
   * Test that draining servers are ignored even after killing regionserver(s). Verify that the
   * draining server is not given any of the dead servers regions.
   *
   * @throws KeeperException
   * @throws IOException
   */
  @Test(timeout = 30000)
  public void testDrainingServerWithAbort() throws KeeperException, Exception {
    HMaster master = TEST_UTIL.getHBaseCluster().getMaster();

    waitForAllRegionsOnline();

    final long regionCount = TEST_UTIL.getMiniHBaseCluster().countServedRegions();

    // Let's get a copy of the regions today.
    Collection<HRegion> regions = new ArrayList<HRegion>();
    for (int i = 0; i < NB_SLAVES; i++) {
      HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i);
      regions.addAll(hrs.getCopyOfOnlineRegionsSortedBySize().values());
    }

    // Choose the draining server
    HRegionServer drainingServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
    final int regionsOnDrainingServer = drainingServer.getNumberOfOnlineRegions();
    Assert.assertTrue(regionsOnDrainingServer > 0);

    ServerManager sm = master.getServerManager();

    Collection<HRegion> regionsBefore =
        drainingServer.getCopyOfOnlineRegionsSortedBySize().values();
    LOG.info("Regions of drained server are: " + regionsBefore);

    try {
      // Add first server to draining servers up in zk.
      setDrainingServer(drainingServer);

      // wait for the master to receive and manage the event
      while (sm.createDestinationServersList().contains(drainingServer.getServerName())) {
        Thread.sleep(1);
      }

      LOG.info("The available servers are: " + sm.createDestinationServersList());

      Assert.assertEquals(
          "Nothing should have happened here.",
          regionsOnDrainingServer,
          drainingServer.getNumberOfOnlineRegions());
      Assert.assertFalse(
          "We should not have regions in transition here. List is: "
              + master.getAssignmentManager().getRegionStates().getRegionsInTransition(),
          master.getAssignmentManager().getRegionStates().isRegionsInTransition());

      // Kill a few regionservers.
      for (int aborted = 0; aborted <= 2; aborted++) {
        HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(aborted + 1);
        hrs.abort("Aborting");
      }

      // Wait for regions to come back online again.
      waitForAllRegionsOnline();

      Collection<HRegion> regionsAfter =
          drainingServer.getCopyOfOnlineRegionsSortedBySize().values();
      LOG.info("Regions of drained server are: " + regionsAfter);

      Assert.assertEquals(
          "Test conditions are not met: regions were" + " created/deleted during the test. ",
          regionCount,
          TEST_UTIL.getMiniHBaseCluster().countServedRegions());

      // Assert the draining server still has the same regions.
      StringBuilder result = new StringBuilder();
      for (HRegion r : regionsAfter) {
        if (!regionsBefore.contains(r)) {
          result.append(r).append(" was added after the drain");
          if (regions.contains(r)) {
            result.append("(existing region");
          } else {
            result.append("(new region)");
          }
          result.append("; ");
        }
      }
      for (HRegion r : regionsBefore) {
        if (!regionsAfter.contains(r)) {
          result.append(r).append(" was removed after the drain; ");
        }
      }
      Assert.assertTrue("Errors are: " + result.toString(), result.length() == 0);

    } finally {
      unsetDrainingServer(drainingServer);
    }
  }
Ejemplo n.º 21
0
  @Test
  public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
    AssignmentManager am;
    Configuration conf = TEST_UTIL.getConfiguration();
    final HMaster master = Mockito.mock(HMaster.class);
    final Server server = Mockito.mock(Server.class);
    final ServerManager serverManager = Mockito.mock(ServerManager.class);
    final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
    final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
    final HRegionInfo REGIONINFO =
        new HRegionInfo(
            TableName.valueOf("table_test"),
            HConstants.EMPTY_START_ROW,
            HConstants.EMPTY_START_ROW);

    ZooKeeperWatcher zkWatcher =
        new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "zkWatcher-Test", abortable, true);

    Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();

    onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);

    Mockito.when(server.getConfiguration()).thenReturn(conf);
    Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
    Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);

    CoordinatedStateManager cp = new ZkCoordinatedStateManager();
    cp.initialize(server);
    cp.start();

    Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);

    Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
    Mockito.when(serverManager.getOnlineServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    Mockito.when(serverManager.createDestinationServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
    Mockito.when(serverManager.createDestinationServersList(null))
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    for (ServerName sn : onlineServers.keySet()) {
      Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
      Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
      Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
      Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
          .thenReturn(RegionOpeningState.OPENED);
      Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
          .thenReturn(RegionOpeningState.OPENED);
      Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
    }

    Mockito.when(master.getServerManager()).thenReturn(serverManager);

    am =
        new AssignmentManager(
            server,
            serverManager,
            balancer,
            startupMasterExecutor("mockExecutorService"),
            null,
            null);

    Mockito.when(master.getAssignmentManager()).thenReturn(am);
    Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);

    am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));

    zkWatcher.registerListenerFirst(am);

    addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);

    am.assign(REGIONINFO, true);

    setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
    setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);

    am.waitForAssignment(REGIONINFO);

    assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
    assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
  }
Ejemplo n.º 22
0
  /**
   * Method execute
   *
   * @param mapping
   * @param form
   * @param request
   * @param response
   * @return ActionForward
   */
  public ActionForward execute(
      ActionMapping mapping,
      ActionForm form,
      HttpServletRequest request,
      HttpServletResponse response) {
    // TODO Auto-generated method stub
    HMaster master = (HMaster) getServlet().getServletContext().getAttribute(HMaster.MASTER);

    tableOperation to = new tableOperation();
    authorityOperation ao = new authorityOperation();

    Configuration conf = master.getConfiguration();
    boolean security = conf.getBoolean("hbase.server.security", false);

    HTableDescriptor[] adminTables = null;
    Map<byte[], ACLField> userTables = null;
    List<String> tableList = new ArrayList<String>();

    if (security) {
      String username = (String) request.getSession().getAttribute("username");
      String usertype = (String) request.getSession().getAttribute("usertype");
      try {
        if (usertype.equals("admin")) {
          adminTables = to.getAllTable(master);
          for (HTableDescriptor tableDesc : adminTables) {
            tableList.add(tableDesc.getNameAsString());
          }
        } else {
          userTables = ao.getUserVisibleTable(master, username);
          for (Map.Entry<byte[], ACLField> entry : userTables.entrySet()) {
            tableList.add(Bytes.toString(entry.getKey()));
          }
        }
      } catch (MasterNotRunningException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (ZooKeeperConnectionException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    } else {
      try {
        adminTables = to.getAllTable(master);
      } catch (MasterNotRunningException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (ZooKeeperConnectionException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
      for (HTableDescriptor tableDesc : adminTables) {
        tableList.add(tableDesc.getNameAsString());
      }
    }

    request.setAttribute("tableList", tableList);

    return mapping.findForward("success");
  }
  @SuppressWarnings("deprecation")
  @Test
  public void testCleanMergeReference() throws Exception {
    LOG.info("Starting testCleanMergeReference");
    admin.enableCatalogJanitor(false);
    try {
      final TableName tableName = TableName.valueOf("testCleanMergeReference");
      // Create table and load data.
      Table table = createTableAndLoadData(master, tableName);
      // Merge 1st and 2nd region
      mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1, INITIAL_REGION_NUM - 1);
      verifyRowCount(table, ROWSIZE);
      table.close();

      List<Pair<HRegionInfo, ServerName>> tableRegions =
          MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tableName);
      HRegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
      HTableDescriptor tableDescritor = master.getTableDescriptors().get(tableName);
      Result mergedRegionResult =
          MetaTableAccessor.getRegionResult(
              master.getConnection(), mergedRegionInfo.getRegionName());

      // contains merge reference in META
      assertTrue(
          mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER)
              != null);
      assertTrue(
          mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER)
              != null);

      // merging regions' directory are in the file system all the same
      PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(mergedRegionResult);
      HRegionInfo regionA = p.getFirst();
      HRegionInfo regionB = p.getSecond();
      FileSystem fs = master.getMasterFileSystem().getFileSystem();
      Path rootDir = master.getMasterFileSystem().getRootDir();

      Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
      Path regionAdir = new Path(tabledir, regionA.getEncodedName());
      Path regionBdir = new Path(tabledir, regionB.getEncodedName());
      assertTrue(fs.exists(regionAdir));
      assertTrue(fs.exists(regionBdir));

      admin.compactRegion(mergedRegionInfo.getRegionName());
      // wait until merged region doesn't have reference file
      long timeout = System.currentTimeMillis() + waitTime;
      HRegionFileSystem hrfs =
          new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
      while (System.currentTimeMillis() < timeout) {
        if (!hrfs.hasReferences(tableDescritor)) {
          break;
        }
        Thread.sleep(50);
      }
      assertFalse(hrfs.hasReferences(tableDescritor));

      // run CatalogJanitor to clean merge references in hbase:meta and archive the
      // files of merging regions
      int cleaned = admin.runCatalogScan();
      assertTrue(cleaned > 0);
      assertFalse(fs.exists(regionAdir));
      assertFalse(fs.exists(regionBdir));

      mergedRegionResult =
          MetaTableAccessor.getRegionResult(
              master.getConnection(), mergedRegionInfo.getRegionName());
      assertFalse(
          mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER)
              != null);
      assertFalse(
          mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER)
              != null);

    } finally {
      admin.enableCatalogJanitor(true);
    }
  }