Exemplo n.º 1
0
  @Test
  public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
    AssignmentManager am;
    final HMaster master = Mockito.mock(HMaster.class);
    final Server server = Mockito.mock(Server.class);
    final ServerManager serverManager = Mockito.mock(ServerManager.class);
    final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
    final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
    final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
    final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
    final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
    final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();

    Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();

    HRegionInfo REGIONINFO_A =
        new HRegionInfo(
            TableName.valueOf("table_A"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_B =
        new HRegionInfo(
            TableName.valueOf("table_B"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_C =
        new HRegionInfo(
            TableName.valueOf("table_C"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_D =
        new HRegionInfo(
            TableName.valueOf("table_D"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
    HRegionInfo REGIONINFO_E =
        new HRegionInfo(
            TableName.valueOf("table_E"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);

    Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
    List<ServerName> drainedServers = new ArrayList<ServerName>();

    onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);

    bulk.put(REGIONINFO_A, SERVERNAME_A);
    bulk.put(REGIONINFO_B, SERVERNAME_B);
    bulk.put(REGIONINFO_C, SERVERNAME_C);
    bulk.put(REGIONINFO_D, SERVERNAME_D);
    bulk.put(REGIONINFO_E, SERVERNAME_E);

    ZooKeeperWatcher zkWatcher =
        new ZooKeeperWatcher(
            TEST_UTIL.getConfiguration(), "zkWatcher-BulkAssignTest", abortable, true);

    Mockito.when(server.getConfiguration()).thenReturn(conf);
    Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
    Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);

    CoordinatedStateManager cp = new ZkCoordinatedStateManager();
    cp.initialize(server);
    cp.start();

    Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);

    Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
    Mockito.when(serverManager.getOnlineServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    Mockito.when(serverManager.createDestinationServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
    Mockito.when(serverManager.createDestinationServersList(null))
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
      Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
      Mockito.when(serverManager.sendRegionClose(entry.getValue(), entry.getKey(), -1))
          .thenReturn(true);
      Mockito.when(serverManager.sendRegionOpen(entry.getValue(), entry.getKey(), -1, null))
          .thenReturn(RegionOpeningState.OPENED);
      Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
    }

    Mockito.when(master.getServerManager()).thenReturn(serverManager);

    drainedServers.add(SERVERNAME_A);
    drainedServers.add(SERVERNAME_B);
    drainedServers.add(SERVERNAME_C);
    drainedServers.add(SERVERNAME_D);

    am =
        new AssignmentManager(
            server,
            serverManager,
            balancer,
            startupMasterExecutor("mockExecutorServiceBulk"),
            null,
            null);

    Mockito.when(master.getAssignmentManager()).thenReturn(am);

    zkWatcher.registerListener(am);

    for (ServerName drained : drainedServers) {
      addServerToDrainedList(drained, onlineServers, serverManager);
    }

    am.assign(bulk);

    Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
    for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
      setRegionOpenedOnZK(
          zkWatcher, entry.getValue().getServerName(), entry.getValue().getRegion());
    }

    am.waitForAssignment(REGIONINFO_A);
    am.waitForAssignment(REGIONINFO_B);
    am.waitForAssignment(REGIONINFO_C);
    am.waitForAssignment(REGIONINFO_D);
    am.waitForAssignment(REGIONINFO_E);

    Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
    for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
      LOG.info(
          "Region Assignment: "
              + entry.getKey().getRegionNameAsString()
              + " Server: "
              + entry.getValue());
      bunchServersAssigned.add(entry.getValue());
    }

    for (ServerName sn : drainedServers) {
      assertFalse(bunchServersAssigned.contains(sn));
    }
  }
Exemplo n.º 2
0
  @Test
  public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
    AssignmentManager am;
    Configuration conf = TEST_UTIL.getConfiguration();
    final HMaster master = Mockito.mock(HMaster.class);
    final Server server = Mockito.mock(Server.class);
    final ServerManager serverManager = Mockito.mock(ServerManager.class);
    final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
    final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
    final HRegionInfo REGIONINFO =
        new HRegionInfo(
            TableName.valueOf("table_test"),
            HConstants.EMPTY_START_ROW,
            HConstants.EMPTY_START_ROW);

    ZooKeeperWatcher zkWatcher =
        new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "zkWatcher-Test", abortable, true);

    Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();

    onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
    onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);

    Mockito.when(server.getConfiguration()).thenReturn(conf);
    Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
    Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);

    CoordinatedStateManager cp = new ZkCoordinatedStateManager();
    cp.initialize(server);
    cp.start();

    Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);

    Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
    Mockito.when(serverManager.getOnlineServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    Mockito.when(serverManager.createDestinationServersList())
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
    Mockito.when(serverManager.createDestinationServersList(null))
        .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));

    for (ServerName sn : onlineServers.keySet()) {
      Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
      Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
      Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
      Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
          .thenReturn(RegionOpeningState.OPENED);
      Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
          .thenReturn(RegionOpeningState.OPENED);
      Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
    }

    Mockito.when(master.getServerManager()).thenReturn(serverManager);

    am =
        new AssignmentManager(
            server,
            serverManager,
            balancer,
            startupMasterExecutor("mockExecutorService"),
            null,
            null);

    Mockito.when(master.getAssignmentManager()).thenReturn(am);
    Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);

    am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));

    zkWatcher.registerListenerFirst(am);

    addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);

    am.assign(REGIONINFO, true);

    setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
    setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);

    am.waitForAssignment(REGIONINFO);

    assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
    assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
  }
Exemplo n.º 3
0
  @Test
  public void testLogCleaning() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    // set TTL
    long ttl = 10000;
    conf.setLong("hbase.master.logcleaner.ttl", ttl);
    conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
    Replication.decorateMasterConfiguration(conf);
    Server server = new DummyServer();
    ReplicationQueues repQueues =
        ReplicationFactory.getReplicationQueues(server.getZooKeeper(), conf, server);
    repQueues.init(server.getServerName().toString());
    final Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HREGION_OLDLOGDIR_NAME);
    String fakeMachineName = URLEncoder.encode(server.getServerName().toString(), "UTF8");

    final FileSystem fs = FileSystem.get(conf);

    // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
    long now = System.currentTimeMillis();
    fs.delete(oldLogDir, true);
    fs.mkdirs(oldLogDir);
    // Case 1: 2 invalid files, which would be deleted directly
    fs.createNewFile(new Path(oldLogDir, "a"));
    fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a"));
    // Case 2: 1 "recent" file, not even deletable for the first log cleaner
    // (TimeToLiveLogCleaner), so we are not going down the chain
    System.out.println("Now is: " + now);
    for (int i = 1; i < 31; i++) {
      // Case 3: old files which would be deletable for the first log cleaner
      // (TimeToLiveLogCleaner), and also for the second (ReplicationLogCleaner)
      Path fileName = new Path(oldLogDir, fakeMachineName + "." + (now - i));
      fs.createNewFile(fileName);
      // Case 4: put 3 old log files in ZK indicating that they are scheduled
      // for replication so these files would pass the first log cleaner
      // (TimeToLiveLogCleaner) but would be rejected by the second
      // (ReplicationLogCleaner)
      if (i % (30 / 3) == 1) {
        repQueues.addLog(fakeMachineName, fileName.getName());
        System.out.println("Replication log file: " + fileName);
      }
    }

    // sleep for sometime to get newer modifcation time
    Thread.sleep(ttl);
    fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now));

    // Case 2: 1 newer file, not even deletable for the first log cleaner
    // (TimeToLiveLogCleaner), so we are not going down the chain
    fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000)));

    for (FileStatus stat : fs.listStatus(oldLogDir)) {
      System.out.println(stat.getPath().toString());
    }

    assertEquals(34, fs.listStatus(oldLogDir).length);

    LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir);
    cleaner.chore();

    // We end up with the current log file, a newer one and the 3 old log
    // files which are scheduled for replication
    TEST_UTIL.waitFor(
        1000,
        new Waiter.Predicate<Exception>() {
          @Override
          public boolean evaluate() throws Exception {
            return 5 == fs.listStatus(oldLogDir).length;
          }
        });

    for (FileStatus file : fs.listStatus(oldLogDir)) {
      System.out.println("Kept log files: " + file.getPath().getName());
    }
  }