Exemplo n.º 1
0
  /**
   * Tests an on-the-fly RPC that was scheduled for the earlier RS on the same port for openRegion.
   * The region server should reject this RPC. (HBASE-9721)
   */
  @Test
  public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception {
    Assert.assertTrue(getRS().getRegion(regionName).isAvailable());

    ServerName sn = getRS().getServerName();
    ServerName earlierServerName = ServerName.valueOf(sn.getHostname(), sn.getPort(), 1);

    try {
      CloseRegionRequest request =
          RequestConverter.buildCloseRegionRequest(earlierServerName, regionName);
      getRS().getRSRpcServices().closeRegion(null, request);
      Assert.fail("The closeRegion should have been rejected");
    } catch (ServiceException se) {
      Assert.assertTrue(se.getCause() instanceof IOException);
      Assert.assertTrue(
          se.getCause().getMessage().contains("This RPC was intended for a different server"));
    }

    // actual close
    closeRegionNoZK();
    try {
      AdminProtos.OpenRegionRequest orr =
          RequestConverter.buildOpenRegionRequest(earlierServerName, hri, null, null);
      getRS().getRSRpcServices().openRegion(null, orr);
      Assert.fail("The openRegion should have been rejected");
    } catch (ServiceException se) {
      Assert.assertTrue(se.getCause() instanceof IOException);
      Assert.assertTrue(
          se.getCause().getMessage().contains("This RPC was intended for a different server"));
    } finally {
      openRegion(HTU, getRS(), hri);
    }
  }
Exemplo n.º 2
0
 /**
  * @param count
  * @return Return <code>count</code> servernames.
  */
 private static ServerName[] makeServerNames(final int count) {
   ServerName[] sns = new ServerName[count];
   for (int i = 0; i < count; i++) {
     sns[i] = ServerName.valueOf("" + i + ".example.org", 60010, i);
   }
   return sns;
 }
  /**
   * @param favoredNodesStr The String of favored nodes
   * @return the list of ServerName for the byte array of favored nodes.
   */
  public static List<ServerName> getFavoredNodeList(String favoredNodesStr) {
    String[] favoredNodesArray = StringUtils.split(favoredNodesStr, ",");
    if (favoredNodesArray == null) return null;

    List<ServerName> serverList = new ArrayList<ServerName>();
    for (String hostNameAndPort : favoredNodesArray) {
      serverList.add(ServerName.valueOf(hostNameAndPort, ServerName.NON_STARTCODE));
    }
    return serverList;
  }
Exemplo n.º 4
0
 protected ServerAndLoad randomServer(final int numRegionsPerServer) {
   if (!this.serverQueue.isEmpty()) {
     ServerName sn = this.serverQueue.poll();
     return new ServerAndLoad(sn, numRegionsPerServer);
   }
   String host = "srv" + rand.nextInt(Integer.MAX_VALUE);
   int port = rand.nextInt(60000);
   long startCode = rand.nextLong();
   ServerName sn = ServerName.valueOf(host, port, startCode);
   return new ServerAndLoad(sn, numRegionsPerServer);
 }
Exemplo n.º 5
0
  @Test
  public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception {
    // this test is for when distributed log replay is enabled
    if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return;

    LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization");
    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
    MasterFileSystem fs = master.getMasterFileSystem();

    String failedRegion = "failedRegoin1";
    String staleRegion = "staleRegion";
    ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1");
    ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1");
    String walPath =
        "/hbase/data/.logs/" + inRecoveryServerName.getServerName() + "-splitting/test";
    // Create a ZKW to use in the test
    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
    zkw.getRecoverableZooKeeper()
        .create(
            ZKSplitLog.getEncodedNodeName(zkw, walPath),
            new SplitLogTask.Owned(inRecoveryServerName, fs.getLogRecoveryMode()).toByteArray(),
            Ids.OPEN_ACL_UNSAFE,
            CreateMode.PERSISTENT);
    String staleRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, staleRegion);
    ZKUtil.createWithParents(zkw, staleRegionPath);
    String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, failedRegion);
    inRecoveringRegionPath =
        ZKUtil.joinZNode(inRecoveringRegionPath, inRecoveryServerName.getServerName());
    ZKUtil.createWithParents(zkw, inRecoveringRegionPath);
    Set<ServerName> servers = new HashSet<ServerName>();
    servers.add(previouselyFaildServerName);
    fs.removeStaleRecoveringRegionsFromZK(servers);

    // verification
    assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1);
    assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1);

    ZKUtil.deleteChildrenRecursively(zkw, zkw.recoveringRegionsZNode);
    ZKUtil.deleteChildrenRecursively(zkw, zkw.splitLogZNode);
    zkw.close();
  }
Exemplo n.º 6
0
      @Override
      public void run() {
        while (!isStopped()) {
          try {
            List<HRegionInfo> regions =
                MetaScanner.listAllRegions(TEST_UTIL.getConfiguration(), connection, false);

            // select a random region
            HRegionInfo parent = regions.get(random.nextInt(regions.size()));
            if (parent == null || !TABLENAME.equals(parent.getTable())) {
              continue;
            }

            long startKey = 0, endKey = Long.MAX_VALUE;
            byte[] start = parent.getStartKey();
            byte[] end = parent.getEndKey();
            if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) {
              startKey = Bytes.toLong(parent.getStartKey());
            }
            if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) {
              endKey = Bytes.toLong(parent.getEndKey());
            }
            if (startKey == endKey) {
              continue;
            }

            long midKey =
                BigDecimal.valueOf(startKey)
                    .add(BigDecimal.valueOf(endKey))
                    .divideToIntegralValue(BigDecimal.valueOf(2))
                    .longValue();

            HRegionInfo splita = new HRegionInfo(TABLENAME, start, Bytes.toBytes(midKey));
            HRegionInfo splitb = new HRegionInfo(TABLENAME, Bytes.toBytes(midKey), end);

            MetaTableAccessor.splitRegion(
                connection, parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0));

            Threads.sleep(random.nextInt(200));
          } catch (Throwable e) {
            ex = e;
            Assert.fail(StringUtils.stringifyException(e));
          }
        }
      }
Exemplo n.º 7
0
 /**
  * Load the meta region state from the meta server ZNode.
  *
  * @param zkw
  * @param replicaId
  * @return regionstate
  * @throws KeeperException
  */
 public static RegionState getMetaRegionState(ZooKeeperWatcher zkw, int replicaId)
     throws KeeperException {
   RegionState.State state = RegionState.State.OPEN;
   ServerName serverName = null;
   try {
     byte[] data = ZKUtil.getData(zkw, zkw.getZNodeForReplica(replicaId));
     if (data != null && data.length > 0 && ProtobufUtil.isPBMagicPrefix(data)) {
       try {
         int prefixLen = ProtobufUtil.lengthOfPBMagic();
         ZooKeeperProtos.MetaRegionServer rl =
             ZooKeeperProtos.MetaRegionServer.PARSER.parseFrom(
                 data, prefixLen, data.length - prefixLen);
         if (rl.hasState()) {
           state = RegionState.State.convert(rl.getState());
         }
         HBaseProtos.ServerName sn = rl.getServer();
         serverName = ServerName.valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode());
       } catch (InvalidProtocolBufferException e) {
         throw new DeserializationException("Unable to parse meta region location");
       }
     } else {
       // old style of meta region location?
       serverName = ServerName.parseFrom(data);
     }
   } catch (DeserializationException e) {
     throw ZKUtil.convert(e);
   } catch (InterruptedException e) {
     Thread.currentThread().interrupt();
   }
   if (serverName == null) {
     state = RegionState.State.OFFLINE;
   }
   return new RegionState(
       RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO, replicaId),
       state,
       serverName);
 }
Exemplo n.º 8
0
/** Test client behavior w/o setting up a cluster. Mock up cluster emissions. */
@Category(SmallTests.class)
public class TestClientNoCluster extends Configured implements Tool {
  private static final Log LOG = LogFactory.getLog(TestClientNoCluster.class);
  private Configuration conf;
  public static final ServerName META_SERVERNAME =
      ServerName.valueOf("meta.example.org", 60010, 12345);

  @Before
  public void setUp() throws Exception {
    this.conf = HBaseConfiguration.create();
    // Run my HConnection overrides.  Use my little HConnectionImplementation below which
    // allows me insert mocks and also use my Registry below rather than the default zk based
    // one so tests run faster and don't have zk dependency.
    this.conf.set("hbase.client.registry.impl", SimpleRegistry.class.getName());
  }

  /** Simple cluster registry inserted in place of our usual zookeeper based one. */
  static class SimpleRegistry implements Registry {
    final ServerName META_HOST = META_SERVERNAME;

    @Override
    public void init(HConnection connection) {}

    @Override
    public HRegionLocation getMetaRegionLocation() throws IOException {
      return new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, META_HOST);
    }

    @Override
    public String getClusterId() {
      return HConstants.CLUSTER_ID_DEFAULT;
    }

    @Override
    public boolean isTableOnlineState(TableName tableName, boolean enabled) throws IOException {
      return enabled;
    }

    @Override
    public int getCurrentNrHRS() throws IOException {
      return 1;
    }
  }

  /**
   * Remove the @Ignore to try out timeout and retry asettings
   *
   * @throws IOException
   */
  @Ignore
  @Test
  public void testTimeoutAndRetries() throws IOException {
    Configuration localConfig = HBaseConfiguration.create(this.conf);
    // This override mocks up our exists/get call to throw a RegionServerStoppedException.
    localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName());
    HTable table = new HTable(localConfig, TableName.META_TABLE_NAME);
    Throwable t = null;
    LOG.info("Start");
    try {
      // An exists call turns into a get w/ a flag.
      table.exists(new Get(Bytes.toBytes("abc")));
    } catch (SocketTimeoutException e) {
      // I expect this exception.
      LOG.info("Got expected exception", e);
      t = e;
    } catch (RetriesExhaustedException e) {
      // This is the old, unwanted behavior.  If we get here FAIL!!!
      fail();
    } finally {
      table.close();
    }
    LOG.info("Stop");
    assertTrue(t != null);
  }

  /**
   * Test that operation timeout prevails over rpc default timeout and retries, etc.
   *
   * @throws IOException
   */
  @Test
  public void testRocTimeout() throws IOException {
    Configuration localConfig = HBaseConfiguration.create(this.conf);
    // This override mocks up our exists/get call to throw a RegionServerStoppedException.
    localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName());
    int pause = 10;
    localConfig.setInt("hbase.client.pause", pause);
    localConfig.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10);
    // Set the operation timeout to be < the pause.  Expectation is that after first pause, we will
    // fail out of the rpc because the rpc timeout will have been set to the operation tiemout
    // and it has expired.  Otherwise, if this functionality is broke, all retries will be run --
    // all ten of them -- and we'll get the RetriesExhaustedException exception.
    localConfig.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, pause - 1);
    HTable table = new HTable(localConfig, TableName.META_TABLE_NAME);
    Throwable t = null;
    try {
      // An exists call turns into a get w/ a flag.
      table.exists(new Get(Bytes.toBytes("abc")));
    } catch (SocketTimeoutException e) {
      // I expect this exception.
      LOG.info("Got expected exception", e);
      t = e;
    } catch (RetriesExhaustedException e) {
      // This is the old, unwanted behavior.  If we get here FAIL!!!
      fail();
    } finally {
      table.close();
    }
    assertTrue(t != null);
  }

  @Test
  public void testDoNotRetryMetaScanner() throws IOException {
    this.conf.set(
        "hbase.client.connection.impl", RegionServerStoppedOnScannerOpenConnection.class.getName());
    MetaScanner.metaScan(this.conf, null);
  }

  @Test
  public void testDoNotRetryOnScanNext() throws IOException {
    this.conf.set(
        "hbase.client.connection.impl", RegionServerStoppedOnScannerOpenConnection.class.getName());
    // Go against meta else we will try to find first region for the table on construction which
    // means we'll have to do a bunch more mocking.  Tests that go against meta only should be
    // good for a bit of testing.
    HTable table = new HTable(this.conf, TableName.META_TABLE_NAME);
    ResultScanner scanner = table.getScanner(HConstants.CATALOG_FAMILY);
    try {
      Result result = null;
      while ((result = scanner.next()) != null) {
        LOG.info(result);
      }
    } finally {
      scanner.close();
      table.close();
    }
  }

  @Test
  public void testRegionServerStoppedOnScannerOpen() throws IOException {
    this.conf.set(
        "hbase.client.connection.impl", RegionServerStoppedOnScannerOpenConnection.class.getName());
    // Go against meta else we will try to find first region for the table on construction which
    // means we'll have to do a bunch more mocking.  Tests that go against meta only should be
    // good for a bit of testing.
    HTable table = new HTable(this.conf, TableName.META_TABLE_NAME);
    ResultScanner scanner = table.getScanner(HConstants.CATALOG_FAMILY);
    try {
      Result result = null;
      while ((result = scanner.next()) != null) {
        LOG.info(result);
      }
    } finally {
      scanner.close();
      table.close();
    }
  }

  /** Override to shutdown going to zookeeper for cluster id and meta location. */
  static class ScanOpenNextThenExceptionThenRecoverConnection
      extends HConnectionManager.HConnectionImplementation {
    final ClientService.BlockingInterface stub;

    ScanOpenNextThenExceptionThenRecoverConnection(
        Configuration conf, boolean managed, ExecutorService pool) throws IOException {
      super(conf, managed);
      // Mock up my stub so open scanner returns a scanner id and then on next, we throw
      // exceptions for three times and then after that, we return no more to scan.
      this.stub = Mockito.mock(ClientService.BlockingInterface.class);
      long sid = 12345L;
      try {
        Mockito.when(
                stub.scan((RpcController) Mockito.any(), (ClientProtos.ScanRequest) Mockito.any()))
            .thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build())
            .thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito")))
            .thenReturn(
                ClientProtos.ScanResponse.newBuilder()
                    .setScannerId(sid)
                    .setMoreResults(false)
                    .build());
      } catch (ServiceException e) {
        throw new IOException(e);
      }
    }

    @Override
    public BlockingInterface getClient(ServerName sn) throws IOException {
      return this.stub;
    }
  }

  /** Override to shutdown going to zookeeper for cluster id and meta location. */
  static class RegionServerStoppedOnScannerOpenConnection
      extends HConnectionManager.HConnectionImplementation {
    final ClientService.BlockingInterface stub;

    RegionServerStoppedOnScannerOpenConnection(
        Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException {
      super(conf, managed);
      // Mock up my stub so open scanner returns a scanner id and then on next, we throw
      // exceptions for three times and then after that, we return no more to scan.
      this.stub = Mockito.mock(ClientService.BlockingInterface.class);
      long sid = 12345L;
      try {
        Mockito.when(
                stub.scan((RpcController) Mockito.any(), (ClientProtos.ScanRequest) Mockito.any()))
            .thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build())
            .thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito")))
            .thenReturn(
                ClientProtos.ScanResponse.newBuilder()
                    .setScannerId(sid)
                    .setMoreResults(false)
                    .build());
      } catch (ServiceException e) {
        throw new IOException(e);
      }
    }

    @Override
    public BlockingInterface getClient(ServerName sn) throws IOException {
      return this.stub;
    }
  }

  /** Override to check we are setting rpc timeout right. */
  static class RpcTimeoutConnection extends HConnectionManager.HConnectionImplementation {
    final ClientService.BlockingInterface stub;

    RpcTimeoutConnection(Configuration conf, boolean managed, ExecutorService pool, User user)
        throws IOException {
      super(conf, managed);
      // Mock up my stub so an exists call -- which turns into a get -- throws an exception
      this.stub = Mockito.mock(ClientService.BlockingInterface.class);
      try {
        Mockito.when(
                stub.get((RpcController) Mockito.any(), (ClientProtos.GetRequest) Mockito.any()))
            .thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito")));
      } catch (ServiceException e) {
        throw new IOException(e);
      }
    }

    @Override
    public BlockingInterface getClient(ServerName sn) throws IOException {
      return this.stub;
    }
  }

  /** Fake many regionservers and many regions on a connection implementation. */
  static class ManyServersManyRegionsConnection
      extends HConnectionManager.HConnectionImplementation {
    // All access should be synchronized
    final Map<ServerName, ClientService.BlockingInterface> serversByClient;

    /** Map of faked-up rows of a 'meta table'. */
    final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta;

    final AtomicLong sequenceids = new AtomicLong(0);
    private final Configuration conf;

    ManyServersManyRegionsConnection(
        Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException {
      super(conf, managed, pool, user);
      int serverCount = conf.getInt("hbase.test.servers", 10);
      this.serversByClient = new HashMap<ServerName, ClientService.BlockingInterface>(serverCount);
      this.meta =
          makeMeta(
              Bytes.toBytes(conf.get("hbase.test.tablename", Bytes.toString(BIG_USER_TABLE))),
              conf.getInt("hbase.test.regions", 100),
              conf.getLong("hbase.test.namespace.span", 1000),
              serverCount);
      this.conf = conf;
    }

    @Override
    public ClientService.BlockingInterface getClient(ServerName sn) throws IOException {
      // if (!sn.toString().startsWith("meta")) LOG.info(sn);
      ClientService.BlockingInterface stub = null;
      synchronized (this.serversByClient) {
        stub = this.serversByClient.get(sn);
        if (stub == null) {
          stub = new FakeServer(this.conf, meta, sequenceids);
          this.serversByClient.put(sn, stub);
        }
      }
      return stub;
    }
  }

  static MultiResponse doMultiResponse(
      final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta,
      final AtomicLong sequenceids,
      final MultiRequest request) {
    // Make a response to match the request.  Act like there were no failures.
    ClientProtos.MultiResponse.Builder builder = ClientProtos.MultiResponse.newBuilder();
    // Per Region.
    RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
    ResultOrException.Builder roeBuilder = ResultOrException.newBuilder();
    for (RegionAction regionAction : request.getRegionActionList()) {
      regionActionResultBuilder.clear();
      // Per Action in a Region.
      for (ClientProtos.Action action : regionAction.getActionList()) {
        roeBuilder.clear();
        // Return empty Result and proper index as result.
        roeBuilder.setResult(ClientProtos.Result.getDefaultInstance());
        roeBuilder.setIndex(action.getIndex());
        regionActionResultBuilder.addResultOrException(roeBuilder.build());
      }
      builder.addRegionActionResult(regionActionResultBuilder.build());
    }
    return builder.build();
  }

  /**
   * Fake 'server'. Implements the ClientService responding as though it were a 'server' (presumes a
   * new ClientService.BlockingInterface made per server).
   */
  static class FakeServer implements ClientService.BlockingInterface {
    private AtomicInteger multiInvocationsCount = new AtomicInteger(0);
    private final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta;
    private final AtomicLong sequenceids;
    private final long multiPause;
    private final int tooManyMultiRequests;

    FakeServer(
        final Configuration c,
        final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta,
        final AtomicLong sequenceids) {
      this.meta = meta;
      this.sequenceids = sequenceids;

      // Pause to simulate the server taking time applying the edits.  This will drive up the
      // number of threads used over in client.
      this.multiPause = c.getLong("hbase.test.multi.pause.when.done", 0);
      this.tooManyMultiRequests = c.getInt("hbase.test.multi.too.many", 3);
    }

    @Override
    public GetResponse get(RpcController controller, GetRequest request) throws ServiceException {
      boolean metaRegion =
          isMetaRegion(request.getRegion().getValue().toByteArray(), request.getRegion().getType());
      if (!metaRegion) {
        return doGetResponse(request);
      }
      return doMetaGetResponse(meta, request);
    }

    private GetResponse doGetResponse(GetRequest request) {
      ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder();
      ByteString row = request.getGet().getRow();
      resultBuilder.addCell(getStartCode(row));
      GetResponse.Builder builder = GetResponse.newBuilder();
      builder.setResult(resultBuilder.build());
      return builder.build();
    }

    @Override
    public MutateResponse mutate(RpcController controller, MutateRequest request)
        throws ServiceException {
      throw new NotImplementedException();
    }

    @Override
    public ScanResponse scan(RpcController controller, ScanRequest request)
        throws ServiceException {
      // Presume it is a scan of meta for now. Not all scans provide a region spec expecting
      // the server to keep reference by scannerid.  TODO.
      return doMetaScanResponse(meta, sequenceids, request);
    }

    @Override
    public BulkLoadHFileResponse bulkLoadHFile(
        RpcController controller, BulkLoadHFileRequest request) throws ServiceException {
      throw new NotImplementedException();
    }

    @Override
    public CoprocessorServiceResponse execService(
        RpcController controller, CoprocessorServiceRequest request) throws ServiceException {
      throw new NotImplementedException();
    }

    @Override
    public MultiResponse multi(RpcController controller, MultiRequest request)
        throws ServiceException {
      int concurrentInvocations = this.multiInvocationsCount.incrementAndGet();
      try {
        if (concurrentInvocations >= tooManyMultiRequests) {
          throw new ServiceException(
              new RegionTooBusyException("concurrentInvocations=" + concurrentInvocations));
        }
        Threads.sleep(multiPause);
        return doMultiResponse(meta, sequenceids, request);
      } finally {
        this.multiInvocationsCount.decrementAndGet();
      }
    }
  }

  static ScanResponse doMetaScanResponse(
      final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta,
      final AtomicLong sequenceids,
      final ScanRequest request) {
    ScanResponse.Builder builder = ScanResponse.newBuilder();
    int max = request.getNumberOfRows();
    int count = 0;
    Map<byte[], Pair<HRegionInfo, ServerName>> tail =
        request.hasScan() ? meta.tailMap(request.getScan().getStartRow().toByteArray()) : meta;
    ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder();
    for (Map.Entry<byte[], Pair<HRegionInfo, ServerName>> e : tail.entrySet()) {
      // Can be 0 on open of a scanner -- i.e. rpc to setup scannerid only.
      if (max <= 0) break;
      if (++count > max) break;
      HRegionInfo hri = e.getValue().getFirst();
      ByteString row = HBaseZeroCopyByteString.wrap(hri.getRegionName());
      resultBuilder.clear();
      resultBuilder.addCell(getRegionInfo(row, hri));
      resultBuilder.addCell(getServer(row, e.getValue().getSecond()));
      resultBuilder.addCell(getStartCode(row));
      builder.addResults(resultBuilder.build());
      // Set more to false if we are on the last region in table.
      if (hri.getEndKey().length <= 0) builder.setMoreResults(false);
      else builder.setMoreResults(true);
    }
    // If no scannerid, set one.
    builder.setScannerId(
        request.hasScannerId() ? request.getScannerId() : sequenceids.incrementAndGet());
    return builder.build();
  }

  static GetResponse doMetaGetResponse(
      final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta, final GetRequest request) {
    ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder();
    ByteString row = request.getGet().getRow();
    Pair<HRegionInfo, ServerName> p = meta.get(row.toByteArray());
    if (p == null) {
      if (request.getGet().getClosestRowBefore()) {
        byte[] bytes = row.toByteArray();
        SortedMap<byte[], Pair<HRegionInfo, ServerName>> head =
            bytes != null ? meta.headMap(bytes) : meta;
        p = head == null ? null : head.get(head.lastKey());
      }
    }
    if (p != null) {
      resultBuilder.addCell(getRegionInfo(row, p.getFirst()));
      resultBuilder.addCell(getServer(row, p.getSecond()));
    }
    resultBuilder.addCell(getStartCode(row));
    GetResponse.Builder builder = GetResponse.newBuilder();
    builder.setResult(resultBuilder.build());
    return builder.build();
  }

  /**
   * @param name region name or encoded region name.
   * @param type
   * @return True if we are dealing with a hbase:meta region.
   */
  static boolean isMetaRegion(final byte[] name, final RegionSpecifierType type) {
    switch (type) {
      case REGION_NAME:
        return Bytes.equals(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), name);
      case ENCODED_REGION_NAME:
        return Bytes.equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), name);
      default:
        throw new UnsupportedOperationException();
    }
  }

  private static final ByteString CATALOG_FAMILY_BYTESTRING =
      HBaseZeroCopyByteString.wrap(HConstants.CATALOG_FAMILY);
  private static final ByteString REGIONINFO_QUALIFIER_BYTESTRING =
      HBaseZeroCopyByteString.wrap(HConstants.REGIONINFO_QUALIFIER);
  private static final ByteString SERVER_QUALIFIER_BYTESTRING =
      HBaseZeroCopyByteString.wrap(HConstants.SERVER_QUALIFIER);

  static CellProtos.Cell.Builder getBaseCellBuilder(final ByteString row) {
    CellProtos.Cell.Builder cellBuilder = CellProtos.Cell.newBuilder();
    cellBuilder.setRow(row);
    cellBuilder.setFamily(CATALOG_FAMILY_BYTESTRING);
    cellBuilder.setTimestamp(System.currentTimeMillis());
    return cellBuilder;
  }

  static CellProtos.Cell getRegionInfo(final ByteString row, final HRegionInfo hri) {
    CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row);
    cellBuilder.setQualifier(REGIONINFO_QUALIFIER_BYTESTRING);
    cellBuilder.setValue(HBaseZeroCopyByteString.wrap(hri.toByteArray()));
    return cellBuilder.build();
  }

  static CellProtos.Cell getServer(final ByteString row, final ServerName sn) {
    CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row);
    cellBuilder.setQualifier(SERVER_QUALIFIER_BYTESTRING);
    cellBuilder.setValue(ByteString.copyFromUtf8(sn.getHostAndPort()));
    return cellBuilder.build();
  }

  static CellProtos.Cell getStartCode(final ByteString row) {
    CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row);
    cellBuilder.setQualifier(HBaseZeroCopyByteString.wrap(HConstants.STARTCODE_QUALIFIER));
    // TODO:
    cellBuilder.setValue(
        HBaseZeroCopyByteString.wrap(Bytes.toBytes(META_SERVERNAME.getStartcode())));
    return cellBuilder.build();
  }

  private static final byte[] BIG_USER_TABLE = Bytes.toBytes("t");

  /**
   * Format passed integer. Zero-pad. Copied from hbase-server PE class and small amendment. Make
   * them share.
   *
   * @param number
   * @return Returns zero-prefixed 10-byte wide decimal version of passed number (Does absolute in
   *     case number is negative).
   */
  private static byte[] format(final long number) {
    byte[] b = new byte[10];
    long d = number;
    for (int i = b.length - 1; i >= 0; i--) {
      b[i] = (byte) ((d % 10) + '0');
      d /= 10;
    }
    return b;
  }

  /**
   * @param count
   * @param namespaceSpan
   * @return <code>count</code> regions
   */
  private static HRegionInfo[] makeHRegionInfos(
      final byte[] tableName, final int count, final long namespaceSpan) {
    byte[] startKey = HConstants.EMPTY_BYTE_ARRAY;
    byte[] endKey = HConstants.EMPTY_BYTE_ARRAY;
    long interval = namespaceSpan / count;
    HRegionInfo[] hris = new HRegionInfo[count];
    for (int i = 0; i < count; i++) {
      if (i == 0) {
        endKey = format(interval);
      } else {
        startKey = endKey;
        if (i == count - 1) endKey = HConstants.EMPTY_BYTE_ARRAY;
        else endKey = format((i + 1) * interval);
      }
      hris[i] = new HRegionInfo(TableName.valueOf(tableName), startKey, endKey);
    }
    return hris;
  }

  /**
   * @param count
   * @return Return <code>count</code> servernames.
   */
  private static ServerName[] makeServerNames(final int count) {
    ServerName[] sns = new ServerName[count];
    for (int i = 0; i < count; i++) {
      sns[i] = ServerName.valueOf("" + i + ".example.org", 60010, i);
    }
    return sns;
  }

  /** Comparator for meta row keys. */
  private static class MetaRowsComparator implements Comparator<byte[]> {
    private final KeyValue.KVComparator delegate = new KeyValue.MetaComparator();

    @Override
    public int compare(byte[] left, byte[] right) {
      return delegate.compareRows(left, 0, left.length, right, 0, right.length);
    }
  }

  /**
   * Create up a map that is keyed by meta row name and whose value is the HRegionInfo and
   * ServerName to return for this row.
   *
   * @return Map with faked hbase:meta content in it.
   */
  static SortedMap<byte[], Pair<HRegionInfo, ServerName>> makeMeta(
      final byte[] tableName,
      final int regionCount,
      final long namespaceSpan,
      final int serverCount) {
    // I need a comparator for meta rows so we sort properly.
    SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta =
        new ConcurrentSkipListMap<byte[], Pair<HRegionInfo, ServerName>>(new MetaRowsComparator());
    HRegionInfo[] hris = makeHRegionInfos(tableName, regionCount, namespaceSpan);
    ServerName[] serverNames = makeServerNames(serverCount);
    int per = regionCount / serverCount;
    int count = 0;
    for (HRegionInfo hri : hris) {
      Pair<HRegionInfo, ServerName> p =
          new Pair<HRegionInfo, ServerName>(hri, serverNames[count++ / per]);
      meta.put(hri.getRegionName(), p);
    }
    return meta;
  }

  /**
   * Code for each 'client' to run.
   *
   * @param id
   * @param c
   * @param sharedConnection
   * @throws IOException
   */
  static void cycle(int id, final Configuration c, final HConnection sharedConnection)
      throws IOException {
    HTableInterface table = sharedConnection.getTable(BIG_USER_TABLE);
    table.setAutoFlushTo(false);
    long namespaceSpan = c.getLong("hbase.test.namespace.span", 1000000);
    long startTime = System.currentTimeMillis();
    final int printInterval = 100000;
    Random rd = new Random(id);
    boolean get = c.getBoolean("hbase.test.do.gets", false);
    try {
      Stopwatch stopWatch = new Stopwatch();
      stopWatch.start();
      for (int i = 0; i < namespaceSpan; i++) {
        byte[] b = format(rd.nextLong());
        if (get) {
          Get g = new Get(b);
          table.get(g);
        } else {
          Put p = new Put(b);
          p.add(HConstants.CATALOG_FAMILY, b, b);
          table.put(p);
        }
        if (i % printInterval == 0) {
          LOG.info("Put " + printInterval + "/" + stopWatch.elapsedMillis());
          stopWatch.reset();
          stopWatch.start();
        }
      }
      LOG.info(
          "Finished a cycle putting "
              + namespaceSpan
              + " in "
              + (System.currentTimeMillis() - startTime)
              + "ms");
    } finally {
      table.close();
    }
  }

  @Override
  public int run(String[] arg0) throws Exception {
    int errCode = 0;
    // TODO: Make command options.
    // How many servers to fake.
    final int servers = 1;
    // How many regions to put on the faked servers.
    final int regions = 100000;
    // How many 'keys' in the faked regions.
    final long namespaceSpan = 50000000;
    // How long to take to pause after doing a put; make this long if you want to fake a struggling
    // server.
    final long multiPause = 0;
    // Check args make basic sense.
    if ((namespaceSpan < regions) || (regions < servers)) {
      throw new IllegalArgumentException(
          "namespaceSpan="
              + namespaceSpan
              + " must be > regions="
              + regions
              + " which must be > servers="
              + servers);
    }

    // Set my many servers and many regions faking connection in place.
    getConf().set("hbase.client.connection.impl", ManyServersManyRegionsConnection.class.getName());
    // Use simple kv registry rather than zk
    getConf().set("hbase.client.registry.impl", SimpleRegistry.class.getName());
    // When to report fails.  Default is we report the 10th.  This means we'll see log everytime
    // an exception is thrown -- usually RegionTooBusyException when we have more than
    // hbase.test.multi.too.many requests outstanding at any time.
    getConf().setInt("hbase.client.start.log.errors.counter", 0);

    // Ugly but this is only way to pass in configs.into ManyServersManyRegionsConnection class.
    getConf().setInt("hbase.test.regions", regions);
    getConf().setLong("hbase.test.namespace.span", namespaceSpan);
    getConf().setLong("hbase.test.servers", servers);
    getConf().set("hbase.test.tablename", Bytes.toString(BIG_USER_TABLE));
    getConf().setLong("hbase.test.multi.pause.when.done", multiPause);
    // Let there be ten outstanding requests at a time before we throw RegionBusyException.
    getConf().setInt("hbase.test.multi.too.many", 10);
    final int clients = 2;

    // Have them all share the same connection so they all share the same instance of
    // ManyServersManyRegionsConnection so I can keep an eye on how many requests by server.
    final ExecutorService pool = Executors.newCachedThreadPool(Threads.getNamedThreadFactory("p"));
    // Executors.newFixedThreadPool(servers * 10, Threads.getNamedThreadFactory("p"));
    // Share a connection so I can keep counts in the 'server' on concurrency.
    final HConnection sharedConnection = HConnectionManager.createConnection(getConf() /*, pool*/);
    try {
      Thread[] ts = new Thread[clients];
      for (int j = 0; j < ts.length; j++) {
        final int id = j;
        ts[j] =
            new Thread("" + j) {
              final Configuration c = getConf();

              @Override
              public void run() {
                try {
                  cycle(id, c, sharedConnection);
                } catch (IOException e) {
                  e.printStackTrace();
                }
              }
            };
        ts[j].start();
      }
      for (int j = 0; j < ts.length; j++) {
        ts[j].join();
      }
    } finally {
      sharedConnection.close();
    }
    return errCode;
  }

  /**
   * Run a client instance against a faked up server.
   *
   * @param args TODO
   * @throws Exception
   */
  public static void main(String[] args) throws Exception {
    System.exit(ToolRunner.run(HBaseConfiguration.create(), new TestClientNoCluster(), args));
  }
}
  /**
   * Generate the assignment plan for the existing table
   *
   * @param tableName
   * @param assignmentSnapshot
   * @param regionLocalityMap
   * @param plan
   * @param munkresForSecondaryAndTertiary if set on true the assignment plan for the tertiary and
   *     secondary will be generated with Munkres algorithm, otherwise will be generated using
   *     placeSecondaryAndTertiaryRS
   * @throws IOException
   */
  private void genAssignmentPlan(
      TableName tableName,
      SnapshotOfRegionAssignmentFromMeta assignmentSnapshot,
      Map<String, Map<String, Float>> regionLocalityMap,
      FavoredNodesPlan plan,
      boolean munkresForSecondaryAndTertiary)
      throws IOException {
    // Get the all the regions for the current table
    List<HRegionInfo> regions = assignmentSnapshot.getTableToRegionMap().get(tableName);
    int numRegions = regions.size();

    // Get the current assignment map
    Map<HRegionInfo, ServerName> currentAssignmentMap =
        assignmentSnapshot.getRegionToRegionServerMap();

    // Get the all the region servers
    List<ServerName> servers = new ArrayList<ServerName>();
    try (Admin admin = this.connection.getAdmin()) {
      servers.addAll(admin.getClusterStatus().getServers());
    }

    LOG.info(
        "Start to generate assignment plan for "
            + numRegions
            + " regions from table "
            + tableName
            + " with "
            + servers.size()
            + " region servers");

    int slotsPerServer = (int) Math.ceil((float) numRegions / servers.size());
    int regionSlots = slotsPerServer * servers.size();

    // Compute the primary, secondary and tertiary costs for each region/server
    // pair. These costs are based only on node locality and rack locality, and
    // will be modified later.
    float[][] primaryCost = new float[numRegions][regionSlots];
    float[][] secondaryCost = new float[numRegions][regionSlots];
    float[][] tertiaryCost = new float[numRegions][regionSlots];

    if (this.enforceLocality && regionLocalityMap != null) {
      // Transform the locality mapping into a 2D array, assuming that any
      // unspecified locality value is 0.
      float[][] localityPerServer = new float[numRegions][regionSlots];
      for (int i = 0; i < numRegions; i++) {
        Map<String, Float> serverLocalityMap =
            regionLocalityMap.get(regions.get(i).getEncodedName());
        if (serverLocalityMap == null) {
          continue;
        }
        for (int j = 0; j < servers.size(); j++) {
          String serverName = servers.get(j).getHostname();
          if (serverName == null) {
            continue;
          }
          Float locality = serverLocalityMap.get(serverName);
          if (locality == null) {
            continue;
          }
          for (int k = 0; k < slotsPerServer; k++) {
            // If we can't find the locality of a region to a server, which occurs
            // because locality is only reported for servers which have some
            // blocks of a region local, then the locality for that pair is 0.
            localityPerServer[i][j * slotsPerServer + k] = locality.floatValue();
          }
        }
      }

      // Compute the total rack locality for each region in each rack. The total
      // rack locality is the sum of the localities of a region on all servers in
      // a rack.
      Map<String, Map<HRegionInfo, Float>> rackRegionLocality =
          new HashMap<String, Map<HRegionInfo, Float>>();
      for (int i = 0; i < numRegions; i++) {
        HRegionInfo region = regions.get(i);
        for (int j = 0; j < regionSlots; j += slotsPerServer) {
          String rack = rackManager.getRack(servers.get(j / slotsPerServer));
          Map<HRegionInfo, Float> rackLocality = rackRegionLocality.get(rack);
          if (rackLocality == null) {
            rackLocality = new HashMap<HRegionInfo, Float>();
            rackRegionLocality.put(rack, rackLocality);
          }
          Float localityObj = rackLocality.get(region);
          float locality = localityObj == null ? 0 : localityObj.floatValue();
          locality += localityPerServer[i][j];
          rackLocality.put(region, locality);
        }
      }
      for (int i = 0; i < numRegions; i++) {
        for (int j = 0; j < regionSlots; j++) {
          String rack = rackManager.getRack(servers.get(j / slotsPerServer));
          Float totalRackLocalityObj = rackRegionLocality.get(rack).get(regions.get(i));
          float totalRackLocality =
              totalRackLocalityObj == null ? 0 : totalRackLocalityObj.floatValue();

          // Primary cost aims to favor servers with high node locality and low
          // rack locality, so that secondaries and tertiaries can be chosen for
          // nodes with high rack locality. This might give primaries with
          // slightly less locality at first compared to a cost which only
          // considers the node locality, but should be better in the long run.
          primaryCost[i][j] = 1 - (2 * localityPerServer[i][j] - totalRackLocality);

          // Secondary cost aims to favor servers with high node locality and high
          // rack locality since the tertiary will be chosen from the same rack as
          // the secondary. This could be negative, but that is okay.
          secondaryCost[i][j] = 2 - (localityPerServer[i][j] + totalRackLocality);

          // Tertiary cost is only concerned with the node locality. It will later
          // be restricted to only hosts on the same rack as the secondary.
          tertiaryCost[i][j] = 1 - localityPerServer[i][j];
        }
      }
    }

    if (this.enforceMinAssignmentMove && currentAssignmentMap != null) {
      // We want to minimize the number of regions which move as the result of a
      // new assignment. Therefore, slightly penalize any placement which is for
      // a host that is not currently serving the region.
      for (int i = 0; i < numRegions; i++) {
        for (int j = 0; j < servers.size(); j++) {
          ServerName currentAddress = currentAssignmentMap.get(regions.get(i));
          if (currentAddress != null && !currentAddress.equals(servers.get(j))) {
            for (int k = 0; k < slotsPerServer; k++) {
              primaryCost[i][j * slotsPerServer + k] += NOT_CURRENT_HOST_PENALTY;
            }
          }
        }
      }
    }

    // Artificially increase cost of last slot of each server to evenly
    // distribute the slop, otherwise there will be a few servers with too few
    // regions and many servers with the max number of regions.
    for (int i = 0; i < numRegions; i++) {
      for (int j = 0; j < regionSlots; j += slotsPerServer) {
        primaryCost[i][j] += LAST_SLOT_COST_PENALTY;
        secondaryCost[i][j] += LAST_SLOT_COST_PENALTY;
        tertiaryCost[i][j] += LAST_SLOT_COST_PENALTY;
      }
    }

    RandomizedMatrix randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots);
    primaryCost = randomizedMatrix.transform(primaryCost);
    int[] primaryAssignment = new MunkresAssignment(primaryCost).solve();
    primaryAssignment = randomizedMatrix.invertIndices(primaryAssignment);

    // Modify the secondary and tertiary costs for each region/server pair to
    // prevent a region from being assigned to the same rack for both primary
    // and either one of secondary or tertiary.
    for (int i = 0; i < numRegions; i++) {
      int slot = primaryAssignment[i];
      String rack = rackManager.getRack(servers.get(slot / slotsPerServer));
      for (int k = 0; k < servers.size(); k++) {
        if (!rackManager.getRack(servers.get(k)).equals(rack)) {
          continue;
        }
        if (k == slot / slotsPerServer) {
          // Same node, do not place secondary or tertiary here ever.
          for (int m = 0; m < slotsPerServer; m++) {
            secondaryCost[i][k * slotsPerServer + m] = MAX_COST;
            tertiaryCost[i][k * slotsPerServer + m] = MAX_COST;
          }
        } else {
          // Same rack, do not place secondary or tertiary here if possible.
          for (int m = 0; m < slotsPerServer; m++) {
            secondaryCost[i][k * slotsPerServer + m] = AVOID_COST;
            tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST;
          }
        }
      }
    }
    if (munkresForSecondaryAndTertiary) {
      randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots);
      secondaryCost = randomizedMatrix.transform(secondaryCost);
      int[] secondaryAssignment = new MunkresAssignment(secondaryCost).solve();
      secondaryAssignment = randomizedMatrix.invertIndices(secondaryAssignment);

      // Modify the tertiary costs for each region/server pair to ensure that a
      // region is assigned to a tertiary server on the same rack as its secondary
      // server, but not the same server in that rack.
      for (int i = 0; i < numRegions; i++) {
        int slot = secondaryAssignment[i];
        String rack = rackManager.getRack(servers.get(slot / slotsPerServer));
        for (int k = 0; k < servers.size(); k++) {
          if (k == slot / slotsPerServer) {
            // Same node, do not place tertiary here ever.
            for (int m = 0; m < slotsPerServer; m++) {
              tertiaryCost[i][k * slotsPerServer + m] = MAX_COST;
            }
          } else {
            if (rackManager.getRack(servers.get(k)).equals(rack)) {
              continue;
            }
            // Different rack, do not place tertiary here if possible.
            for (int m = 0; m < slotsPerServer; m++) {
              tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST;
            }
          }
        }
      }

      randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots);
      tertiaryCost = randomizedMatrix.transform(tertiaryCost);
      int[] tertiaryAssignment = new MunkresAssignment(tertiaryCost).solve();
      tertiaryAssignment = randomizedMatrix.invertIndices(tertiaryAssignment);

      for (int i = 0; i < numRegions; i++) {
        List<ServerName> favoredServers =
            new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM);
        ServerName s = servers.get(primaryAssignment[i] / slotsPerServer);
        favoredServers.add(
            ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));

        s = servers.get(secondaryAssignment[i] / slotsPerServer);
        favoredServers.add(
            ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));

        s = servers.get(tertiaryAssignment[i] / slotsPerServer);
        favoredServers.add(
            ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
        // Update the assignment plan
        plan.updateAssignmentPlan(regions.get(i), favoredServers);
      }
      LOG.info(
          "Generated the assignment plan for "
              + numRegions
              + " regions from table "
              + tableName
              + " with "
              + servers.size()
              + " region servers");
      LOG.info("Assignment plan for secondary and tertiary generated " + "using MunkresAssignment");
    } else {
      Map<HRegionInfo, ServerName> primaryRSMap = new HashMap<HRegionInfo, ServerName>();
      for (int i = 0; i < numRegions; i++) {
        primaryRSMap.put(regions.get(i), servers.get(primaryAssignment[i] / slotsPerServer));
      }
      FavoredNodeAssignmentHelper favoredNodeHelper =
          new FavoredNodeAssignmentHelper(servers, conf);
      favoredNodeHelper.initialize();
      Map<HRegionInfo, ServerName[]> secondaryAndTertiaryMap =
          favoredNodeHelper.placeSecondaryAndTertiaryWithRestrictions(primaryRSMap);
      for (int i = 0; i < numRegions; i++) {
        List<ServerName> favoredServers =
            new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM);
        HRegionInfo currentRegion = regions.get(i);
        ServerName s = primaryRSMap.get(currentRegion);
        favoredServers.add(
            ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));

        ServerName[] secondaryAndTertiary = secondaryAndTertiaryMap.get(currentRegion);
        s = secondaryAndTertiary[0];
        favoredServers.add(
            ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));

        s = secondaryAndTertiary[1];
        favoredServers.add(
            ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
        // Update the assignment plan
        plan.updateAssignmentPlan(regions.get(i), favoredServers);
      }
      LOG.info(
          "Generated the assignment plan for "
              + numRegions
              + " regions from table "
              + tableName
              + " with "
              + servers.size()
              + " region servers");
      LOG.info(
          "Assignment plan for secondary and tertiary generated "
              + "using placeSecondaryAndTertiaryWithRestrictions method");
    }
  }
Exemplo n.º 10
0
 @Override
 public ServerName getServerName() {
   return ServerName.valueOf(this.serverName);
 }
Exemplo n.º 11
0
  /**
   * Reproduce locking up that happens when there's no further syncs after append fails, and causing
   * an isolated sync then infinite wait. See HBASE-16960. If below is broken, we will see this test
   * timeout because it is locked up.
   *
   * <p>Steps for reproduce:<br>
   * 1. Trigger server abort through dodgyWAL1<br>
   * 2. Add a {@link DummyWALActionsListener} to dodgyWAL2 to cause ringbuffer event handler thread
   * sleep for a while thus keeping {@code endOfBatch} false<br>
   * 3. Publish a sync then an append which will throw exception, check whether the sync could
   * return
   */
  @Test(timeout = 20000)
  public void testLockup16960() throws IOException {
    // A WAL that we can have throw exceptions when a flag is set.
    class DodgyFSLog extends FSHLog {
      // Set this when want the WAL to start throwing exceptions.
      volatile boolean throwException = false;

      public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration conf)
          throws IOException {
        super(fs, root, logDir, conf);
      }

      @Override
      protected Writer createWriterInstance(Path path) throws IOException {
        final Writer w = super.createWriterInstance(path);
        return new Writer() {
          @Override
          public void close() throws IOException {
            w.close();
          }

          @Override
          public void sync() throws IOException {
            if (throwException) {
              throw new IOException("FAKE! Failed to replace a bad datanode...SYNC");
            }
            w.sync();
          }

          @Override
          public void append(Entry entry) throws IOException {
            if (throwException) {
              throw new IOException("FAKE! Failed to replace a bad datanode...APPEND");
            }
            w.append(entry);
          }

          @Override
          public long getLength() {
            return w.getLength();
          }
        };
      }

      @Override
      protected long doReplaceWriter(Path oldPath, Path newPath, Writer nextWriter)
          throws IOException {
        if (throwException) {
          throw new FailedLogCloseException("oldPath=" + oldPath + ", newPath=" + newPath);
        }
        long oldFileLen = 0L;
        oldFileLen = super.doReplaceWriter(oldPath, newPath, nextWriter);
        return oldFileLen;
      }
    }

    // Mocked up server and regionserver services. Needed below.
    Server server =
        new DummyServer(CONF, ServerName.valueOf("hostname1.example.org", 1234, 1L).toString());
    RegionServerServices services = Mockito.mock(RegionServerServices.class);

    CONF.setLong("hbase.regionserver.hlog.sync.timeout", 10000);

    // OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL,
    // go ahead with test.
    FileSystem fs = FileSystem.get(CONF);
    Path rootDir = new Path(dir + getName());
    DodgyFSLog dodgyWAL1 = new DodgyFSLog(fs, rootDir, getName(), CONF);

    Path rootDir2 = new Path(dir + getName() + "2");
    final DodgyFSLog dodgyWAL2 = new DodgyFSLog(fs, rootDir2, getName() + "2", CONF);
    // Add a listener to force ringbuffer event handler sleep for a while
    dodgyWAL2.registerWALActionsListener(new DummyWALActionsListener());

    // I need a log roller running.
    LogRoller logRoller = new LogRoller(server, services);
    logRoller.addWAL(dodgyWAL1);
    logRoller.addWAL(dodgyWAL2);
    // There is no 'stop' once a logRoller is running.. it just dies.
    logRoller.start();
    // Now get a region and start adding in edits.
    HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME);
    final HRegion region = initHRegion(tableName, null, null, dodgyWAL1);
    byte[] bytes = Bytes.toBytes(getName());
    NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
    scopes.put(COLUMN_FAMILY_BYTES, 0);
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    try {
      Put put = new Put(bytes);
      put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
      WALKey key =
          new WALKey(
              region.getRegionInfo().getEncodedNameAsBytes(),
              htd.getTableName(),
              System.currentTimeMillis(),
              mvcc,
              scopes);
      WALEdit edit = new WALEdit();
      CellScanner CellScanner = put.cellScanner();
      assertTrue(CellScanner.advance());
      edit.add(CellScanner.current());

      LOG.info("SET throwing of exception on append");
      dodgyWAL1.throwException = true;
      // This append provokes a WAL roll request
      dodgyWAL1.append(region.getRegionInfo(), key, edit, true);
      boolean exception = false;
      try {
        dodgyWAL1.sync();
      } catch (Exception e) {
        exception = true;
      }
      assertTrue("Did not get sync exception", exception);

      // LogRoller call dodgyWAL1.rollWriter get FailedLogCloseException and
      // cause server abort.
      try {
        // wait LogRoller exit.
        Thread.sleep(50);
      } catch (InterruptedException e) {
        e.printStackTrace();
      }

      final CountDownLatch latch = new CountDownLatch(1);

      // make RingBufferEventHandler sleep 1s, so the following sync
      // endOfBatch=false
      key =
          new WALKey(
              region.getRegionInfo().getEncodedNameAsBytes(),
              TableName.valueOf("sleep"),
              System.currentTimeMillis(),
              mvcc,
              scopes);
      dodgyWAL2.append(region.getRegionInfo(), key, edit, true);

      Thread t =
          new Thread("Sync") {
            public void run() {
              try {
                dodgyWAL2.sync();
              } catch (IOException e) {
                LOG.info("In sync", e);
              }
              latch.countDown();
              LOG.info("Sync exiting");
            };
          };
      t.setDaemon(true);
      t.start();
      try {
        // make sure sync have published.
        Thread.sleep(100);
      } catch (InterruptedException e1) {
        e1.printStackTrace();
      }
      // make append throw DamagedWALException
      key =
          new WALKey(
              region.getRegionInfo().getEncodedNameAsBytes(),
              TableName.valueOf("DamagedWALException"),
              System.currentTimeMillis(),
              mvcc,
              scopes);
      dodgyWAL2.append(region.getRegionInfo(), key, edit, true);

      while (latch.getCount() > 0) {
        Threads.sleep(100);
      }
      assertTrue(server.isAborted());
    } finally {
      if (logRoller != null) {
        logRoller.close();
      }
      try {
        if (region != null) {
          region.close();
        }
        if (dodgyWAL1 != null) {
          dodgyWAL1.close();
        }
        if (dodgyWAL2 != null) {
          dodgyWAL2.close();
        }
      } catch (Exception e) {
        LOG.info("On way out", e);
      }
    }
  }
Exemplo n.º 12
0
 @Override
 public ServerName getServerName() {
   return ServerName.valueOf("regionserver,60020,000000");
 }