/** * Tests an on-the-fly RPC that was scheduled for the earlier RS on the same port for openRegion. * The region server should reject this RPC. (HBASE-9721) */ @Test public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception { Assert.assertTrue(getRS().getRegion(regionName).isAvailable()); ServerName sn = getRS().getServerName(); ServerName earlierServerName = ServerName.valueOf(sn.getHostname(), sn.getPort(), 1); try { CloseRegionRequest request = RequestConverter.buildCloseRegionRequest(earlierServerName, regionName); getRS().getRSRpcServices().closeRegion(null, request); Assert.fail("The closeRegion should have been rejected"); } catch (ServiceException se) { Assert.assertTrue(se.getCause() instanceof IOException); Assert.assertTrue( se.getCause().getMessage().contains("This RPC was intended for a different server")); } // actual close closeRegionNoZK(); try { AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(earlierServerName, hri, null, null); getRS().getRSRpcServices().openRegion(null, orr); Assert.fail("The openRegion should have been rejected"); } catch (ServiceException se) { Assert.assertTrue(se.getCause() instanceof IOException); Assert.assertTrue( se.getCause().getMessage().contains("This RPC was intended for a different server")); } finally { openRegion(HTU, getRS(), hri); } }
/** * @param count * @return Return <code>count</code> servernames. */ private static ServerName[] makeServerNames(final int count) { ServerName[] sns = new ServerName[count]; for (int i = 0; i < count; i++) { sns[i] = ServerName.valueOf("" + i + ".example.org", 60010, i); } return sns; }
/** * @param favoredNodesStr The String of favored nodes * @return the list of ServerName for the byte array of favored nodes. */ public static List<ServerName> getFavoredNodeList(String favoredNodesStr) { String[] favoredNodesArray = StringUtils.split(favoredNodesStr, ","); if (favoredNodesArray == null) return null; List<ServerName> serverList = new ArrayList<ServerName>(); for (String hostNameAndPort : favoredNodesArray) { serverList.add(ServerName.valueOf(hostNameAndPort, ServerName.NON_STARTCODE)); } return serverList; }
protected ServerAndLoad randomServer(final int numRegionsPerServer) { if (!this.serverQueue.isEmpty()) { ServerName sn = this.serverQueue.poll(); return new ServerAndLoad(sn, numRegionsPerServer); } String host = "srv" + rand.nextInt(Integer.MAX_VALUE); int port = rand.nextInt(60000); long startCode = rand.nextLong(); ServerName sn = ServerName.valueOf(host, port, startCode); return new ServerAndLoad(sn, numRegionsPerServer); }
@Test public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception { // this test is for when distributed log replay is enabled if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return; LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization"); HMaster master = UTIL.getMiniHBaseCluster().getMaster(); MasterFileSystem fs = master.getMasterFileSystem(); String failedRegion = "failedRegoin1"; String staleRegion = "staleRegion"; ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1"); ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1"); String walPath = "/hbase/data/.logs/" + inRecoveryServerName.getServerName() + "-splitting/test"; // Create a ZKW to use in the test ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); zkw.getRecoverableZooKeeper() .create( ZKSplitLog.getEncodedNodeName(zkw, walPath), new SplitLogTask.Owned(inRecoveryServerName, fs.getLogRecoveryMode()).toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); String staleRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, staleRegion); ZKUtil.createWithParents(zkw, staleRegionPath); String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, failedRegion); inRecoveringRegionPath = ZKUtil.joinZNode(inRecoveringRegionPath, inRecoveryServerName.getServerName()); ZKUtil.createWithParents(zkw, inRecoveringRegionPath); Set<ServerName> servers = new HashSet<ServerName>(); servers.add(previouselyFaildServerName); fs.removeStaleRecoveringRegionsFromZK(servers); // verification assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1); assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1); ZKUtil.deleteChildrenRecursively(zkw, zkw.recoveringRegionsZNode); ZKUtil.deleteChildrenRecursively(zkw, zkw.splitLogZNode); zkw.close(); }
@Override public void run() { while (!isStopped()) { try { List<HRegionInfo> regions = MetaScanner.listAllRegions(TEST_UTIL.getConfiguration(), connection, false); // select a random region HRegionInfo parent = regions.get(random.nextInt(regions.size())); if (parent == null || !TABLENAME.equals(parent.getTable())) { continue; } long startKey = 0, endKey = Long.MAX_VALUE; byte[] start = parent.getStartKey(); byte[] end = parent.getEndKey(); if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) { startKey = Bytes.toLong(parent.getStartKey()); } if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) { endKey = Bytes.toLong(parent.getEndKey()); } if (startKey == endKey) { continue; } long midKey = BigDecimal.valueOf(startKey) .add(BigDecimal.valueOf(endKey)) .divideToIntegralValue(BigDecimal.valueOf(2)) .longValue(); HRegionInfo splita = new HRegionInfo(TABLENAME, start, Bytes.toBytes(midKey)); HRegionInfo splitb = new HRegionInfo(TABLENAME, Bytes.toBytes(midKey), end); MetaTableAccessor.splitRegion( connection, parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0)); Threads.sleep(random.nextInt(200)); } catch (Throwable e) { ex = e; Assert.fail(StringUtils.stringifyException(e)); } } }
/** * Load the meta region state from the meta server ZNode. * * @param zkw * @param replicaId * @return regionstate * @throws KeeperException */ public static RegionState getMetaRegionState(ZooKeeperWatcher zkw, int replicaId) throws KeeperException { RegionState.State state = RegionState.State.OPEN; ServerName serverName = null; try { byte[] data = ZKUtil.getData(zkw, zkw.getZNodeForReplica(replicaId)); if (data != null && data.length > 0 && ProtobufUtil.isPBMagicPrefix(data)) { try { int prefixLen = ProtobufUtil.lengthOfPBMagic(); ZooKeeperProtos.MetaRegionServer rl = ZooKeeperProtos.MetaRegionServer.PARSER.parseFrom( data, prefixLen, data.length - prefixLen); if (rl.hasState()) { state = RegionState.State.convert(rl.getState()); } HBaseProtos.ServerName sn = rl.getServer(); serverName = ServerName.valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode()); } catch (InvalidProtocolBufferException e) { throw new DeserializationException("Unable to parse meta region location"); } } else { // old style of meta region location? serverName = ServerName.parseFrom(data); } } catch (DeserializationException e) { throw ZKUtil.convert(e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } if (serverName == null) { state = RegionState.State.OFFLINE; } return new RegionState( RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO, replicaId), state, serverName); }
/** Test client behavior w/o setting up a cluster. Mock up cluster emissions. */ @Category(SmallTests.class) public class TestClientNoCluster extends Configured implements Tool { private static final Log LOG = LogFactory.getLog(TestClientNoCluster.class); private Configuration conf; public static final ServerName META_SERVERNAME = ServerName.valueOf("meta.example.org", 60010, 12345); @Before public void setUp() throws Exception { this.conf = HBaseConfiguration.create(); // Run my HConnection overrides. Use my little HConnectionImplementation below which // allows me insert mocks and also use my Registry below rather than the default zk based // one so tests run faster and don't have zk dependency. this.conf.set("hbase.client.registry.impl", SimpleRegistry.class.getName()); } /** Simple cluster registry inserted in place of our usual zookeeper based one. */ static class SimpleRegistry implements Registry { final ServerName META_HOST = META_SERVERNAME; @Override public void init(HConnection connection) {} @Override public HRegionLocation getMetaRegionLocation() throws IOException { return new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, META_HOST); } @Override public String getClusterId() { return HConstants.CLUSTER_ID_DEFAULT; } @Override public boolean isTableOnlineState(TableName tableName, boolean enabled) throws IOException { return enabled; } @Override public int getCurrentNrHRS() throws IOException { return 1; } } /** * Remove the @Ignore to try out timeout and retry asettings * * @throws IOException */ @Ignore @Test public void testTimeoutAndRetries() throws IOException { Configuration localConfig = HBaseConfiguration.create(this.conf); // This override mocks up our exists/get call to throw a RegionServerStoppedException. localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName()); HTable table = new HTable(localConfig, TableName.META_TABLE_NAME); Throwable t = null; LOG.info("Start"); try { // An exists call turns into a get w/ a flag. table.exists(new Get(Bytes.toBytes("abc"))); } catch (SocketTimeoutException e) { // I expect this exception. LOG.info("Got expected exception", e); t = e; } catch (RetriesExhaustedException e) { // This is the old, unwanted behavior. If we get here FAIL!!! fail(); } finally { table.close(); } LOG.info("Stop"); assertTrue(t != null); } /** * Test that operation timeout prevails over rpc default timeout and retries, etc. * * @throws IOException */ @Test public void testRocTimeout() throws IOException { Configuration localConfig = HBaseConfiguration.create(this.conf); // This override mocks up our exists/get call to throw a RegionServerStoppedException. localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName()); int pause = 10; localConfig.setInt("hbase.client.pause", pause); localConfig.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10); // Set the operation timeout to be < the pause. Expectation is that after first pause, we will // fail out of the rpc because the rpc timeout will have been set to the operation tiemout // and it has expired. Otherwise, if this functionality is broke, all retries will be run -- // all ten of them -- and we'll get the RetriesExhaustedException exception. localConfig.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, pause - 1); HTable table = new HTable(localConfig, TableName.META_TABLE_NAME); Throwable t = null; try { // An exists call turns into a get w/ a flag. table.exists(new Get(Bytes.toBytes("abc"))); } catch (SocketTimeoutException e) { // I expect this exception. LOG.info("Got expected exception", e); t = e; } catch (RetriesExhaustedException e) { // This is the old, unwanted behavior. If we get here FAIL!!! fail(); } finally { table.close(); } assertTrue(t != null); } @Test public void testDoNotRetryMetaScanner() throws IOException { this.conf.set( "hbase.client.connection.impl", RegionServerStoppedOnScannerOpenConnection.class.getName()); MetaScanner.metaScan(this.conf, null); } @Test public void testDoNotRetryOnScanNext() throws IOException { this.conf.set( "hbase.client.connection.impl", RegionServerStoppedOnScannerOpenConnection.class.getName()); // Go against meta else we will try to find first region for the table on construction which // means we'll have to do a bunch more mocking. Tests that go against meta only should be // good for a bit of testing. HTable table = new HTable(this.conf, TableName.META_TABLE_NAME); ResultScanner scanner = table.getScanner(HConstants.CATALOG_FAMILY); try { Result result = null; while ((result = scanner.next()) != null) { LOG.info(result); } } finally { scanner.close(); table.close(); } } @Test public void testRegionServerStoppedOnScannerOpen() throws IOException { this.conf.set( "hbase.client.connection.impl", RegionServerStoppedOnScannerOpenConnection.class.getName()); // Go against meta else we will try to find first region for the table on construction which // means we'll have to do a bunch more mocking. Tests that go against meta only should be // good for a bit of testing. HTable table = new HTable(this.conf, TableName.META_TABLE_NAME); ResultScanner scanner = table.getScanner(HConstants.CATALOG_FAMILY); try { Result result = null; while ((result = scanner.next()) != null) { LOG.info(result); } } finally { scanner.close(); table.close(); } } /** Override to shutdown going to zookeeper for cluster id and meta location. */ static class ScanOpenNextThenExceptionThenRecoverConnection extends HConnectionManager.HConnectionImplementation { final ClientService.BlockingInterface stub; ScanOpenNextThenExceptionThenRecoverConnection( Configuration conf, boolean managed, ExecutorService pool) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when( stub.scan((RpcController) Mockito.any(), (ClientProtos.ScanRequest) Mockito.any())) .thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()) .thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))) .thenReturn( ClientProtos.ScanResponse.newBuilder() .setScannerId(sid) .setMoreResults(false) .build()); } catch (ServiceException e) { throw new IOException(e); } } @Override public BlockingInterface getClient(ServerName sn) throws IOException { return this.stub; } } /** Override to shutdown going to zookeeper for cluster id and meta location. */ static class RegionServerStoppedOnScannerOpenConnection extends HConnectionManager.HConnectionImplementation { final ClientService.BlockingInterface stub; RegionServerStoppedOnScannerOpenConnection( Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when( stub.scan((RpcController) Mockito.any(), (ClientProtos.ScanRequest) Mockito.any())) .thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()) .thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))) .thenReturn( ClientProtos.ScanResponse.newBuilder() .setScannerId(sid) .setMoreResults(false) .build()); } catch (ServiceException e) { throw new IOException(e); } } @Override public BlockingInterface getClient(ServerName sn) throws IOException { return this.stub; } } /** Override to check we are setting rpc timeout right. */ static class RpcTimeoutConnection extends HConnectionManager.HConnectionImplementation { final ClientService.BlockingInterface stub; RpcTimeoutConnection(Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException { super(conf, managed); // Mock up my stub so an exists call -- which turns into a get -- throws an exception this.stub = Mockito.mock(ClientService.BlockingInterface.class); try { Mockito.when( stub.get((RpcController) Mockito.any(), (ClientProtos.GetRequest) Mockito.any())) .thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))); } catch (ServiceException e) { throw new IOException(e); } } @Override public BlockingInterface getClient(ServerName sn) throws IOException { return this.stub; } } /** Fake many regionservers and many regions on a connection implementation. */ static class ManyServersManyRegionsConnection extends HConnectionManager.HConnectionImplementation { // All access should be synchronized final Map<ServerName, ClientService.BlockingInterface> serversByClient; /** Map of faked-up rows of a 'meta table'. */ final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta; final AtomicLong sequenceids = new AtomicLong(0); private final Configuration conf; ManyServersManyRegionsConnection( Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException { super(conf, managed, pool, user); int serverCount = conf.getInt("hbase.test.servers", 10); this.serversByClient = new HashMap<ServerName, ClientService.BlockingInterface>(serverCount); this.meta = makeMeta( Bytes.toBytes(conf.get("hbase.test.tablename", Bytes.toString(BIG_USER_TABLE))), conf.getInt("hbase.test.regions", 100), conf.getLong("hbase.test.namespace.span", 1000), serverCount); this.conf = conf; } @Override public ClientService.BlockingInterface getClient(ServerName sn) throws IOException { // if (!sn.toString().startsWith("meta")) LOG.info(sn); ClientService.BlockingInterface stub = null; synchronized (this.serversByClient) { stub = this.serversByClient.get(sn); if (stub == null) { stub = new FakeServer(this.conf, meta, sequenceids); this.serversByClient.put(sn, stub); } } return stub; } } static MultiResponse doMultiResponse( final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta, final AtomicLong sequenceids, final MultiRequest request) { // Make a response to match the request. Act like there were no failures. ClientProtos.MultiResponse.Builder builder = ClientProtos.MultiResponse.newBuilder(); // Per Region. RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder(); ResultOrException.Builder roeBuilder = ResultOrException.newBuilder(); for (RegionAction regionAction : request.getRegionActionList()) { regionActionResultBuilder.clear(); // Per Action in a Region. for (ClientProtos.Action action : regionAction.getActionList()) { roeBuilder.clear(); // Return empty Result and proper index as result. roeBuilder.setResult(ClientProtos.Result.getDefaultInstance()); roeBuilder.setIndex(action.getIndex()); regionActionResultBuilder.addResultOrException(roeBuilder.build()); } builder.addRegionActionResult(regionActionResultBuilder.build()); } return builder.build(); } /** * Fake 'server'. Implements the ClientService responding as though it were a 'server' (presumes a * new ClientService.BlockingInterface made per server). */ static class FakeServer implements ClientService.BlockingInterface { private AtomicInteger multiInvocationsCount = new AtomicInteger(0); private final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta; private final AtomicLong sequenceids; private final long multiPause; private final int tooManyMultiRequests; FakeServer( final Configuration c, final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta, final AtomicLong sequenceids) { this.meta = meta; this.sequenceids = sequenceids; // Pause to simulate the server taking time applying the edits. This will drive up the // number of threads used over in client. this.multiPause = c.getLong("hbase.test.multi.pause.when.done", 0); this.tooManyMultiRequests = c.getInt("hbase.test.multi.too.many", 3); } @Override public GetResponse get(RpcController controller, GetRequest request) throws ServiceException { boolean metaRegion = isMetaRegion(request.getRegion().getValue().toByteArray(), request.getRegion().getType()); if (!metaRegion) { return doGetResponse(request); } return doMetaGetResponse(meta, request); } private GetResponse doGetResponse(GetRequest request) { ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder(); ByteString row = request.getGet().getRow(); resultBuilder.addCell(getStartCode(row)); GetResponse.Builder builder = GetResponse.newBuilder(); builder.setResult(resultBuilder.build()); return builder.build(); } @Override public MutateResponse mutate(RpcController controller, MutateRequest request) throws ServiceException { throw new NotImplementedException(); } @Override public ScanResponse scan(RpcController controller, ScanRequest request) throws ServiceException { // Presume it is a scan of meta for now. Not all scans provide a region spec expecting // the server to keep reference by scannerid. TODO. return doMetaScanResponse(meta, sequenceids, request); } @Override public BulkLoadHFileResponse bulkLoadHFile( RpcController controller, BulkLoadHFileRequest request) throws ServiceException { throw new NotImplementedException(); } @Override public CoprocessorServiceResponse execService( RpcController controller, CoprocessorServiceRequest request) throws ServiceException { throw new NotImplementedException(); } @Override public MultiResponse multi(RpcController controller, MultiRequest request) throws ServiceException { int concurrentInvocations = this.multiInvocationsCount.incrementAndGet(); try { if (concurrentInvocations >= tooManyMultiRequests) { throw new ServiceException( new RegionTooBusyException("concurrentInvocations=" + concurrentInvocations)); } Threads.sleep(multiPause); return doMultiResponse(meta, sequenceids, request); } finally { this.multiInvocationsCount.decrementAndGet(); } } } static ScanResponse doMetaScanResponse( final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta, final AtomicLong sequenceids, final ScanRequest request) { ScanResponse.Builder builder = ScanResponse.newBuilder(); int max = request.getNumberOfRows(); int count = 0; Map<byte[], Pair<HRegionInfo, ServerName>> tail = request.hasScan() ? meta.tailMap(request.getScan().getStartRow().toByteArray()) : meta; ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder(); for (Map.Entry<byte[], Pair<HRegionInfo, ServerName>> e : tail.entrySet()) { // Can be 0 on open of a scanner -- i.e. rpc to setup scannerid only. if (max <= 0) break; if (++count > max) break; HRegionInfo hri = e.getValue().getFirst(); ByteString row = HBaseZeroCopyByteString.wrap(hri.getRegionName()); resultBuilder.clear(); resultBuilder.addCell(getRegionInfo(row, hri)); resultBuilder.addCell(getServer(row, e.getValue().getSecond())); resultBuilder.addCell(getStartCode(row)); builder.addResults(resultBuilder.build()); // Set more to false if we are on the last region in table. if (hri.getEndKey().length <= 0) builder.setMoreResults(false); else builder.setMoreResults(true); } // If no scannerid, set one. builder.setScannerId( request.hasScannerId() ? request.getScannerId() : sequenceids.incrementAndGet()); return builder.build(); } static GetResponse doMetaGetResponse( final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta, final GetRequest request) { ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder(); ByteString row = request.getGet().getRow(); Pair<HRegionInfo, ServerName> p = meta.get(row.toByteArray()); if (p == null) { if (request.getGet().getClosestRowBefore()) { byte[] bytes = row.toByteArray(); SortedMap<byte[], Pair<HRegionInfo, ServerName>> head = bytes != null ? meta.headMap(bytes) : meta; p = head == null ? null : head.get(head.lastKey()); } } if (p != null) { resultBuilder.addCell(getRegionInfo(row, p.getFirst())); resultBuilder.addCell(getServer(row, p.getSecond())); } resultBuilder.addCell(getStartCode(row)); GetResponse.Builder builder = GetResponse.newBuilder(); builder.setResult(resultBuilder.build()); return builder.build(); } /** * @param name region name or encoded region name. * @param type * @return True if we are dealing with a hbase:meta region. */ static boolean isMetaRegion(final byte[] name, final RegionSpecifierType type) { switch (type) { case REGION_NAME: return Bytes.equals(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), name); case ENCODED_REGION_NAME: return Bytes.equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), name); default: throw new UnsupportedOperationException(); } } private static final ByteString CATALOG_FAMILY_BYTESTRING = HBaseZeroCopyByteString.wrap(HConstants.CATALOG_FAMILY); private static final ByteString REGIONINFO_QUALIFIER_BYTESTRING = HBaseZeroCopyByteString.wrap(HConstants.REGIONINFO_QUALIFIER); private static final ByteString SERVER_QUALIFIER_BYTESTRING = HBaseZeroCopyByteString.wrap(HConstants.SERVER_QUALIFIER); static CellProtos.Cell.Builder getBaseCellBuilder(final ByteString row) { CellProtos.Cell.Builder cellBuilder = CellProtos.Cell.newBuilder(); cellBuilder.setRow(row); cellBuilder.setFamily(CATALOG_FAMILY_BYTESTRING); cellBuilder.setTimestamp(System.currentTimeMillis()); return cellBuilder; } static CellProtos.Cell getRegionInfo(final ByteString row, final HRegionInfo hri) { CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row); cellBuilder.setQualifier(REGIONINFO_QUALIFIER_BYTESTRING); cellBuilder.setValue(HBaseZeroCopyByteString.wrap(hri.toByteArray())); return cellBuilder.build(); } static CellProtos.Cell getServer(final ByteString row, final ServerName sn) { CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row); cellBuilder.setQualifier(SERVER_QUALIFIER_BYTESTRING); cellBuilder.setValue(ByteString.copyFromUtf8(sn.getHostAndPort())); return cellBuilder.build(); } static CellProtos.Cell getStartCode(final ByteString row) { CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row); cellBuilder.setQualifier(HBaseZeroCopyByteString.wrap(HConstants.STARTCODE_QUALIFIER)); // TODO: cellBuilder.setValue( HBaseZeroCopyByteString.wrap(Bytes.toBytes(META_SERVERNAME.getStartcode()))); return cellBuilder.build(); } private static final byte[] BIG_USER_TABLE = Bytes.toBytes("t"); /** * Format passed integer. Zero-pad. Copied from hbase-server PE class and small amendment. Make * them share. * * @param number * @return Returns zero-prefixed 10-byte wide decimal version of passed number (Does absolute in * case number is negative). */ private static byte[] format(final long number) { byte[] b = new byte[10]; long d = number; for (int i = b.length - 1; i >= 0; i--) { b[i] = (byte) ((d % 10) + '0'); d /= 10; } return b; } /** * @param count * @param namespaceSpan * @return <code>count</code> regions */ private static HRegionInfo[] makeHRegionInfos( final byte[] tableName, final int count, final long namespaceSpan) { byte[] startKey = HConstants.EMPTY_BYTE_ARRAY; byte[] endKey = HConstants.EMPTY_BYTE_ARRAY; long interval = namespaceSpan / count; HRegionInfo[] hris = new HRegionInfo[count]; for (int i = 0; i < count; i++) { if (i == 0) { endKey = format(interval); } else { startKey = endKey; if (i == count - 1) endKey = HConstants.EMPTY_BYTE_ARRAY; else endKey = format((i + 1) * interval); } hris[i] = new HRegionInfo(TableName.valueOf(tableName), startKey, endKey); } return hris; } /** * @param count * @return Return <code>count</code> servernames. */ private static ServerName[] makeServerNames(final int count) { ServerName[] sns = new ServerName[count]; for (int i = 0; i < count; i++) { sns[i] = ServerName.valueOf("" + i + ".example.org", 60010, i); } return sns; } /** Comparator for meta row keys. */ private static class MetaRowsComparator implements Comparator<byte[]> { private final KeyValue.KVComparator delegate = new KeyValue.MetaComparator(); @Override public int compare(byte[] left, byte[] right) { return delegate.compareRows(left, 0, left.length, right, 0, right.length); } } /** * Create up a map that is keyed by meta row name and whose value is the HRegionInfo and * ServerName to return for this row. * * @return Map with faked hbase:meta content in it. */ static SortedMap<byte[], Pair<HRegionInfo, ServerName>> makeMeta( final byte[] tableName, final int regionCount, final long namespaceSpan, final int serverCount) { // I need a comparator for meta rows so we sort properly. SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta = new ConcurrentSkipListMap<byte[], Pair<HRegionInfo, ServerName>>(new MetaRowsComparator()); HRegionInfo[] hris = makeHRegionInfos(tableName, regionCount, namespaceSpan); ServerName[] serverNames = makeServerNames(serverCount); int per = regionCount / serverCount; int count = 0; for (HRegionInfo hri : hris) { Pair<HRegionInfo, ServerName> p = new Pair<HRegionInfo, ServerName>(hri, serverNames[count++ / per]); meta.put(hri.getRegionName(), p); } return meta; } /** * Code for each 'client' to run. * * @param id * @param c * @param sharedConnection * @throws IOException */ static void cycle(int id, final Configuration c, final HConnection sharedConnection) throws IOException { HTableInterface table = sharedConnection.getTable(BIG_USER_TABLE); table.setAutoFlushTo(false); long namespaceSpan = c.getLong("hbase.test.namespace.span", 1000000); long startTime = System.currentTimeMillis(); final int printInterval = 100000; Random rd = new Random(id); boolean get = c.getBoolean("hbase.test.do.gets", false); try { Stopwatch stopWatch = new Stopwatch(); stopWatch.start(); for (int i = 0; i < namespaceSpan; i++) { byte[] b = format(rd.nextLong()); if (get) { Get g = new Get(b); table.get(g); } else { Put p = new Put(b); p.add(HConstants.CATALOG_FAMILY, b, b); table.put(p); } if (i % printInterval == 0) { LOG.info("Put " + printInterval + "/" + stopWatch.elapsedMillis()); stopWatch.reset(); stopWatch.start(); } } LOG.info( "Finished a cycle putting " + namespaceSpan + " in " + (System.currentTimeMillis() - startTime) + "ms"); } finally { table.close(); } } @Override public int run(String[] arg0) throws Exception { int errCode = 0; // TODO: Make command options. // How many servers to fake. final int servers = 1; // How many regions to put on the faked servers. final int regions = 100000; // How many 'keys' in the faked regions. final long namespaceSpan = 50000000; // How long to take to pause after doing a put; make this long if you want to fake a struggling // server. final long multiPause = 0; // Check args make basic sense. if ((namespaceSpan < regions) || (regions < servers)) { throw new IllegalArgumentException( "namespaceSpan=" + namespaceSpan + " must be > regions=" + regions + " which must be > servers=" + servers); } // Set my many servers and many regions faking connection in place. getConf().set("hbase.client.connection.impl", ManyServersManyRegionsConnection.class.getName()); // Use simple kv registry rather than zk getConf().set("hbase.client.registry.impl", SimpleRegistry.class.getName()); // When to report fails. Default is we report the 10th. This means we'll see log everytime // an exception is thrown -- usually RegionTooBusyException when we have more than // hbase.test.multi.too.many requests outstanding at any time. getConf().setInt("hbase.client.start.log.errors.counter", 0); // Ugly but this is only way to pass in configs.into ManyServersManyRegionsConnection class. getConf().setInt("hbase.test.regions", regions); getConf().setLong("hbase.test.namespace.span", namespaceSpan); getConf().setLong("hbase.test.servers", servers); getConf().set("hbase.test.tablename", Bytes.toString(BIG_USER_TABLE)); getConf().setLong("hbase.test.multi.pause.when.done", multiPause); // Let there be ten outstanding requests at a time before we throw RegionBusyException. getConf().setInt("hbase.test.multi.too.many", 10); final int clients = 2; // Have them all share the same connection so they all share the same instance of // ManyServersManyRegionsConnection so I can keep an eye on how many requests by server. final ExecutorService pool = Executors.newCachedThreadPool(Threads.getNamedThreadFactory("p")); // Executors.newFixedThreadPool(servers * 10, Threads.getNamedThreadFactory("p")); // Share a connection so I can keep counts in the 'server' on concurrency. final HConnection sharedConnection = HConnectionManager.createConnection(getConf() /*, pool*/); try { Thread[] ts = new Thread[clients]; for (int j = 0; j < ts.length; j++) { final int id = j; ts[j] = new Thread("" + j) { final Configuration c = getConf(); @Override public void run() { try { cycle(id, c, sharedConnection); } catch (IOException e) { e.printStackTrace(); } } }; ts[j].start(); } for (int j = 0; j < ts.length; j++) { ts[j].join(); } } finally { sharedConnection.close(); } return errCode; } /** * Run a client instance against a faked up server. * * @param args TODO * @throws Exception */ public static void main(String[] args) throws Exception { System.exit(ToolRunner.run(HBaseConfiguration.create(), new TestClientNoCluster(), args)); } }
/** * Generate the assignment plan for the existing table * * @param tableName * @param assignmentSnapshot * @param regionLocalityMap * @param plan * @param munkresForSecondaryAndTertiary if set on true the assignment plan for the tertiary and * secondary will be generated with Munkres algorithm, otherwise will be generated using * placeSecondaryAndTertiaryRS * @throws IOException */ private void genAssignmentPlan( TableName tableName, SnapshotOfRegionAssignmentFromMeta assignmentSnapshot, Map<String, Map<String, Float>> regionLocalityMap, FavoredNodesPlan plan, boolean munkresForSecondaryAndTertiary) throws IOException { // Get the all the regions for the current table List<HRegionInfo> regions = assignmentSnapshot.getTableToRegionMap().get(tableName); int numRegions = regions.size(); // Get the current assignment map Map<HRegionInfo, ServerName> currentAssignmentMap = assignmentSnapshot.getRegionToRegionServerMap(); // Get the all the region servers List<ServerName> servers = new ArrayList<ServerName>(); try (Admin admin = this.connection.getAdmin()) { servers.addAll(admin.getClusterStatus().getServers()); } LOG.info( "Start to generate assignment plan for " + numRegions + " regions from table " + tableName + " with " + servers.size() + " region servers"); int slotsPerServer = (int) Math.ceil((float) numRegions / servers.size()); int regionSlots = slotsPerServer * servers.size(); // Compute the primary, secondary and tertiary costs for each region/server // pair. These costs are based only on node locality and rack locality, and // will be modified later. float[][] primaryCost = new float[numRegions][regionSlots]; float[][] secondaryCost = new float[numRegions][regionSlots]; float[][] tertiaryCost = new float[numRegions][regionSlots]; if (this.enforceLocality && regionLocalityMap != null) { // Transform the locality mapping into a 2D array, assuming that any // unspecified locality value is 0. float[][] localityPerServer = new float[numRegions][regionSlots]; for (int i = 0; i < numRegions; i++) { Map<String, Float> serverLocalityMap = regionLocalityMap.get(regions.get(i).getEncodedName()); if (serverLocalityMap == null) { continue; } for (int j = 0; j < servers.size(); j++) { String serverName = servers.get(j).getHostname(); if (serverName == null) { continue; } Float locality = serverLocalityMap.get(serverName); if (locality == null) { continue; } for (int k = 0; k < slotsPerServer; k++) { // If we can't find the locality of a region to a server, which occurs // because locality is only reported for servers which have some // blocks of a region local, then the locality for that pair is 0. localityPerServer[i][j * slotsPerServer + k] = locality.floatValue(); } } } // Compute the total rack locality for each region in each rack. The total // rack locality is the sum of the localities of a region on all servers in // a rack. Map<String, Map<HRegionInfo, Float>> rackRegionLocality = new HashMap<String, Map<HRegionInfo, Float>>(); for (int i = 0; i < numRegions; i++) { HRegionInfo region = regions.get(i); for (int j = 0; j < regionSlots; j += slotsPerServer) { String rack = rackManager.getRack(servers.get(j / slotsPerServer)); Map<HRegionInfo, Float> rackLocality = rackRegionLocality.get(rack); if (rackLocality == null) { rackLocality = new HashMap<HRegionInfo, Float>(); rackRegionLocality.put(rack, rackLocality); } Float localityObj = rackLocality.get(region); float locality = localityObj == null ? 0 : localityObj.floatValue(); locality += localityPerServer[i][j]; rackLocality.put(region, locality); } } for (int i = 0; i < numRegions; i++) { for (int j = 0; j < regionSlots; j++) { String rack = rackManager.getRack(servers.get(j / slotsPerServer)); Float totalRackLocalityObj = rackRegionLocality.get(rack).get(regions.get(i)); float totalRackLocality = totalRackLocalityObj == null ? 0 : totalRackLocalityObj.floatValue(); // Primary cost aims to favor servers with high node locality and low // rack locality, so that secondaries and tertiaries can be chosen for // nodes with high rack locality. This might give primaries with // slightly less locality at first compared to a cost which only // considers the node locality, but should be better in the long run. primaryCost[i][j] = 1 - (2 * localityPerServer[i][j] - totalRackLocality); // Secondary cost aims to favor servers with high node locality and high // rack locality since the tertiary will be chosen from the same rack as // the secondary. This could be negative, but that is okay. secondaryCost[i][j] = 2 - (localityPerServer[i][j] + totalRackLocality); // Tertiary cost is only concerned with the node locality. It will later // be restricted to only hosts on the same rack as the secondary. tertiaryCost[i][j] = 1 - localityPerServer[i][j]; } } } if (this.enforceMinAssignmentMove && currentAssignmentMap != null) { // We want to minimize the number of regions which move as the result of a // new assignment. Therefore, slightly penalize any placement which is for // a host that is not currently serving the region. for (int i = 0; i < numRegions; i++) { for (int j = 0; j < servers.size(); j++) { ServerName currentAddress = currentAssignmentMap.get(regions.get(i)); if (currentAddress != null && !currentAddress.equals(servers.get(j))) { for (int k = 0; k < slotsPerServer; k++) { primaryCost[i][j * slotsPerServer + k] += NOT_CURRENT_HOST_PENALTY; } } } } } // Artificially increase cost of last slot of each server to evenly // distribute the slop, otherwise there will be a few servers with too few // regions and many servers with the max number of regions. for (int i = 0; i < numRegions; i++) { for (int j = 0; j < regionSlots; j += slotsPerServer) { primaryCost[i][j] += LAST_SLOT_COST_PENALTY; secondaryCost[i][j] += LAST_SLOT_COST_PENALTY; tertiaryCost[i][j] += LAST_SLOT_COST_PENALTY; } } RandomizedMatrix randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots); primaryCost = randomizedMatrix.transform(primaryCost); int[] primaryAssignment = new MunkresAssignment(primaryCost).solve(); primaryAssignment = randomizedMatrix.invertIndices(primaryAssignment); // Modify the secondary and tertiary costs for each region/server pair to // prevent a region from being assigned to the same rack for both primary // and either one of secondary or tertiary. for (int i = 0; i < numRegions; i++) { int slot = primaryAssignment[i]; String rack = rackManager.getRack(servers.get(slot / slotsPerServer)); for (int k = 0; k < servers.size(); k++) { if (!rackManager.getRack(servers.get(k)).equals(rack)) { continue; } if (k == slot / slotsPerServer) { // Same node, do not place secondary or tertiary here ever. for (int m = 0; m < slotsPerServer; m++) { secondaryCost[i][k * slotsPerServer + m] = MAX_COST; tertiaryCost[i][k * slotsPerServer + m] = MAX_COST; } } else { // Same rack, do not place secondary or tertiary here if possible. for (int m = 0; m < slotsPerServer; m++) { secondaryCost[i][k * slotsPerServer + m] = AVOID_COST; tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST; } } } } if (munkresForSecondaryAndTertiary) { randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots); secondaryCost = randomizedMatrix.transform(secondaryCost); int[] secondaryAssignment = new MunkresAssignment(secondaryCost).solve(); secondaryAssignment = randomizedMatrix.invertIndices(secondaryAssignment); // Modify the tertiary costs for each region/server pair to ensure that a // region is assigned to a tertiary server on the same rack as its secondary // server, but not the same server in that rack. for (int i = 0; i < numRegions; i++) { int slot = secondaryAssignment[i]; String rack = rackManager.getRack(servers.get(slot / slotsPerServer)); for (int k = 0; k < servers.size(); k++) { if (k == slot / slotsPerServer) { // Same node, do not place tertiary here ever. for (int m = 0; m < slotsPerServer; m++) { tertiaryCost[i][k * slotsPerServer + m] = MAX_COST; } } else { if (rackManager.getRack(servers.get(k)).equals(rack)) { continue; } // Different rack, do not place tertiary here if possible. for (int m = 0; m < slotsPerServer; m++) { tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST; } } } } randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots); tertiaryCost = randomizedMatrix.transform(tertiaryCost); int[] tertiaryAssignment = new MunkresAssignment(tertiaryCost).solve(); tertiaryAssignment = randomizedMatrix.invertIndices(tertiaryAssignment); for (int i = 0; i < numRegions; i++) { List<ServerName> favoredServers = new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); ServerName s = servers.get(primaryAssignment[i] / slotsPerServer); favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); s = servers.get(secondaryAssignment[i] / slotsPerServer); favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); s = servers.get(tertiaryAssignment[i] / slotsPerServer); favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); // Update the assignment plan plan.updateAssignmentPlan(regions.get(i), favoredServers); } LOG.info( "Generated the assignment plan for " + numRegions + " regions from table " + tableName + " with " + servers.size() + " region servers"); LOG.info("Assignment plan for secondary and tertiary generated " + "using MunkresAssignment"); } else { Map<HRegionInfo, ServerName> primaryRSMap = new HashMap<HRegionInfo, ServerName>(); for (int i = 0; i < numRegions; i++) { primaryRSMap.put(regions.get(i), servers.get(primaryAssignment[i] / slotsPerServer)); } FavoredNodeAssignmentHelper favoredNodeHelper = new FavoredNodeAssignmentHelper(servers, conf); favoredNodeHelper.initialize(); Map<HRegionInfo, ServerName[]> secondaryAndTertiaryMap = favoredNodeHelper.placeSecondaryAndTertiaryWithRestrictions(primaryRSMap); for (int i = 0; i < numRegions; i++) { List<ServerName> favoredServers = new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); HRegionInfo currentRegion = regions.get(i); ServerName s = primaryRSMap.get(currentRegion); favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); ServerName[] secondaryAndTertiary = secondaryAndTertiaryMap.get(currentRegion); s = secondaryAndTertiary[0]; favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); s = secondaryAndTertiary[1]; favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); // Update the assignment plan plan.updateAssignmentPlan(regions.get(i), favoredServers); } LOG.info( "Generated the assignment plan for " + numRegions + " regions from table " + tableName + " with " + servers.size() + " region servers"); LOG.info( "Assignment plan for secondary and tertiary generated " + "using placeSecondaryAndTertiaryWithRestrictions method"); } }
@Override public ServerName getServerName() { return ServerName.valueOf(this.serverName); }
/** * Reproduce locking up that happens when there's no further syncs after append fails, and causing * an isolated sync then infinite wait. See HBASE-16960. If below is broken, we will see this test * timeout because it is locked up. * * <p>Steps for reproduce:<br> * 1. Trigger server abort through dodgyWAL1<br> * 2. Add a {@link DummyWALActionsListener} to dodgyWAL2 to cause ringbuffer event handler thread * sleep for a while thus keeping {@code endOfBatch} false<br> * 3. Publish a sync then an append which will throw exception, check whether the sync could * return */ @Test(timeout = 20000) public void testLockup16960() throws IOException { // A WAL that we can have throw exceptions when a flag is set. class DodgyFSLog extends FSHLog { // Set this when want the WAL to start throwing exceptions. volatile boolean throwException = false; public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration conf) throws IOException { super(fs, root, logDir, conf); } @Override protected Writer createWriterInstance(Path path) throws IOException { final Writer w = super.createWriterInstance(path); return new Writer() { @Override public void close() throws IOException { w.close(); } @Override public void sync() throws IOException { if (throwException) { throw new IOException("FAKE! Failed to replace a bad datanode...SYNC"); } w.sync(); } @Override public void append(Entry entry) throws IOException { if (throwException) { throw new IOException("FAKE! Failed to replace a bad datanode...APPEND"); } w.append(entry); } @Override public long getLength() { return w.getLength(); } }; } @Override protected long doReplaceWriter(Path oldPath, Path newPath, Writer nextWriter) throws IOException { if (throwException) { throw new FailedLogCloseException("oldPath=" + oldPath + ", newPath=" + newPath); } long oldFileLen = 0L; oldFileLen = super.doReplaceWriter(oldPath, newPath, nextWriter); return oldFileLen; } } // Mocked up server and regionserver services. Needed below. Server server = new DummyServer(CONF, ServerName.valueOf("hostname1.example.org", 1234, 1L).toString()); RegionServerServices services = Mockito.mock(RegionServerServices.class); CONF.setLong("hbase.regionserver.hlog.sync.timeout", 10000); // OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL, // go ahead with test. FileSystem fs = FileSystem.get(CONF); Path rootDir = new Path(dir + getName()); DodgyFSLog dodgyWAL1 = new DodgyFSLog(fs, rootDir, getName(), CONF); Path rootDir2 = new Path(dir + getName() + "2"); final DodgyFSLog dodgyWAL2 = new DodgyFSLog(fs, rootDir2, getName() + "2", CONF); // Add a listener to force ringbuffer event handler sleep for a while dodgyWAL2.registerWALActionsListener(new DummyWALActionsListener()); // I need a log roller running. LogRoller logRoller = new LogRoller(server, services); logRoller.addWAL(dodgyWAL1); logRoller.addWAL(dodgyWAL2); // There is no 'stop' once a logRoller is running.. it just dies. logRoller.start(); // Now get a region and start adding in edits. HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); final HRegion region = initHRegion(tableName, null, null, dodgyWAL1); byte[] bytes = Bytes.toBytes(getName()); NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR); scopes.put(COLUMN_FAMILY_BYTES, 0); MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); try { Put put = new Put(bytes); put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes); WALKey key = new WALKey( region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(), mvcc, scopes); WALEdit edit = new WALEdit(); CellScanner CellScanner = put.cellScanner(); assertTrue(CellScanner.advance()); edit.add(CellScanner.current()); LOG.info("SET throwing of exception on append"); dodgyWAL1.throwException = true; // This append provokes a WAL roll request dodgyWAL1.append(region.getRegionInfo(), key, edit, true); boolean exception = false; try { dodgyWAL1.sync(); } catch (Exception e) { exception = true; } assertTrue("Did not get sync exception", exception); // LogRoller call dodgyWAL1.rollWriter get FailedLogCloseException and // cause server abort. try { // wait LogRoller exit. Thread.sleep(50); } catch (InterruptedException e) { e.printStackTrace(); } final CountDownLatch latch = new CountDownLatch(1); // make RingBufferEventHandler sleep 1s, so the following sync // endOfBatch=false key = new WALKey( region.getRegionInfo().getEncodedNameAsBytes(), TableName.valueOf("sleep"), System.currentTimeMillis(), mvcc, scopes); dodgyWAL2.append(region.getRegionInfo(), key, edit, true); Thread t = new Thread("Sync") { public void run() { try { dodgyWAL2.sync(); } catch (IOException e) { LOG.info("In sync", e); } latch.countDown(); LOG.info("Sync exiting"); }; }; t.setDaemon(true); t.start(); try { // make sure sync have published. Thread.sleep(100); } catch (InterruptedException e1) { e1.printStackTrace(); } // make append throw DamagedWALException key = new WALKey( region.getRegionInfo().getEncodedNameAsBytes(), TableName.valueOf("DamagedWALException"), System.currentTimeMillis(), mvcc, scopes); dodgyWAL2.append(region.getRegionInfo(), key, edit, true); while (latch.getCount() > 0) { Threads.sleep(100); } assertTrue(server.isAborted()); } finally { if (logRoller != null) { logRoller.close(); } try { if (region != null) { region.close(); } if (dodgyWAL1 != null) { dodgyWAL1.close(); } if (dodgyWAL2 != null) { dodgyWAL2.close(); } } catch (Exception e) { LOG.info("On way out", e); } } }
@Override public ServerName getServerName() { return ServerName.valueOf("regionserver,60020,000000"); }