/**
* Tests an on-the-fly RPC that was scheduled for the earlier RS on the same port for openRegion.
* The region server should reject this RPC. (HBASE-9721)
*/
@Test
public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception {
Assert.assertTrue(getRS().getRegion(regionName).isAvailable());
ServerName sn = getRS().getServerName();
ServerName earlierServerName = ServerName.valueOf(sn.getHostname(), sn.getPort(), 1);
try {
CloseRegionRequest request =
RequestConverter.buildCloseRegionRequest(earlierServerName, regionName);
getRS().getRSRpcServices().closeRegion(null, request);
Assert.fail("The closeRegion should have been rejected");
} catch (ServiceException se) {
Assert.assertTrue(se.getCause() instanceof IOException);
Assert.assertTrue(
se.getCause().getMessage().contains("This RPC was intended for a different server"));
}
// actual close
closeRegionNoZK();
try {
AdminProtos.OpenRegionRequest orr =
RequestConverter.buildOpenRegionRequest(earlierServerName, hri, null, null);
getRS().getRSRpcServices().openRegion(null, orr);
Assert.fail("The openRegion should have been rejected");
} catch (ServiceException se) {
Assert.assertTrue(se.getCause() instanceof IOException);
Assert.assertTrue(
se.getCause().getMessage().contains("This RPC was intended for a different server"));
} finally {
openRegion(HTU, getRS(), hri);
}
}
/**
* @param count
* @return Return <code>count</code> servernames.
*/
private static ServerName[] makeServerNames(final int count) {
ServerName[] sns = new ServerName[count];
for (int i = 0; i < count; i++) {
sns[i] = ServerName.valueOf("" + i + ".example.org", 60010, i);
}
return sns;
}
/**
* @param favoredNodesStr The String of favored nodes
* @return the list of ServerName for the byte array of favored nodes.
*/
public static List<ServerName> getFavoredNodeList(String favoredNodesStr) {
String[] favoredNodesArray = StringUtils.split(favoredNodesStr, ",");
if (favoredNodesArray == null) return null;
List<ServerName> serverList = new ArrayList<ServerName>();
for (String hostNameAndPort : favoredNodesArray) {
serverList.add(ServerName.valueOf(hostNameAndPort, ServerName.NON_STARTCODE));
}
return serverList;
}
protected ServerAndLoad randomServer(final int numRegionsPerServer) {
if (!this.serverQueue.isEmpty()) {
ServerName sn = this.serverQueue.poll();
return new ServerAndLoad(sn, numRegionsPerServer);
}
String host = "srv" + rand.nextInt(Integer.MAX_VALUE);
int port = rand.nextInt(60000);
long startCode = rand.nextLong();
ServerName sn = ServerName.valueOf(host, port, startCode);
return new ServerAndLoad(sn, numRegionsPerServer);
}
@Test
public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception {
// this test is for when distributed log replay is enabled
if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return;
LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization");
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
MasterFileSystem fs = master.getMasterFileSystem();
String failedRegion = "failedRegoin1";
String staleRegion = "staleRegion";
ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1");
ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1");
String walPath =
"/hbase/data/.logs/" + inRecoveryServerName.getServerName() + "-splitting/test";
// Create a ZKW to use in the test
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
zkw.getRecoverableZooKeeper()
.create(
ZKSplitLog.getEncodedNodeName(zkw, walPath),
new SplitLogTask.Owned(inRecoveryServerName, fs.getLogRecoveryMode()).toByteArray(),
Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
String staleRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, staleRegion);
ZKUtil.createWithParents(zkw, staleRegionPath);
String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, failedRegion);
inRecoveringRegionPath =
ZKUtil.joinZNode(inRecoveringRegionPath, inRecoveryServerName.getServerName());
ZKUtil.createWithParents(zkw, inRecoveringRegionPath);
Set<ServerName> servers = new HashSet<ServerName>();
servers.add(previouselyFaildServerName);
fs.removeStaleRecoveringRegionsFromZK(servers);
// verification
assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1);
assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1);
ZKUtil.deleteChildrenRecursively(zkw, zkw.recoveringRegionsZNode);
ZKUtil.deleteChildrenRecursively(zkw, zkw.splitLogZNode);
zkw.close();
}
@Override
public void run() {
while (!isStopped()) {
try {
List<HRegionInfo> regions =
MetaScanner.listAllRegions(TEST_UTIL.getConfiguration(), connection, false);
// select a random region
HRegionInfo parent = regions.get(random.nextInt(regions.size()));
if (parent == null || !TABLENAME.equals(parent.getTable())) {
continue;
}
long startKey = 0, endKey = Long.MAX_VALUE;
byte[] start = parent.getStartKey();
byte[] end = parent.getEndKey();
if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) {
startKey = Bytes.toLong(parent.getStartKey());
}
if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) {
endKey = Bytes.toLong(parent.getEndKey());
}
if (startKey == endKey) {
continue;
}
long midKey =
BigDecimal.valueOf(startKey)
.add(BigDecimal.valueOf(endKey))
.divideToIntegralValue(BigDecimal.valueOf(2))
.longValue();
HRegionInfo splita = new HRegionInfo(TABLENAME, start, Bytes.toBytes(midKey));
HRegionInfo splitb = new HRegionInfo(TABLENAME, Bytes.toBytes(midKey), end);
MetaTableAccessor.splitRegion(
connection, parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0));
Threads.sleep(random.nextInt(200));
} catch (Throwable e) {
ex = e;
Assert.fail(StringUtils.stringifyException(e));
}
}
}
/**
* Load the meta region state from the meta server ZNode.
*
* @param zkw
* @param replicaId
* @return regionstate
* @throws KeeperException
*/
public static RegionState getMetaRegionState(ZooKeeperWatcher zkw, int replicaId)
throws KeeperException {
RegionState.State state = RegionState.State.OPEN;
ServerName serverName = null;
try {
byte[] data = ZKUtil.getData(zkw, zkw.getZNodeForReplica(replicaId));
if (data != null && data.length > 0 && ProtobufUtil.isPBMagicPrefix(data)) {
try {
int prefixLen = ProtobufUtil.lengthOfPBMagic();
ZooKeeperProtos.MetaRegionServer rl =
ZooKeeperProtos.MetaRegionServer.PARSER.parseFrom(
data, prefixLen, data.length - prefixLen);
if (rl.hasState()) {
state = RegionState.State.convert(rl.getState());
}
HBaseProtos.ServerName sn = rl.getServer();
serverName = ServerName.valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode());
} catch (InvalidProtocolBufferException e) {
throw new DeserializationException("Unable to parse meta region location");
}
} else {
// old style of meta region location?
serverName = ServerName.parseFrom(data);
}
} catch (DeserializationException e) {
throw ZKUtil.convert(e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
if (serverName == null) {
state = RegionState.State.OFFLINE;
}
return new RegionState(
RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO, replicaId),
state,
serverName);
}
/** Test client behavior w/o setting up a cluster. Mock up cluster emissions. */
@Category(SmallTests.class)
public class TestClientNoCluster extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(TestClientNoCluster.class);
private Configuration conf;
public static final ServerName META_SERVERNAME =
ServerName.valueOf("meta.example.org", 60010, 12345);
@Before
public void setUp() throws Exception {
this.conf = HBaseConfiguration.create();
// Run my HConnection overrides. Use my little HConnectionImplementation below which
// allows me insert mocks and also use my Registry below rather than the default zk based
// one so tests run faster and don't have zk dependency.
this.conf.set("hbase.client.registry.impl", SimpleRegistry.class.getName());
}
/** Simple cluster registry inserted in place of our usual zookeeper based one. */
static class SimpleRegistry implements Registry {
final ServerName META_HOST = META_SERVERNAME;
@Override
public void init(HConnection connection) {}
@Override
public HRegionLocation getMetaRegionLocation() throws IOException {
return new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, META_HOST);
}
@Override
public String getClusterId() {
return HConstants.CLUSTER_ID_DEFAULT;
}
@Override
public boolean isTableOnlineState(TableName tableName, boolean enabled) throws IOException {
return enabled;
}
@Override
public int getCurrentNrHRS() throws IOException {
return 1;
}
}
/**
* Remove the @Ignore to try out timeout and retry asettings
*
* @throws IOException
*/
@Ignore
@Test
public void testTimeoutAndRetries() throws IOException {
Configuration localConfig = HBaseConfiguration.create(this.conf);
// This override mocks up our exists/get call to throw a RegionServerStoppedException.
localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName());
HTable table = new HTable(localConfig, TableName.META_TABLE_NAME);
Throwable t = null;
LOG.info("Start");
try {
// An exists call turns into a get w/ a flag.
table.exists(new Get(Bytes.toBytes("abc")));
} catch (SocketTimeoutException e) {
// I expect this exception.
LOG.info("Got expected exception", e);
t = e;
} catch (RetriesExhaustedException e) {
// This is the old, unwanted behavior. If we get here FAIL!!!
fail();
} finally {
table.close();
}
LOG.info("Stop");
assertTrue(t != null);
}
/**
* Test that operation timeout prevails over rpc default timeout and retries, etc.
*
* @throws IOException
*/
@Test
public void testRocTimeout() throws IOException {
Configuration localConfig = HBaseConfiguration.create(this.conf);
// This override mocks up our exists/get call to throw a RegionServerStoppedException.
localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName());
int pause = 10;
localConfig.setInt("hbase.client.pause", pause);
localConfig.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10);
// Set the operation timeout to be < the pause. Expectation is that after first pause, we will
// fail out of the rpc because the rpc timeout will have been set to the operation tiemout
// and it has expired. Otherwise, if this functionality is broke, all retries will be run --
// all ten of them -- and we'll get the RetriesExhaustedException exception.
localConfig.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, pause - 1);
HTable table = new HTable(localConfig, TableName.META_TABLE_NAME);
Throwable t = null;
try {
// An exists call turns into a get w/ a flag.
table.exists(new Get(Bytes.toBytes("abc")));
} catch (SocketTimeoutException e) {
// I expect this exception.
LOG.info("Got expected exception", e);
t = e;
} catch (RetriesExhaustedException e) {
// This is the old, unwanted behavior. If we get here FAIL!!!
fail();
} finally {
table.close();
}
assertTrue(t != null);
}
@Test
public void testDoNotRetryMetaScanner() throws IOException {
this.conf.set(
"hbase.client.connection.impl", RegionServerStoppedOnScannerOpenConnection.class.getName());
MetaScanner.metaScan(this.conf, null);
}
@Test
public void testDoNotRetryOnScanNext() throws IOException {
this.conf.set(
"hbase.client.connection.impl", RegionServerStoppedOnScannerOpenConnection.class.getName());
// Go against meta else we will try to find first region for the table on construction which
// means we'll have to do a bunch more mocking. Tests that go against meta only should be
// good for a bit of testing.
HTable table = new HTable(this.conf, TableName.META_TABLE_NAME);
ResultScanner scanner = table.getScanner(HConstants.CATALOG_FAMILY);
try {
Result result = null;
while ((result = scanner.next()) != null) {
LOG.info(result);
}
} finally {
scanner.close();
table.close();
}
}
@Test
public void testRegionServerStoppedOnScannerOpen() throws IOException {
this.conf.set(
"hbase.client.connection.impl", RegionServerStoppedOnScannerOpenConnection.class.getName());
// Go against meta else we will try to find first region for the table on construction which
// means we'll have to do a bunch more mocking. Tests that go against meta only should be
// good for a bit of testing.
HTable table = new HTable(this.conf, TableName.META_TABLE_NAME);
ResultScanner scanner = table.getScanner(HConstants.CATALOG_FAMILY);
try {
Result result = null;
while ((result = scanner.next()) != null) {
LOG.info(result);
}
} finally {
scanner.close();
table.close();
}
}
/** Override to shutdown going to zookeeper for cluster id and meta location. */
static class ScanOpenNextThenExceptionThenRecoverConnection
extends HConnectionManager.HConnectionImplementation {
final ClientService.BlockingInterface stub;
ScanOpenNextThenExceptionThenRecoverConnection(
Configuration conf, boolean managed, ExecutorService pool) throws IOException {
super(conf, managed);
// Mock up my stub so open scanner returns a scanner id and then on next, we throw
// exceptions for three times and then after that, we return no more to scan.
this.stub = Mockito.mock(ClientService.BlockingInterface.class);
long sid = 12345L;
try {
Mockito.when(
stub.scan((RpcController) Mockito.any(), (ClientProtos.ScanRequest) Mockito.any()))
.thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build())
.thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito")))
.thenReturn(
ClientProtos.ScanResponse.newBuilder()
.setScannerId(sid)
.setMoreResults(false)
.build());
} catch (ServiceException e) {
throw new IOException(e);
}
}
@Override
public BlockingInterface getClient(ServerName sn) throws IOException {
return this.stub;
}
}
/** Override to shutdown going to zookeeper for cluster id and meta location. */
static class RegionServerStoppedOnScannerOpenConnection
extends HConnectionManager.HConnectionImplementation {
final ClientService.BlockingInterface stub;
RegionServerStoppedOnScannerOpenConnection(
Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException {
super(conf, managed);
// Mock up my stub so open scanner returns a scanner id and then on next, we throw
// exceptions for three times and then after that, we return no more to scan.
this.stub = Mockito.mock(ClientService.BlockingInterface.class);
long sid = 12345L;
try {
Mockito.when(
stub.scan((RpcController) Mockito.any(), (ClientProtos.ScanRequest) Mockito.any()))
.thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build())
.thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito")))
.thenReturn(
ClientProtos.ScanResponse.newBuilder()
.setScannerId(sid)
.setMoreResults(false)
.build());
} catch (ServiceException e) {
throw new IOException(e);
}
}
@Override
public BlockingInterface getClient(ServerName sn) throws IOException {
return this.stub;
}
}
/** Override to check we are setting rpc timeout right. */
static class RpcTimeoutConnection extends HConnectionManager.HConnectionImplementation {
final ClientService.BlockingInterface stub;
RpcTimeoutConnection(Configuration conf, boolean managed, ExecutorService pool, User user)
throws IOException {
super(conf, managed);
// Mock up my stub so an exists call -- which turns into a get -- throws an exception
this.stub = Mockito.mock(ClientService.BlockingInterface.class);
try {
Mockito.when(
stub.get((RpcController) Mockito.any(), (ClientProtos.GetRequest) Mockito.any()))
.thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito")));
} catch (ServiceException e) {
throw new IOException(e);
}
}
@Override
public BlockingInterface getClient(ServerName sn) throws IOException {
return this.stub;
}
}
/** Fake many regionservers and many regions on a connection implementation. */
static class ManyServersManyRegionsConnection
extends HConnectionManager.HConnectionImplementation {
// All access should be synchronized
final Map<ServerName, ClientService.BlockingInterface> serversByClient;
/** Map of faked-up rows of a 'meta table'. */
final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta;
final AtomicLong sequenceids = new AtomicLong(0);
private final Configuration conf;
ManyServersManyRegionsConnection(
Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException {
super(conf, managed, pool, user);
int serverCount = conf.getInt("hbase.test.servers", 10);
this.serversByClient = new HashMap<ServerName, ClientService.BlockingInterface>(serverCount);
this.meta =
makeMeta(
Bytes.toBytes(conf.get("hbase.test.tablename", Bytes.toString(BIG_USER_TABLE))),
conf.getInt("hbase.test.regions", 100),
conf.getLong("hbase.test.namespace.span", 1000),
serverCount);
this.conf = conf;
}
@Override
public ClientService.BlockingInterface getClient(ServerName sn) throws IOException {
// if (!sn.toString().startsWith("meta")) LOG.info(sn);
ClientService.BlockingInterface stub = null;
synchronized (this.serversByClient) {
stub = this.serversByClient.get(sn);
if (stub == null) {
stub = new FakeServer(this.conf, meta, sequenceids);
this.serversByClient.put(sn, stub);
}
}
return stub;
}
}
static MultiResponse doMultiResponse(
final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta,
final AtomicLong sequenceids,
final MultiRequest request) {
// Make a response to match the request. Act like there were no failures.
ClientProtos.MultiResponse.Builder builder = ClientProtos.MultiResponse.newBuilder();
// Per Region.
RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
ResultOrException.Builder roeBuilder = ResultOrException.newBuilder();
for (RegionAction regionAction : request.getRegionActionList()) {
regionActionResultBuilder.clear();
// Per Action in a Region.
for (ClientProtos.Action action : regionAction.getActionList()) {
roeBuilder.clear();
// Return empty Result and proper index as result.
roeBuilder.setResult(ClientProtos.Result.getDefaultInstance());
roeBuilder.setIndex(action.getIndex());
regionActionResultBuilder.addResultOrException(roeBuilder.build());
}
builder.addRegionActionResult(regionActionResultBuilder.build());
}
return builder.build();
}
/**
* Fake 'server'. Implements the ClientService responding as though it were a 'server' (presumes a
* new ClientService.BlockingInterface made per server).
*/
static class FakeServer implements ClientService.BlockingInterface {
private AtomicInteger multiInvocationsCount = new AtomicInteger(0);
private final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta;
private final AtomicLong sequenceids;
private final long multiPause;
private final int tooManyMultiRequests;
FakeServer(
final Configuration c,
final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta,
final AtomicLong sequenceids) {
this.meta = meta;
this.sequenceids = sequenceids;
// Pause to simulate the server taking time applying the edits. This will drive up the
// number of threads used over in client.
this.multiPause = c.getLong("hbase.test.multi.pause.when.done", 0);
this.tooManyMultiRequests = c.getInt("hbase.test.multi.too.many", 3);
}
@Override
public GetResponse get(RpcController controller, GetRequest request) throws ServiceException {
boolean metaRegion =
isMetaRegion(request.getRegion().getValue().toByteArray(), request.getRegion().getType());
if (!metaRegion) {
return doGetResponse(request);
}
return doMetaGetResponse(meta, request);
}
private GetResponse doGetResponse(GetRequest request) {
ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder();
ByteString row = request.getGet().getRow();
resultBuilder.addCell(getStartCode(row));
GetResponse.Builder builder = GetResponse.newBuilder();
builder.setResult(resultBuilder.build());
return builder.build();
}
@Override
public MutateResponse mutate(RpcController controller, MutateRequest request)
throws ServiceException {
throw new NotImplementedException();
}
@Override
public ScanResponse scan(RpcController controller, ScanRequest request)
throws ServiceException {
// Presume it is a scan of meta for now. Not all scans provide a region spec expecting
// the server to keep reference by scannerid. TODO.
return doMetaScanResponse(meta, sequenceids, request);
}
@Override
public BulkLoadHFileResponse bulkLoadHFile(
RpcController controller, BulkLoadHFileRequest request) throws ServiceException {
throw new NotImplementedException();
}
@Override
public CoprocessorServiceResponse execService(
RpcController controller, CoprocessorServiceRequest request) throws ServiceException {
throw new NotImplementedException();
}
@Override
public MultiResponse multi(RpcController controller, MultiRequest request)
throws ServiceException {
int concurrentInvocations = this.multiInvocationsCount.incrementAndGet();
try {
if (concurrentInvocations >= tooManyMultiRequests) {
throw new ServiceException(
new RegionTooBusyException("concurrentInvocations=" + concurrentInvocations));
}
Threads.sleep(multiPause);
return doMultiResponse(meta, sequenceids, request);
} finally {
this.multiInvocationsCount.decrementAndGet();
}
}
}
static ScanResponse doMetaScanResponse(
final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta,
final AtomicLong sequenceids,
final ScanRequest request) {
ScanResponse.Builder builder = ScanResponse.newBuilder();
int max = request.getNumberOfRows();
int count = 0;
Map<byte[], Pair<HRegionInfo, ServerName>> tail =
request.hasScan() ? meta.tailMap(request.getScan().getStartRow().toByteArray()) : meta;
ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder();
for (Map.Entry<byte[], Pair<HRegionInfo, ServerName>> e : tail.entrySet()) {
// Can be 0 on open of a scanner -- i.e. rpc to setup scannerid only.
if (max <= 0) break;
if (++count > max) break;
HRegionInfo hri = e.getValue().getFirst();
ByteString row = HBaseZeroCopyByteString.wrap(hri.getRegionName());
resultBuilder.clear();
resultBuilder.addCell(getRegionInfo(row, hri));
resultBuilder.addCell(getServer(row, e.getValue().getSecond()));
resultBuilder.addCell(getStartCode(row));
builder.addResults(resultBuilder.build());
// Set more to false if we are on the last region in table.
if (hri.getEndKey().length <= 0) builder.setMoreResults(false);
else builder.setMoreResults(true);
}
// If no scannerid, set one.
builder.setScannerId(
request.hasScannerId() ? request.getScannerId() : sequenceids.incrementAndGet());
return builder.build();
}
static GetResponse doMetaGetResponse(
final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta, final GetRequest request) {
ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder();
ByteString row = request.getGet().getRow();
Pair<HRegionInfo, ServerName> p = meta.get(row.toByteArray());
if (p == null) {
if (request.getGet().getClosestRowBefore()) {
byte[] bytes = row.toByteArray();
SortedMap<byte[], Pair<HRegionInfo, ServerName>> head =
bytes != null ? meta.headMap(bytes) : meta;
p = head == null ? null : head.get(head.lastKey());
}
}
if (p != null) {
resultBuilder.addCell(getRegionInfo(row, p.getFirst()));
resultBuilder.addCell(getServer(row, p.getSecond()));
}
resultBuilder.addCell(getStartCode(row));
GetResponse.Builder builder = GetResponse.newBuilder();
builder.setResult(resultBuilder.build());
return builder.build();
}
/**
* @param name region name or encoded region name.
* @param type
* @return True if we are dealing with a hbase:meta region.
*/
static boolean isMetaRegion(final byte[] name, final RegionSpecifierType type) {
switch (type) {
case REGION_NAME:
return Bytes.equals(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), name);
case ENCODED_REGION_NAME:
return Bytes.equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), name);
default:
throw new UnsupportedOperationException();
}
}
private static final ByteString CATALOG_FAMILY_BYTESTRING =
HBaseZeroCopyByteString.wrap(HConstants.CATALOG_FAMILY);
private static final ByteString REGIONINFO_QUALIFIER_BYTESTRING =
HBaseZeroCopyByteString.wrap(HConstants.REGIONINFO_QUALIFIER);
private static final ByteString SERVER_QUALIFIER_BYTESTRING =
HBaseZeroCopyByteString.wrap(HConstants.SERVER_QUALIFIER);
static CellProtos.Cell.Builder getBaseCellBuilder(final ByteString row) {
CellProtos.Cell.Builder cellBuilder = CellProtos.Cell.newBuilder();
cellBuilder.setRow(row);
cellBuilder.setFamily(CATALOG_FAMILY_BYTESTRING);
cellBuilder.setTimestamp(System.currentTimeMillis());
return cellBuilder;
}
static CellProtos.Cell getRegionInfo(final ByteString row, final HRegionInfo hri) {
CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row);
cellBuilder.setQualifier(REGIONINFO_QUALIFIER_BYTESTRING);
cellBuilder.setValue(HBaseZeroCopyByteString.wrap(hri.toByteArray()));
return cellBuilder.build();
}
static CellProtos.Cell getServer(final ByteString row, final ServerName sn) {
CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row);
cellBuilder.setQualifier(SERVER_QUALIFIER_BYTESTRING);
cellBuilder.setValue(ByteString.copyFromUtf8(sn.getHostAndPort()));
return cellBuilder.build();
}
static CellProtos.Cell getStartCode(final ByteString row) {
CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row);
cellBuilder.setQualifier(HBaseZeroCopyByteString.wrap(HConstants.STARTCODE_QUALIFIER));
// TODO:
cellBuilder.setValue(
HBaseZeroCopyByteString.wrap(Bytes.toBytes(META_SERVERNAME.getStartcode())));
return cellBuilder.build();
}
private static final byte[] BIG_USER_TABLE = Bytes.toBytes("t");
/**
* Format passed integer. Zero-pad. Copied from hbase-server PE class and small amendment. Make
* them share.
*
* @param number
* @return Returns zero-prefixed 10-byte wide decimal version of passed number (Does absolute in
* case number is negative).
*/
private static byte[] format(final long number) {
byte[] b = new byte[10];
long d = number;
for (int i = b.length - 1; i >= 0; i--) {
b[i] = (byte) ((d % 10) + '0');
d /= 10;
}
return b;
}
/**
* @param count
* @param namespaceSpan
* @return <code>count</code> regions
*/
private static HRegionInfo[] makeHRegionInfos(
final byte[] tableName, final int count, final long namespaceSpan) {
byte[] startKey = HConstants.EMPTY_BYTE_ARRAY;
byte[] endKey = HConstants.EMPTY_BYTE_ARRAY;
long interval = namespaceSpan / count;
HRegionInfo[] hris = new HRegionInfo[count];
for (int i = 0; i < count; i++) {
if (i == 0) {
endKey = format(interval);
} else {
startKey = endKey;
if (i == count - 1) endKey = HConstants.EMPTY_BYTE_ARRAY;
else endKey = format((i + 1) * interval);
}
hris[i] = new HRegionInfo(TableName.valueOf(tableName), startKey, endKey);
}
return hris;
}
/**
* @param count
* @return Return <code>count</code> servernames.
*/
private static ServerName[] makeServerNames(final int count) {
ServerName[] sns = new ServerName[count];
for (int i = 0; i < count; i++) {
sns[i] = ServerName.valueOf("" + i + ".example.org", 60010, i);
}
return sns;
}
/** Comparator for meta row keys. */
private static class MetaRowsComparator implements Comparator<byte[]> {
private final KeyValue.KVComparator delegate = new KeyValue.MetaComparator();
@Override
public int compare(byte[] left, byte[] right) {
return delegate.compareRows(left, 0, left.length, right, 0, right.length);
}
}
/**
* Create up a map that is keyed by meta row name and whose value is the HRegionInfo and
* ServerName to return for this row.
*
* @return Map with faked hbase:meta content in it.
*/
static SortedMap<byte[], Pair<HRegionInfo, ServerName>> makeMeta(
final byte[] tableName,
final int regionCount,
final long namespaceSpan,
final int serverCount) {
// I need a comparator for meta rows so we sort properly.
SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta =
new ConcurrentSkipListMap<byte[], Pair<HRegionInfo, ServerName>>(new MetaRowsComparator());
HRegionInfo[] hris = makeHRegionInfos(tableName, regionCount, namespaceSpan);
ServerName[] serverNames = makeServerNames(serverCount);
int per = regionCount / serverCount;
int count = 0;
for (HRegionInfo hri : hris) {
Pair<HRegionInfo, ServerName> p =
new Pair<HRegionInfo, ServerName>(hri, serverNames[count++ / per]);
meta.put(hri.getRegionName(), p);
}
return meta;
}
/**
* Code for each 'client' to run.
*
* @param id
* @param c
* @param sharedConnection
* @throws IOException
*/
static void cycle(int id, final Configuration c, final HConnection sharedConnection)
throws IOException {
HTableInterface table = sharedConnection.getTable(BIG_USER_TABLE);
table.setAutoFlushTo(false);
long namespaceSpan = c.getLong("hbase.test.namespace.span", 1000000);
long startTime = System.currentTimeMillis();
final int printInterval = 100000;
Random rd = new Random(id);
boolean get = c.getBoolean("hbase.test.do.gets", false);
try {
Stopwatch stopWatch = new Stopwatch();
stopWatch.start();
for (int i = 0; i < namespaceSpan; i++) {
byte[] b = format(rd.nextLong());
if (get) {
Get g = new Get(b);
table.get(g);
} else {
Put p = new Put(b);
p.add(HConstants.CATALOG_FAMILY, b, b);
table.put(p);
}
if (i % printInterval == 0) {
LOG.info("Put " + printInterval + "/" + stopWatch.elapsedMillis());
stopWatch.reset();
stopWatch.start();
}
}
LOG.info(
"Finished a cycle putting "
+ namespaceSpan
+ " in "
+ (System.currentTimeMillis() - startTime)
+ "ms");
} finally {
table.close();
}
}
@Override
public int run(String[] arg0) throws Exception {
int errCode = 0;
// TODO: Make command options.
// How many servers to fake.
final int servers = 1;
// How many regions to put on the faked servers.
final int regions = 100000;
// How many 'keys' in the faked regions.
final long namespaceSpan = 50000000;
// How long to take to pause after doing a put; make this long if you want to fake a struggling
// server.
final long multiPause = 0;
// Check args make basic sense.
if ((namespaceSpan < regions) || (regions < servers)) {
throw new IllegalArgumentException(
"namespaceSpan="
+ namespaceSpan
+ " must be > regions="
+ regions
+ " which must be > servers="
+ servers);
}
// Set my many servers and many regions faking connection in place.
getConf().set("hbase.client.connection.impl", ManyServersManyRegionsConnection.class.getName());
// Use simple kv registry rather than zk
getConf().set("hbase.client.registry.impl", SimpleRegistry.class.getName());
// When to report fails. Default is we report the 10th. This means we'll see log everytime
// an exception is thrown -- usually RegionTooBusyException when we have more than
// hbase.test.multi.too.many requests outstanding at any time.
getConf().setInt("hbase.client.start.log.errors.counter", 0);
// Ugly but this is only way to pass in configs.into ManyServersManyRegionsConnection class.
getConf().setInt("hbase.test.regions", regions);
getConf().setLong("hbase.test.namespace.span", namespaceSpan);
getConf().setLong("hbase.test.servers", servers);
getConf().set("hbase.test.tablename", Bytes.toString(BIG_USER_TABLE));
getConf().setLong("hbase.test.multi.pause.when.done", multiPause);
// Let there be ten outstanding requests at a time before we throw RegionBusyException.
getConf().setInt("hbase.test.multi.too.many", 10);
final int clients = 2;
// Have them all share the same connection so they all share the same instance of
// ManyServersManyRegionsConnection so I can keep an eye on how many requests by server.
final ExecutorService pool = Executors.newCachedThreadPool(Threads.getNamedThreadFactory("p"));
// Executors.newFixedThreadPool(servers * 10, Threads.getNamedThreadFactory("p"));
// Share a connection so I can keep counts in the 'server' on concurrency.
final HConnection sharedConnection = HConnectionManager.createConnection(getConf() /*, pool*/);
try {
Thread[] ts = new Thread[clients];
for (int j = 0; j < ts.length; j++) {
final int id = j;
ts[j] =
new Thread("" + j) {
final Configuration c = getConf();
@Override
public void run() {
try {
cycle(id, c, sharedConnection);
} catch (IOException e) {
e.printStackTrace();
}
}
};
ts[j].start();
}
for (int j = 0; j < ts.length; j++) {
ts[j].join();
}
} finally {
sharedConnection.close();
}
return errCode;
}
/**
* Run a client instance against a faked up server.
*
* @param args TODO
* @throws Exception
*/
public static void main(String[] args) throws Exception {
System.exit(ToolRunner.run(HBaseConfiguration.create(), new TestClientNoCluster(), args));
}
}
/**
* Generate the assignment plan for the existing table
*
* @param tableName
* @param assignmentSnapshot
* @param regionLocalityMap
* @param plan
* @param munkresForSecondaryAndTertiary if set on true the assignment plan for the tertiary and
* secondary will be generated with Munkres algorithm, otherwise will be generated using
* placeSecondaryAndTertiaryRS
* @throws IOException
*/
private void genAssignmentPlan(
TableName tableName,
SnapshotOfRegionAssignmentFromMeta assignmentSnapshot,
Map<String, Map<String, Float>> regionLocalityMap,
FavoredNodesPlan plan,
boolean munkresForSecondaryAndTertiary)
throws IOException {
// Get the all the regions for the current table
List<HRegionInfo> regions = assignmentSnapshot.getTableToRegionMap().get(tableName);
int numRegions = regions.size();
// Get the current assignment map
Map<HRegionInfo, ServerName> currentAssignmentMap =
assignmentSnapshot.getRegionToRegionServerMap();
// Get the all the region servers
List<ServerName> servers = new ArrayList<ServerName>();
try (Admin admin = this.connection.getAdmin()) {
servers.addAll(admin.getClusterStatus().getServers());
}
LOG.info(
"Start to generate assignment plan for "
+ numRegions
+ " regions from table "
+ tableName
+ " with "
+ servers.size()
+ " region servers");
int slotsPerServer = (int) Math.ceil((float) numRegions / servers.size());
int regionSlots = slotsPerServer * servers.size();
// Compute the primary, secondary and tertiary costs for each region/server
// pair. These costs are based only on node locality and rack locality, and
// will be modified later.
float[][] primaryCost = new float[numRegions][regionSlots];
float[][] secondaryCost = new float[numRegions][regionSlots];
float[][] tertiaryCost = new float[numRegions][regionSlots];
if (this.enforceLocality && regionLocalityMap != null) {
// Transform the locality mapping into a 2D array, assuming that any
// unspecified locality value is 0.
float[][] localityPerServer = new float[numRegions][regionSlots];
for (int i = 0; i < numRegions; i++) {
Map<String, Float> serverLocalityMap =
regionLocalityMap.get(regions.get(i).getEncodedName());
if (serverLocalityMap == null) {
continue;
}
for (int j = 0; j < servers.size(); j++) {
String serverName = servers.get(j).getHostname();
if (serverName == null) {
continue;
}
Float locality = serverLocalityMap.get(serverName);
if (locality == null) {
continue;
}
for (int k = 0; k < slotsPerServer; k++) {
// If we can't find the locality of a region to a server, which occurs
// because locality is only reported for servers which have some
// blocks of a region local, then the locality for that pair is 0.
localityPerServer[i][j * slotsPerServer + k] = locality.floatValue();
}
}
}
// Compute the total rack locality for each region in each rack. The total
// rack locality is the sum of the localities of a region on all servers in
// a rack.
Map<String, Map<HRegionInfo, Float>> rackRegionLocality =
new HashMap<String, Map<HRegionInfo, Float>>();
for (int i = 0; i < numRegions; i++) {
HRegionInfo region = regions.get(i);
for (int j = 0; j < regionSlots; j += slotsPerServer) {
String rack = rackManager.getRack(servers.get(j / slotsPerServer));
Map<HRegionInfo, Float> rackLocality = rackRegionLocality.get(rack);
if (rackLocality == null) {
rackLocality = new HashMap<HRegionInfo, Float>();
rackRegionLocality.put(rack, rackLocality);
}
Float localityObj = rackLocality.get(region);
float locality = localityObj == null ? 0 : localityObj.floatValue();
locality += localityPerServer[i][j];
rackLocality.put(region, locality);
}
}
for (int i = 0; i < numRegions; i++) {
for (int j = 0; j < regionSlots; j++) {
String rack = rackManager.getRack(servers.get(j / slotsPerServer));
Float totalRackLocalityObj = rackRegionLocality.get(rack).get(regions.get(i));
float totalRackLocality =
totalRackLocalityObj == null ? 0 : totalRackLocalityObj.floatValue();
// Primary cost aims to favor servers with high node locality and low
// rack locality, so that secondaries and tertiaries can be chosen for
// nodes with high rack locality. This might give primaries with
// slightly less locality at first compared to a cost which only
// considers the node locality, but should be better in the long run.
primaryCost[i][j] = 1 - (2 * localityPerServer[i][j] - totalRackLocality);
// Secondary cost aims to favor servers with high node locality and high
// rack locality since the tertiary will be chosen from the same rack as
// the secondary. This could be negative, but that is okay.
secondaryCost[i][j] = 2 - (localityPerServer[i][j] + totalRackLocality);
// Tertiary cost is only concerned with the node locality. It will later
// be restricted to only hosts on the same rack as the secondary.
tertiaryCost[i][j] = 1 - localityPerServer[i][j];
}
}
}
if (this.enforceMinAssignmentMove && currentAssignmentMap != null) {
// We want to minimize the number of regions which move as the result of a
// new assignment. Therefore, slightly penalize any placement which is for
// a host that is not currently serving the region.
for (int i = 0; i < numRegions; i++) {
for (int j = 0; j < servers.size(); j++) {
ServerName currentAddress = currentAssignmentMap.get(regions.get(i));
if (currentAddress != null && !currentAddress.equals(servers.get(j))) {
for (int k = 0; k < slotsPerServer; k++) {
primaryCost[i][j * slotsPerServer + k] += NOT_CURRENT_HOST_PENALTY;
}
}
}
}
}
// Artificially increase cost of last slot of each server to evenly
// distribute the slop, otherwise there will be a few servers with too few
// regions and many servers with the max number of regions.
for (int i = 0; i < numRegions; i++) {
for (int j = 0; j < regionSlots; j += slotsPerServer) {
primaryCost[i][j] += LAST_SLOT_COST_PENALTY;
secondaryCost[i][j] += LAST_SLOT_COST_PENALTY;
tertiaryCost[i][j] += LAST_SLOT_COST_PENALTY;
}
}
RandomizedMatrix randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots);
primaryCost = randomizedMatrix.transform(primaryCost);
int[] primaryAssignment = new MunkresAssignment(primaryCost).solve();
primaryAssignment = randomizedMatrix.invertIndices(primaryAssignment);
// Modify the secondary and tertiary costs for each region/server pair to
// prevent a region from being assigned to the same rack for both primary
// and either one of secondary or tertiary.
for (int i = 0; i < numRegions; i++) {
int slot = primaryAssignment[i];
String rack = rackManager.getRack(servers.get(slot / slotsPerServer));
for (int k = 0; k < servers.size(); k++) {
if (!rackManager.getRack(servers.get(k)).equals(rack)) {
continue;
}
if (k == slot / slotsPerServer) {
// Same node, do not place secondary or tertiary here ever.
for (int m = 0; m < slotsPerServer; m++) {
secondaryCost[i][k * slotsPerServer + m] = MAX_COST;
tertiaryCost[i][k * slotsPerServer + m] = MAX_COST;
}
} else {
// Same rack, do not place secondary or tertiary here if possible.
for (int m = 0; m < slotsPerServer; m++) {
secondaryCost[i][k * slotsPerServer + m] = AVOID_COST;
tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST;
}
}
}
}
if (munkresForSecondaryAndTertiary) {
randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots);
secondaryCost = randomizedMatrix.transform(secondaryCost);
int[] secondaryAssignment = new MunkresAssignment(secondaryCost).solve();
secondaryAssignment = randomizedMatrix.invertIndices(secondaryAssignment);
// Modify the tertiary costs for each region/server pair to ensure that a
// region is assigned to a tertiary server on the same rack as its secondary
// server, but not the same server in that rack.
for (int i = 0; i < numRegions; i++) {
int slot = secondaryAssignment[i];
String rack = rackManager.getRack(servers.get(slot / slotsPerServer));
for (int k = 0; k < servers.size(); k++) {
if (k == slot / slotsPerServer) {
// Same node, do not place tertiary here ever.
for (int m = 0; m < slotsPerServer; m++) {
tertiaryCost[i][k * slotsPerServer + m] = MAX_COST;
}
} else {
if (rackManager.getRack(servers.get(k)).equals(rack)) {
continue;
}
// Different rack, do not place tertiary here if possible.
for (int m = 0; m < slotsPerServer; m++) {
tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST;
}
}
}
}
randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots);
tertiaryCost = randomizedMatrix.transform(tertiaryCost);
int[] tertiaryAssignment = new MunkresAssignment(tertiaryCost).solve();
tertiaryAssignment = randomizedMatrix.invertIndices(tertiaryAssignment);
for (int i = 0; i < numRegions; i++) {
List<ServerName> favoredServers =
new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM);
ServerName s = servers.get(primaryAssignment[i] / slotsPerServer);
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
s = servers.get(secondaryAssignment[i] / slotsPerServer);
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
s = servers.get(tertiaryAssignment[i] / slotsPerServer);
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
// Update the assignment plan
plan.updateAssignmentPlan(regions.get(i), favoredServers);
}
LOG.info(
"Generated the assignment plan for "
+ numRegions
+ " regions from table "
+ tableName
+ " with "
+ servers.size()
+ " region servers");
LOG.info("Assignment plan for secondary and tertiary generated " + "using MunkresAssignment");
} else {
Map<HRegionInfo, ServerName> primaryRSMap = new HashMap<HRegionInfo, ServerName>();
for (int i = 0; i < numRegions; i++) {
primaryRSMap.put(regions.get(i), servers.get(primaryAssignment[i] / slotsPerServer));
}
FavoredNodeAssignmentHelper favoredNodeHelper =
new FavoredNodeAssignmentHelper(servers, conf);
favoredNodeHelper.initialize();
Map<HRegionInfo, ServerName[]> secondaryAndTertiaryMap =
favoredNodeHelper.placeSecondaryAndTertiaryWithRestrictions(primaryRSMap);
for (int i = 0; i < numRegions; i++) {
List<ServerName> favoredServers =
new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM);
HRegionInfo currentRegion = regions.get(i);
ServerName s = primaryRSMap.get(currentRegion);
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
ServerName[] secondaryAndTertiary = secondaryAndTertiaryMap.get(currentRegion);
s = secondaryAndTertiary[0];
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
s = secondaryAndTertiary[1];
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
// Update the assignment plan
plan.updateAssignmentPlan(regions.get(i), favoredServers);
}
LOG.info(
"Generated the assignment plan for "
+ numRegions
+ " regions from table "
+ tableName
+ " with "
+ servers.size()
+ " region servers");
LOG.info(
"Assignment plan for secondary and tertiary generated "
+ "using placeSecondaryAndTertiaryWithRestrictions method");
}
}
@Override
public ServerName getServerName() {
return ServerName.valueOf(this.serverName);
}
/**
* Reproduce locking up that happens when there's no further syncs after append fails, and causing
* an isolated sync then infinite wait. See HBASE-16960. If below is broken, we will see this test
* timeout because it is locked up.
*
* <p>Steps for reproduce:<br>
* 1. Trigger server abort through dodgyWAL1<br>
* 2. Add a {@link DummyWALActionsListener} to dodgyWAL2 to cause ringbuffer event handler thread
* sleep for a while thus keeping {@code endOfBatch} false<br>
* 3. Publish a sync then an append which will throw exception, check whether the sync could
* return
*/
@Test(timeout = 20000)
public void testLockup16960() throws IOException {
// A WAL that we can have throw exceptions when a flag is set.
class DodgyFSLog extends FSHLog {
// Set this when want the WAL to start throwing exceptions.
volatile boolean throwException = false;
public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration conf)
throws IOException {
super(fs, root, logDir, conf);
}
@Override
protected Writer createWriterInstance(Path path) throws IOException {
final Writer w = super.createWriterInstance(path);
return new Writer() {
@Override
public void close() throws IOException {
w.close();
}
@Override
public void sync() throws IOException {
if (throwException) {
throw new IOException("FAKE! Failed to replace a bad datanode...SYNC");
}
w.sync();
}
@Override
public void append(Entry entry) throws IOException {
if (throwException) {
throw new IOException("FAKE! Failed to replace a bad datanode...APPEND");
}
w.append(entry);
}
@Override
public long getLength() {
return w.getLength();
}
};
}
@Override
protected long doReplaceWriter(Path oldPath, Path newPath, Writer nextWriter)
throws IOException {
if (throwException) {
throw new FailedLogCloseException("oldPath=" + oldPath + ", newPath=" + newPath);
}
long oldFileLen = 0L;
oldFileLen = super.doReplaceWriter(oldPath, newPath, nextWriter);
return oldFileLen;
}
}
// Mocked up server and regionserver services. Needed below.
Server server =
new DummyServer(CONF, ServerName.valueOf("hostname1.example.org", 1234, 1L).toString());
RegionServerServices services = Mockito.mock(RegionServerServices.class);
CONF.setLong("hbase.regionserver.hlog.sync.timeout", 10000);
// OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL,
// go ahead with test.
FileSystem fs = FileSystem.get(CONF);
Path rootDir = new Path(dir + getName());
DodgyFSLog dodgyWAL1 = new DodgyFSLog(fs, rootDir, getName(), CONF);
Path rootDir2 = new Path(dir + getName() + "2");
final DodgyFSLog dodgyWAL2 = new DodgyFSLog(fs, rootDir2, getName() + "2", CONF);
// Add a listener to force ringbuffer event handler sleep for a while
dodgyWAL2.registerWALActionsListener(new DummyWALActionsListener());
// I need a log roller running.
LogRoller logRoller = new LogRoller(server, services);
logRoller.addWAL(dodgyWAL1);
logRoller.addWAL(dodgyWAL2);
// There is no 'stop' once a logRoller is running.. it just dies.
logRoller.start();
// Now get a region and start adding in edits.
HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME);
final HRegion region = initHRegion(tableName, null, null, dodgyWAL1);
byte[] bytes = Bytes.toBytes(getName());
NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
scopes.put(COLUMN_FAMILY_BYTES, 0);
MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
try {
Put put = new Put(bytes);
put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
WALKey key =
new WALKey(
region.getRegionInfo().getEncodedNameAsBytes(),
htd.getTableName(),
System.currentTimeMillis(),
mvcc,
scopes);
WALEdit edit = new WALEdit();
CellScanner CellScanner = put.cellScanner();
assertTrue(CellScanner.advance());
edit.add(CellScanner.current());
LOG.info("SET throwing of exception on append");
dodgyWAL1.throwException = true;
// This append provokes a WAL roll request
dodgyWAL1.append(region.getRegionInfo(), key, edit, true);
boolean exception = false;
try {
dodgyWAL1.sync();
} catch (Exception e) {
exception = true;
}
assertTrue("Did not get sync exception", exception);
// LogRoller call dodgyWAL1.rollWriter get FailedLogCloseException and
// cause server abort.
try {
// wait LogRoller exit.
Thread.sleep(50);
} catch (InterruptedException e) {
e.printStackTrace();
}
final CountDownLatch latch = new CountDownLatch(1);
// make RingBufferEventHandler sleep 1s, so the following sync
// endOfBatch=false
key =
new WALKey(
region.getRegionInfo().getEncodedNameAsBytes(),
TableName.valueOf("sleep"),
System.currentTimeMillis(),
mvcc,
scopes);
dodgyWAL2.append(region.getRegionInfo(), key, edit, true);
Thread t =
new Thread("Sync") {
public void run() {
try {
dodgyWAL2.sync();
} catch (IOException e) {
LOG.info("In sync", e);
}
latch.countDown();
LOG.info("Sync exiting");
};
};
t.setDaemon(true);
t.start();
try {
// make sure sync have published.
Thread.sleep(100);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
// make append throw DamagedWALException
key =
new WALKey(
region.getRegionInfo().getEncodedNameAsBytes(),
TableName.valueOf("DamagedWALException"),
System.currentTimeMillis(),
mvcc,
scopes);
dodgyWAL2.append(region.getRegionInfo(), key, edit, true);
while (latch.getCount() > 0) {
Threads.sleep(100);
}
assertTrue(server.isAborted());
} finally {
if (logRoller != null) {
logRoller.close();
}
try {
if (region != null) {
region.close();
}
if (dodgyWAL1 != null) {
dodgyWAL1.close();
}
if (dodgyWAL2 != null) {
dodgyWAL2.close();
}
} catch (Exception e) {
LOG.info("On way out", e);
}
}
}
@Override
public ServerName getServerName() {
return ServerName.valueOf("regionserver,60020,000000");
}
