/**
* Test interruptable while blocking wait on root and meta.
*
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testInterruptWaitOnMetaAndRoot() throws IOException, InterruptedException {
HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
HConnection connection = mockConnection(implementation);
final CatalogTracker ct = constructAndStartCatalogTracker(connection);
ServerName hsa = ct.getRootLocation();
Assert.assertNull(hsa);
ServerName meta = ct.getMetaLocation();
Assert.assertNull(meta);
Thread t =
new Thread() {
@Override
public void run() {
try {
ct.waitForMeta();
} catch (InterruptedException e) {
throw new RuntimeException("Interrupted", e);
}
}
};
t.start();
while (!t.isAlive()) Threads.sleep(1);
Threads.sleep(1);
assertTrue(t.isAlive());
ct.stop();
// Join the thread... should exit shortly.
t.join();
}
private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
throws IOException, InterruptedException {
CatalogTracker ct =
new CatalogTracker(this.watcher, UTIL.getConfiguration(), c, this.abortable);
ct.start();
return ct;
}
/**
* Test waiting on meta w/ no timeout specified.
*
* @throws Exception
*/
@Ignore // Can't make it work reliably on all platforms; mockito gets confused
// Throwing: org.mockito.exceptions.misusing.WrongTypeOfReturnValue:
// Result cannot be returned by locateRegion()
// If you plug locateRegion, it then throws for incCounter, and if you plug
// that ... and so one.
@Test
public void testNoTimeoutWaitForMeta() throws Exception {
// Mock an HConnection and a HRegionInterface implementation. Have the
// HConnection return the HRI. Have the HRI return a few mocked up responses
// to make our test work.
// Mock an HRegionInterface.
final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
HConnection connection = mockConnection(implementation);
// Now the ct is up... set into the mocks some answers that make it look
// like things have been getting assigned. Make it so we'll return a
// location (no matter what the Get is). Same for getHRegionInfo -- always
// just return the meta region.
final Result result = getMetaTableRowResult();
// TODO: Refactor. This method has been moved out of HConnection.
// It works for now but has been deprecated.
Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>) Mockito.any()))
.thenReturn(result);
Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any()))
.thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
final CatalogTracker ct = constructAndStartCatalogTracker(connection);
ServerName hsa = ct.getMetaLocation();
Assert.assertNull(hsa);
// Now test waiting on meta location getting set.
Thread t =
new WaitOnMetaThread(ct) {
@Override
void doWaiting() throws InterruptedException {
this.ct.waitForMeta();
}
};
startWaitAliveThenWaitItLives(t, 1000);
// This should trigger wake up of meta wait (Its the removal of the meta
// region unassigned node that triggers catalogtrackers that a meta has
// been assigned).
String node = ct.getMetaNodeTracker().getNode();
ZKUtil.createAndFailSilent(this.watcher, node);
MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
ZKUtil.deleteNode(this.watcher, node);
// Go get the new meta location. waitForMeta gets and verifies meta.
Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
// Join the thread... should exit shortly.
t.join();
// Now meta is available.
Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
}
private void testVerifyMetaRegionLocationWithException(Exception ex)
throws IOException, InterruptedException, KeeperException {
// Mock an HRegionInterface.
final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
HConnection connection = mockConnection(implementation);
// If a 'get' is called on mocked interface, throw connection refused.
Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenThrow(ex);
// Now start up the catalogtracker with our doctored Connection.
final CatalogTracker ct = constructAndStartCatalogTracker(connection);
RootLocationEditor.setRootLocation(this.watcher, SN);
long timeout = UTIL.getConfiguration().getLong("hbase.catalog.verification.timeout", 1000);
Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
}
/**
* Test get of root region fails properly if nothing to connect to.
*
* @throws IOException
* @throws InterruptedException
* @throws KeeperException
*/
@Test
public void testVerifyRootRegionLocationFails()
throws IOException, InterruptedException, KeeperException {
HConnection connection = Mockito.mock(HConnection.class);
ConnectException connectException = new ConnectException("Connection refused");
final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).thenThrow(connectException);
Mockito.when(
connection.getHRegionConnection(
Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean()))
.thenReturn(implementation);
final CatalogTracker ct = constructAndStartCatalogTracker(connection);
RootLocationEditor.setRootLocation(
this.watcher, new ServerName("example.com", 1234, System.currentTimeMillis()));
Assert.assertFalse(ct.verifyRootRegionLocation(100));
}
/**
* Test waiting on root w/ no timeout specified.
*
* @throws IOException
* @throws InterruptedException
* @throws KeeperException
*/
@Test
public void testNoTimeoutWaitForRoot() throws IOException, InterruptedException, KeeperException {
HConnection connection = Mockito.mock(HConnection.class);
final CatalogTracker ct = constructAndStartCatalogTracker(connection);
ServerName hsa = ct.getRootLocation();
Assert.assertNull(hsa);
// Now test waiting on root location getting set.
Thread t = new WaitOnMetaThread(ct);
startWaitAliveThenWaitItLives(t, 1000);
// Set a root location.
hsa = setRootLocation();
// Join the thread... should exit shortly.
t.join();
// Now root is available.
Assert.assertTrue(ct.getRootLocation().equals(hsa));
}
/**
* Test that MetaReader will ride over server throwing "Server not running" IOEs.
*
* @see https://issues.apache.org/jira/browse/HBASE-3446
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testRideOverServerNotRunning() throws IOException, InterruptedException {
// Need a zk watcher.
ZooKeeperWatcher zkw =
new ZooKeeperWatcher(
UTIL.getConfiguration(), this.getClass().getSimpleName(), ABORTABLE, true);
// This is a servername we use in a few places below.
ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis());
HConnection connection = null;
CatalogTracker ct = null;
try {
// Mock an HRegionInterface. Our mock implementation will fail a few
// times when we go to open a scanner.
final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
// When openScanner called throw IOE 'Server not running' a few times
// before we return a scanner id. Whats WEIRD is that these
// exceptions do not show in the log because they are caught and only
// printed if we FAIL. We eventually succeed after retry so these don't
// show. We will know if they happened or not because we will ask
// mockito at the end of this test to verify that openscanner was indeed
// called the wanted number of times.
final long scannerid = 123L;
Mockito.when(implementation.openScanner((byte[]) Mockito.any(), (Scan) Mockito.any()))
.thenThrow(new IOException("Server not running (1 of 3)"))
.thenThrow(new IOException("Server not running (2 of 3)"))
.thenThrow(new IOException("Server not running (3 of 3)"))
.thenReturn(scannerid);
// Make it so a verifiable answer comes back when next is called. Return
// the verifiable answer and then a null so we stop scanning. Our
// verifiable answer is something that looks like a row in META with
// a server and startcode that is that of the above defined servername.
List<KeyValue> kvs = new ArrayList<KeyValue>();
final byte[] rowToVerify = Bytes.toBytes("rowToVerify");
kvs.add(
new KeyValue(
rowToVerify,
HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER,
Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
kvs.add(
new KeyValue(
rowToVerify,
HConstants.CATALOG_FAMILY,
HConstants.SERVER_QUALIFIER,
Bytes.toBytes(sn.getHostAndPort())));
kvs.add(
new KeyValue(
rowToVerify,
HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER,
Bytes.toBytes(sn.getStartcode())));
final Result[] result = new Result[] {new Result(kvs)};
Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt()))
.thenReturn(result)
.thenReturn(null);
// Associate a spied-upon HConnection with UTIL.getConfiguration. Need
// to shove this in here first so it gets picked up all over; e.g. by
// HTable.
connection = HConnectionTestingUtility.getSpiedConnection(UTIL.getConfiguration());
// Fix the location lookup so it 'works' though no network. First
// make an 'any location' object.
final HRegionLocation anyLocation =
new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(), sn.getPort());
// Return the any location object when locateRegion is called in HTable
// constructor and when its called by ServerCallable (it uses getRegionLocation).
// The ugly format below comes of 'Important gotcha on spying real objects!' from
// http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html
Mockito.doReturn(anyLocation)
.when(connection)
.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any());
Mockito.doReturn(anyLocation)
.when(connection)
.getRegionLocation((byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean());
// Now shove our HRI implementation into the spied-upon connection.
Mockito.doReturn(implementation)
.when(connection)
.getHRegionConnection(Mockito.anyString(), Mockito.anyInt());
// Now start up the catalogtracker with our doctored Connection.
ct = new CatalogTracker(zkw, null, connection, ABORTABLE, 0);
ct.start();
// Scan meta for user tables and verify we got back expected answer.
NavigableMap<HRegionInfo, Result> hris = MetaReader.getServerUserRegions(ct, sn);
assertTrue(hris.size() == 1);
assertTrue(hris.firstEntry().getKey().equals(HRegionInfo.FIRST_META_REGIONINFO));
assertTrue(Bytes.equals(rowToVerify, hris.firstEntry().getValue().getRow()));
// Finally verify that openscanner was called four times -- three times
// with exception and then on 4th attempt we succeed.
Mockito.verify(implementation, Mockito.times(4))
.openScanner((byte[]) Mockito.any(), (Scan) Mockito.any());
} finally {
if (ct != null) ct.stop();
HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
zkw.close();
}
}
@Test(expected = RetriesExhaustedException.class)
public void testTimeoutWaitForMeta() throws IOException, InterruptedException {
HConnection connection = HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
final CatalogTracker ct = constructAndStartCatalogTracker(connection);
ct.waitForMeta(100);
}
@Test(expected = NotAllMetaRegionsOnlineException.class)
public void testTimeoutWaitForRoot() throws IOException, InterruptedException {
HConnection connection = Mockito.mock(HConnection.class);
final CatalogTracker ct = constructAndStartCatalogTracker(connection);
ct.waitForRoot(100);
}
/**
* Test for HBASE-4288. Throw an IOE when trying to verify meta region and prove it doesn't cause
* master shutdown.
*
* @see <a href="https://issues.apache.org/jira/browse/HBASE-4288">HBASE-4288</a>
* @throws IOException
* @throws InterruptedException
* @throws KeeperException
*/
@Test
public void testServerNotRunningIOException()
throws IOException, InterruptedException, KeeperException {
// Mock an HRegionInterface.
final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
HConnection connection = mockConnection(implementation);
// If a 'getRegionInfo' is called on mocked HRegionInterface, throw IOE
// the first time. 'Succeed' the second time we are called.
Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any()))
.thenThrow(new IOException("Server not running, aborting"))
.thenReturn(new HRegionInfo());
// After we encounter the above 'Server not running', we should catch the
// IOE and go into retrying for the meta mode. We'll do gets on -ROOT- to
// get new meta location. Return something so this 'get' succeeds
// (here we mock up getRegionServerWithRetries, the wrapper around
// the actual get).
// TODO: Refactor. This method has been moved out of HConnection.
// It works for now but has been deprecated.
Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>) Mockito.any()))
.thenReturn(getMetaTableRowResult());
// Now start up the catalogtracker with our doctored Connection.
final CatalogTracker ct = constructAndStartCatalogTracker(connection);
try {
// Set a location for root and meta.
RootLocationEditor.setRootLocation(this.watcher, SN);
ct.setMetaLocation(SN);
// Call the method that HBASE-4288 calls. It will try and verify the
// meta location and will fail on first attempt then go into a long wait.
// So, do this in a thread and then reset meta location to break it out
// of its wait after a bit of time.
final AtomicBoolean metaSet = new AtomicBoolean(false);
final CountDownLatch latch = new CountDownLatch(1);
Thread t =
new Thread() {
@Override
public void run() {
try {
latch.countDown();
metaSet.set(ct.waitForMeta(100000) != null);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
t.start();
latch.await();
Threads.sleep(1);
// Now reset the meta as though it were redeployed.
ct.setMetaLocation(SN);
t.join();
Assert.assertTrue(metaSet.get());
} finally {
// Clean out root and meta locations or later tests will be confused...
// they presume start fresh in zk.
ct.resetMetaLocation();
}
}