/**
   * Test interruptable while blocking wait on root and meta.
   *
   * @throws IOException
   * @throws InterruptedException
   */
  @Test
  public void testInterruptWaitOnMetaAndRoot() throws IOException, InterruptedException {
    HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
    HConnection connection = mockConnection(implementation);

    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ServerName hsa = ct.getRootLocation();
    Assert.assertNull(hsa);
    ServerName meta = ct.getMetaLocation();
    Assert.assertNull(meta);
    Thread t =
        new Thread() {
          @Override
          public void run() {
            try {
              ct.waitForMeta();
            } catch (InterruptedException e) {
              throw new RuntimeException("Interrupted", e);
            }
          }
        };
    t.start();
    while (!t.isAlive()) Threads.sleep(1);
    Threads.sleep(1);
    assertTrue(t.isAlive());
    ct.stop();
    // Join the thread... should exit shortly.
    t.join();
  }
 private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
     throws IOException, InterruptedException {
   CatalogTracker ct =
       new CatalogTracker(this.watcher, UTIL.getConfiguration(), c, this.abortable);
   ct.start();
   return ct;
 }
  /**
   * Test waiting on meta w/ no timeout specified.
   *
   * @throws Exception
   */
  @Ignore // Can't make it work reliably on all platforms; mockito gets confused
  // Throwing: org.mockito.exceptions.misusing.WrongTypeOfReturnValue:
  // Result cannot be returned by locateRegion()
  // If you plug locateRegion, it then throws for incCounter, and if you plug
  // that ... and so one.
  @Test
  public void testNoTimeoutWaitForMeta() throws Exception {
    // Mock an HConnection and a HRegionInterface implementation.  Have the
    // HConnection return the HRI.  Have the HRI return a few mocked up responses
    // to make our test work.
    // Mock an HRegionInterface.
    final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
    HConnection connection = mockConnection(implementation);

    // Now the ct is up... set into the mocks some answers that make it look
    // like things have been getting assigned. Make it so we'll return a
    // location (no matter what the Get is). Same for getHRegionInfo -- always
    // just return the meta region.
    final Result result = getMetaTableRowResult();

    // TODO: Refactor.  This method has been moved out of HConnection.
    // It works for now but has been deprecated.
    Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>) Mockito.any()))
        .thenReturn(result);
    Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any()))
        .thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ServerName hsa = ct.getMetaLocation();
    Assert.assertNull(hsa);

    // Now test waiting on meta location getting set.
    Thread t =
        new WaitOnMetaThread(ct) {
          @Override
          void doWaiting() throws InterruptedException {
            this.ct.waitForMeta();
          }
        };
    startWaitAliveThenWaitItLives(t, 1000);

    // This should trigger wake up of meta wait (Its the removal of the meta
    // region unassigned node that triggers catalogtrackers that a meta has
    // been assigned).
    String node = ct.getMetaNodeTracker().getNode();
    ZKUtil.createAndFailSilent(this.watcher, node);
    MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
    ZKUtil.deleteNode(this.watcher, node);
    // Go get the new meta location. waitForMeta gets and verifies meta.
    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
    // Join the thread... should exit shortly.
    t.join();
    // Now meta is available.
    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
  }
  private void testVerifyMetaRegionLocationWithException(Exception ex)
      throws IOException, InterruptedException, KeeperException {
    // Mock an HRegionInterface.
    final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
    HConnection connection = mockConnection(implementation);

    // If a 'get' is called on mocked interface, throw connection refused.
    Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenThrow(ex);
    // Now start up the catalogtracker with our doctored Connection.
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    RootLocationEditor.setRootLocation(this.watcher, SN);
    long timeout = UTIL.getConfiguration().getLong("hbase.catalog.verification.timeout", 1000);
    Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
  }
 /**
  * Test get of root region fails properly if nothing to connect to.
  *
  * @throws IOException
  * @throws InterruptedException
  * @throws KeeperException
  */
 @Test
 public void testVerifyRootRegionLocationFails()
     throws IOException, InterruptedException, KeeperException {
   HConnection connection = Mockito.mock(HConnection.class);
   ConnectException connectException = new ConnectException("Connection refused");
   final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
   Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).thenThrow(connectException);
   Mockito.when(
           connection.getHRegionConnection(
               Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean()))
       .thenReturn(implementation);
   final CatalogTracker ct = constructAndStartCatalogTracker(connection);
   RootLocationEditor.setRootLocation(
       this.watcher, new ServerName("example.com", 1234, System.currentTimeMillis()));
   Assert.assertFalse(ct.verifyRootRegionLocation(100));
 }
  /**
   * Test waiting on root w/ no timeout specified.
   *
   * @throws IOException
   * @throws InterruptedException
   * @throws KeeperException
   */
  @Test
  public void testNoTimeoutWaitForRoot() throws IOException, InterruptedException, KeeperException {
    HConnection connection = Mockito.mock(HConnection.class);
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ServerName hsa = ct.getRootLocation();
    Assert.assertNull(hsa);

    // Now test waiting on root location getting set.
    Thread t = new WaitOnMetaThread(ct);
    startWaitAliveThenWaitItLives(t, 1000);
    // Set a root location.
    hsa = setRootLocation();
    // Join the thread... should exit shortly.
    t.join();
    // Now root is available.
    Assert.assertTrue(ct.getRootLocation().equals(hsa));
  }
  /**
   * Test that MetaReader will ride over server throwing "Server not running" IOEs.
   *
   * @see https://issues.apache.org/jira/browse/HBASE-3446
   * @throws IOException
   * @throws InterruptedException
   */
  @Test
  public void testRideOverServerNotRunning() throws IOException, InterruptedException {
    // Need a zk watcher.
    ZooKeeperWatcher zkw =
        new ZooKeeperWatcher(
            UTIL.getConfiguration(), this.getClass().getSimpleName(), ABORTABLE, true);
    // This is a servername we use in a few places below.
    ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis());

    HConnection connection = null;
    CatalogTracker ct = null;
    try {
      // Mock an HRegionInterface. Our mock implementation will fail a few
      // times when we go to open a scanner.
      final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
      // When openScanner called throw IOE 'Server not running' a few times
      // before we return a scanner id.  Whats WEIRD is that these
      // exceptions do not show in the log because they are caught and only
      // printed if we FAIL.  We eventually succeed after retry so these don't
      // show.  We will know if they happened or not because we will ask
      // mockito at the end of this test to verify that openscanner was indeed
      // called the wanted number of times.
      final long scannerid = 123L;
      Mockito.when(implementation.openScanner((byte[]) Mockito.any(), (Scan) Mockito.any()))
          .thenThrow(new IOException("Server not running (1 of 3)"))
          .thenThrow(new IOException("Server not running (2 of 3)"))
          .thenThrow(new IOException("Server not running (3 of 3)"))
          .thenReturn(scannerid);
      // Make it so a verifiable answer comes back when next is called.  Return
      // the verifiable answer and then a null so we stop scanning.  Our
      // verifiable answer is something that looks like a row in META with
      // a server and startcode that is that of the above defined servername.
      List<KeyValue> kvs = new ArrayList<KeyValue>();
      final byte[] rowToVerify = Bytes.toBytes("rowToVerify");
      kvs.add(
          new KeyValue(
              rowToVerify,
              HConstants.CATALOG_FAMILY,
              HConstants.REGIONINFO_QUALIFIER,
              Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
      kvs.add(
          new KeyValue(
              rowToVerify,
              HConstants.CATALOG_FAMILY,
              HConstants.SERVER_QUALIFIER,
              Bytes.toBytes(sn.getHostAndPort())));
      kvs.add(
          new KeyValue(
              rowToVerify,
              HConstants.CATALOG_FAMILY,
              HConstants.STARTCODE_QUALIFIER,
              Bytes.toBytes(sn.getStartcode())));
      final Result[] result = new Result[] {new Result(kvs)};
      Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt()))
          .thenReturn(result)
          .thenReturn(null);

      // Associate a spied-upon HConnection with UTIL.getConfiguration.  Need
      // to shove this in here first so it gets picked up all over; e.g. by
      // HTable.
      connection = HConnectionTestingUtility.getSpiedConnection(UTIL.getConfiguration());
      // Fix the location lookup so it 'works' though no network.  First
      // make an 'any location' object.
      final HRegionLocation anyLocation =
          new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(), sn.getPort());
      // Return the any location object when locateRegion is called in HTable
      // constructor and when its called by ServerCallable (it uses getRegionLocation).
      // The ugly format below comes of 'Important gotcha on spying real objects!' from
      // http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html
      Mockito.doReturn(anyLocation)
          .when(connection)
          .locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any());
      Mockito.doReturn(anyLocation)
          .when(connection)
          .getRegionLocation((byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean());

      // Now shove our HRI implementation into the spied-upon connection.
      Mockito.doReturn(implementation)
          .when(connection)
          .getHRegionConnection(Mockito.anyString(), Mockito.anyInt());

      // Now start up the catalogtracker with our doctored Connection.
      ct = new CatalogTracker(zkw, null, connection, ABORTABLE, 0);
      ct.start();
      // Scan meta for user tables and verify we got back expected answer.
      NavigableMap<HRegionInfo, Result> hris = MetaReader.getServerUserRegions(ct, sn);
      assertTrue(hris.size() == 1);
      assertTrue(hris.firstEntry().getKey().equals(HRegionInfo.FIRST_META_REGIONINFO));
      assertTrue(Bytes.equals(rowToVerify, hris.firstEntry().getValue().getRow()));
      // Finally verify that openscanner was called four times -- three times
      // with exception and then on 4th attempt we succeed.
      Mockito.verify(implementation, Mockito.times(4))
          .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any());
    } finally {
      if (ct != null) ct.stop();
      HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
      zkw.close();
    }
  }
 @Test(expected = RetriesExhaustedException.class)
 public void testTimeoutWaitForMeta() throws IOException, InterruptedException {
   HConnection connection = HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
   final CatalogTracker ct = constructAndStartCatalogTracker(connection);
   ct.waitForMeta(100);
 }
 @Test(expected = NotAllMetaRegionsOnlineException.class)
 public void testTimeoutWaitForRoot() throws IOException, InterruptedException {
   HConnection connection = Mockito.mock(HConnection.class);
   final CatalogTracker ct = constructAndStartCatalogTracker(connection);
   ct.waitForRoot(100);
 }
  /**
   * Test for HBASE-4288. Throw an IOE when trying to verify meta region and prove it doesn't cause
   * master shutdown.
   *
   * @see <a href="https://issues.apache.org/jira/browse/HBASE-4288">HBASE-4288</a>
   * @throws IOException
   * @throws InterruptedException
   * @throws KeeperException
   */
  @Test
  public void testServerNotRunningIOException()
      throws IOException, InterruptedException, KeeperException {
    // Mock an HRegionInterface.
    final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
    HConnection connection = mockConnection(implementation);

    // If a 'getRegionInfo' is called on mocked HRegionInterface, throw IOE
    // the first time.  'Succeed' the second time we are called.
    Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any()))
        .thenThrow(new IOException("Server not running, aborting"))
        .thenReturn(new HRegionInfo());

    // After we encounter the above 'Server not running', we should catch the
    // IOE and go into retrying for the meta mode.  We'll do gets on -ROOT- to
    // get new meta location.  Return something so this 'get' succeeds
    // (here we mock up getRegionServerWithRetries, the wrapper around
    // the actual get).

    // TODO: Refactor.  This method has been moved out of HConnection.
    // It works for now but has been deprecated.
    Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>) Mockito.any()))
        .thenReturn(getMetaTableRowResult());

    // Now start up the catalogtracker with our doctored Connection.
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    try {
      // Set a location for root and meta.
      RootLocationEditor.setRootLocation(this.watcher, SN);
      ct.setMetaLocation(SN);
      // Call the method that HBASE-4288 calls.  It will try and verify the
      // meta location and will fail on first attempt then go into a long wait.
      // So, do this in a thread and then reset meta location to break it out
      // of its wait after a bit of time.
      final AtomicBoolean metaSet = new AtomicBoolean(false);
      final CountDownLatch latch = new CountDownLatch(1);
      Thread t =
          new Thread() {
            @Override
            public void run() {
              try {
                latch.countDown();
                metaSet.set(ct.waitForMeta(100000) != null);
              } catch (Exception e) {
                throw new RuntimeException(e);
              }
            }
          };
      t.start();
      latch.await();
      Threads.sleep(1);
      // Now reset the meta as though it were redeployed.
      ct.setMetaLocation(SN);
      t.join();
      Assert.assertTrue(metaSet.get());
    } finally {
      // Clean out root and meta locations or later tests will be confused...
      // they presume start fresh in zk.
      ct.resetMetaLocation();
    }
  }