예제 #1
0
 /**
  * Returns the daughter regions by reading from the corresponding columns of the .META. table
  * Result. If the region is not a split parent region, it returns PairOfSameType(null, null).
  */
 public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
   HRegionInfo splitA =
       Writables.getHRegionInfoOrNull(
           data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER));
   HRegionInfo splitB =
       Writables.getHRegionInfoOrNull(
           data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER));
   return new PairOfSameType<HRegionInfo>(splitA, splitB);
 }
예제 #2
0
 /**
  * Offline parent in meta. Used when splitting.
  *
  * @param catalogTracker
  * @param parent
  * @param a Split daughter region A
  * @param b Split daughter region B
  * @throws NotAllMetaRegionsOnlineException
  * @throws IOException
  */
 public static void offlineParentInMeta(
     CatalogTracker catalogTracker, HRegionInfo parent, final HRegionInfo a, final HRegionInfo b)
     throws NotAllMetaRegionsOnlineException, IOException {
   HRegionInfo copyOfParent = new HRegionInfo(parent);
   copyOfParent.setOffline(true);
   copyOfParent.setSplit(true);
   Put put = new Put(copyOfParent.getRegionName());
   addRegionInfo(put, copyOfParent);
   put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, Writables.getBytes(a));
   put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, Writables.getBytes(b));
   putToMetaTable(catalogTracker, put);
   LOG.info("Offlined parent region " + parent.getRegionNameAsString() + " in META");
 }
예제 #3
0
 public static HRegionInfo getHRegionInfo(Result data) throws IOException {
   byte[] bytes = data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
   if (bytes == null) return null;
   HRegionInfo info = Writables.getHRegionInfo(bytes);
   LOG.info("Current INFO from scan results = " + info);
   return info;
 }
예제 #4
0
  /*
   * Test that {@link HFileOutputFormat2} creates an HFile with TIMERANGE
   * metadata used by time-restricted scans.
   */
  @Test
  public void test_TIMERANGE() throws Exception {
    Configuration conf = new Configuration(this.util.getConfiguration());
    RecordWriter<ImmutableBytesWritable, Cell> writer = null;
    TaskAttemptContext context = null;
    Path dir = util.getDataTestDir("test_TIMERANGE_present");
    LOG.info("Timerange dir writing to dir: " + dir);
    try {
      // build a record writer using HFileOutputFormat2
      Job job = new Job(conf);
      FileOutputFormat.setOutputPath(job, dir);
      context = createTestTaskAttemptContext(job);
      HFileOutputFormat2 hof = new HFileOutputFormat2();
      writer = hof.getRecordWriter(context);

      // Pass two key values with explicit times stamps
      final byte[] b = Bytes.toBytes("b");

      // value 1 with timestamp 2000
      KeyValue kv = new KeyValue(b, b, b, 2000, b);
      KeyValue original = kv.clone();
      writer.write(new ImmutableBytesWritable(), kv);
      assertEquals(original, kv);

      // value 2 with timestamp 1000
      kv = new KeyValue(b, b, b, 1000, b);
      original = kv.clone();
      writer.write(new ImmutableBytesWritable(), kv);
      assertEquals(original, kv);

      // verify that the file has the proper FileInfo.
      writer.close(context);

      // the generated file lives 1 directory down from the attempt directory
      // and is the only file, e.g.
      // _attempt__0000_r_000000_0/b/1979617994050536795
      FileSystem fs = FileSystem.get(conf);
      Path attemptDirectory = hof.getDefaultWorkFile(context, "").getParent();
      FileStatus[] sub1 = fs.listStatus(attemptDirectory);
      FileStatus[] file = fs.listStatus(sub1[0].getPath());

      // open as HFile Reader and pull out TIMERANGE FileInfo.
      HFile.Reader rd = HFile.createReader(fs, file[0].getPath(), new CacheConfig(conf), conf);
      Map<byte[], byte[]> finfo = rd.loadFileInfo();
      byte[] range = finfo.get("TIMERANGE".getBytes());
      assertNotNull(range);

      // unmarshall and check values.
      TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
      Writables.copyWritable(range, timeRangeTracker);
      LOG.info(
          timeRangeTracker.getMinimumTimestamp() + "...." + timeRangeTracker.getMaximumTimestamp());
      assertEquals(1000, timeRangeTracker.getMinimumTimestamp());
      assertEquals(2000, timeRangeTracker.getMaximumTimestamp());
      rd.close();
    } finally {
      if (writer != null && context != null) writer.close(context);
      dir.getFileSystem(conf).delete(dir, true);
    }
  }
예제 #5
0
 /*
  * Add to each of the regions in .META. a value.  Key is the startrow of the
  * region (except its 'aaa' for first region).  Actual value is the row name.
  * @param expected
  * @return
  * @throws IOException
  */
 private static int addToEachStartKey(final int expected) throws IOException {
   HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
   HTable meta = new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
   int rows = 0;
   Scan scan = new Scan();
   scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
   ResultScanner s = meta.getScanner(scan);
   for (Result r = null; (r = s.next()) != null; ) {
     byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
     if (b == null || b.length <= 0) break;
     HRegionInfo hri = Writables.getHRegionInfo(b);
     // If start key, add 'aaa'.
     byte[] row = getStartKey(hri);
     Put p = new Put(row);
     p.setWriteToWAL(false);
     p.add(getTestFamily(), getTestQualifier(), row);
     t.put(p);
     rows++;
   }
   s.close();
   Assert.assertEquals(expected, rows);
   t.close();
   meta.close();
   return rows;
 }
 public static MultiCQKeyValueComparisonFilter parseFrom(final byte[] pbBytes)
     throws DeserializationException {
   try {
     return (MultiCQKeyValueComparisonFilter)
         Writables.getWritable(pbBytes, new MultiCQKeyValueComparisonFilter());
   } catch (IOException e) {
     throw new DeserializationException(e);
   }
 }
예제 #7
0
 /**
  * @return A mocked up Result that fakes a Get on a row in the <code>.META.</code> table.
  * @throws IOException
  */
 private Result getMetaTableRowResult() throws IOException {
   List<KeyValue> kvs = new ArrayList<KeyValue>();
   kvs.add(
       new KeyValue(
           HConstants.EMPTY_BYTE_ARRAY,
           HConstants.CATALOG_FAMILY,
           HConstants.REGIONINFO_QUALIFIER,
           Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
   kvs.add(
       new KeyValue(
           HConstants.EMPTY_BYTE_ARRAY,
           HConstants.CATALOG_FAMILY,
           HConstants.SERVER_QUALIFIER,
           Bytes.toBytes(SN.getHostAndPort())));
   kvs.add(
       new KeyValue(
           HConstants.EMPTY_BYTE_ARRAY,
           HConstants.CATALOG_FAMILY,
           HConstants.STARTCODE_QUALIFIER,
           Bytes.toBytes(SN.getStartcode())));
   return new Result(kvs);
 }
예제 #8
0
 private static Put makePutFromRegionInfo(HRegionInfo regionInfo) throws IOException {
   Put put = new Put(regionInfo.getRegionName());
   put.add(
       HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(regionInfo));
   return put;
 }
예제 #9
0
 private static Put addRegionInfo(final Put p, final HRegionInfo hri) throws IOException {
   p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(hri));
   return p;
 }
  /**
   * Test that MetaReader will ride over server throwing "Server not running" IOEs.
   *
   * @see https://issues.apache.org/jira/browse/HBASE-3446
   * @throws IOException
   * @throws InterruptedException
   */
  @Test
  public void testRideOverServerNotRunning() throws IOException, InterruptedException {
    // Need a zk watcher.
    ZooKeeperWatcher zkw =
        new ZooKeeperWatcher(
            UTIL.getConfiguration(), this.getClass().getSimpleName(), ABORTABLE, true);
    // This is a servername we use in a few places below.
    ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis());

    HConnection connection = null;
    CatalogTracker ct = null;
    try {
      // Mock an HRegionInterface. Our mock implementation will fail a few
      // times when we go to open a scanner.
      final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
      // When openScanner called throw IOE 'Server not running' a few times
      // before we return a scanner id.  Whats WEIRD is that these
      // exceptions do not show in the log because they are caught and only
      // printed if we FAIL.  We eventually succeed after retry so these don't
      // show.  We will know if they happened or not because we will ask
      // mockito at the end of this test to verify that openscanner was indeed
      // called the wanted number of times.
      final long scannerid = 123L;
      Mockito.when(implementation.openScanner((byte[]) Mockito.any(), (Scan) Mockito.any()))
          .thenThrow(new IOException("Server not running (1 of 3)"))
          .thenThrow(new IOException("Server not running (2 of 3)"))
          .thenThrow(new IOException("Server not running (3 of 3)"))
          .thenReturn(scannerid);
      // Make it so a verifiable answer comes back when next is called.  Return
      // the verifiable answer and then a null so we stop scanning.  Our
      // verifiable answer is something that looks like a row in META with
      // a server and startcode that is that of the above defined servername.
      List<KeyValue> kvs = new ArrayList<KeyValue>();
      final byte[] rowToVerify = Bytes.toBytes("rowToVerify");
      kvs.add(
          new KeyValue(
              rowToVerify,
              HConstants.CATALOG_FAMILY,
              HConstants.REGIONINFO_QUALIFIER,
              Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
      kvs.add(
          new KeyValue(
              rowToVerify,
              HConstants.CATALOG_FAMILY,
              HConstants.SERVER_QUALIFIER,
              Bytes.toBytes(sn.getHostAndPort())));
      kvs.add(
          new KeyValue(
              rowToVerify,
              HConstants.CATALOG_FAMILY,
              HConstants.STARTCODE_QUALIFIER,
              Bytes.toBytes(sn.getStartcode())));
      final Result[] result = new Result[] {new Result(kvs)};
      Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt()))
          .thenReturn(result)
          .thenReturn(null);

      // Associate a spied-upon HConnection with UTIL.getConfiguration.  Need
      // to shove this in here first so it gets picked up all over; e.g. by
      // HTable.
      connection = HConnectionTestingUtility.getSpiedConnection(UTIL.getConfiguration());
      // Fix the location lookup so it 'works' though no network.  First
      // make an 'any location' object.
      final HRegionLocation anyLocation =
          new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(), sn.getPort());
      // Return the any location object when locateRegion is called in HTable
      // constructor and when its called by ServerCallable (it uses getRegionLocation).
      // The ugly format below comes of 'Important gotcha on spying real objects!' from
      // http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html
      Mockito.doReturn(anyLocation)
          .when(connection)
          .locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any());
      Mockito.doReturn(anyLocation)
          .when(connection)
          .getRegionLocation((byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean());

      // Now shove our HRI implementation into the spied-upon connection.
      Mockito.doReturn(implementation)
          .when(connection)
          .getHRegionConnection(Mockito.anyString(), Mockito.anyInt());

      // Now start up the catalogtracker with our doctored Connection.
      ct = new CatalogTracker(zkw, null, connection, ABORTABLE, 0);
      ct.start();
      // Scan meta for user tables and verify we got back expected answer.
      NavigableMap<HRegionInfo, Result> hris = MetaReader.getServerUserRegions(ct, sn);
      assertTrue(hris.size() == 1);
      assertTrue(hris.firstEntry().getKey().equals(HRegionInfo.FIRST_META_REGIONINFO));
      assertTrue(Bytes.equals(rowToVerify, hris.firstEntry().getValue().getRow()));
      // Finally verify that openscanner was called four times -- three times
      // with exception and then on 4th attempt we succeed.
      Mockito.verify(implementation, Mockito.times(4))
          .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any());
    } finally {
      if (ct != null) ct.stop();
      HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
      zkw.close();
    }
  }
예제 #11
0
 /**
  * Parse the content of the cell at {@link HConstants#CATALOG_FAMILY} and <code>qualifier</code>
  * as an HRegionInfo and return it, or null. For use on catalog table {@link Result}.
  *
  * @param r Result instance to pull from.
  * @param qualifier Column family qualifier -- either {@link HConstants#SPLITA_QUALIFIER}, {@link
  *     HConstants#SPLITB_QUALIFIER} or {@link HConstants#REGIONINFO_QUALIFIER}.
  * @return An HRegionInfo instance or null.
  * @throws IOException
  */
 public static HRegionInfo parseHRegionInfoFromCatalogResult(final Result r, byte[] qualifier)
     throws IOException {
   byte[] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
   if (bytes == null || bytes.length <= 0) return null;
   return Writables.getHRegionInfoOrNull(bytes);
 }