Example #1
0
/** Simple testing of a few WAL methods. */
@Category({RegionServerTests.class, SmallTests.class})
public class TestWALMethods {
  private static final byte[] TEST_REGION = Bytes.toBytes("test_region");;
  private static final TableName TEST_TABLE = TableName.valueOf("test_table");

  private final HBaseTestingUtility util = new HBaseTestingUtility();

  /**
   * Assert that getSplitEditFilesSorted returns files in expected order and that it skips
   * moved-aside files.
   *
   * @throws IOException
   */
  @Test
  public void testGetSplitEditFilesSorted() throws IOException {
    FileSystem fs = FileSystem.get(util.getConfiguration());
    Path regiondir = util.getDataTestDir("regiondir");
    fs.delete(regiondir, true);
    fs.mkdirs(regiondir);
    Path recoverededits = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
    String first = WALSplitter.formatRecoveredEditsFileName(-1);
    createFile(fs, recoverededits, first);
    createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(0));
    createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(1));
    createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(11));
    createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(2));
    createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(50));
    String last = WALSplitter.formatRecoveredEditsFileName(Long.MAX_VALUE);
    createFile(fs, recoverededits, last);
    createFile(
        fs, recoverededits, Long.toString(Long.MAX_VALUE) + "." + System.currentTimeMillis());

    final Configuration walConf = new Configuration(util.getConfiguration());
    FSUtils.setRootDir(walConf, regiondir);
    (new WALFactory(walConf, null, "dummyLogName")).getWAL(new byte[] {}, null);

    NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regiondir);
    assertEquals(7, files.size());
    assertEquals(files.pollFirst().getName(), first);
    assertEquals(files.pollLast().getName(), last);
    assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(0));
    assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(1));
    assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(2));
    assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(11));
  }

  private void createFile(final FileSystem fs, final Path testdir, final String name)
      throws IOException {
    FSDataOutputStream fdos = fs.create(new Path(testdir, name), true);
    fdos.close();
  }

  @Test
  public void testRegionEntryBuffer() throws Exception {
    WALSplitter.RegionEntryBuffer reb = new WALSplitter.RegionEntryBuffer(TEST_TABLE, TEST_REGION);
    assertEquals(0, reb.heapSize());

    reb.appendEntry(createTestLogEntry(1));
    assertTrue(reb.heapSize() > 0);
  }

  @Test
  public void testEntrySink() throws Exception {
    Configuration conf = new Configuration();
    RecoveryMode mode =
        (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)
            ? RecoveryMode.LOG_REPLAY
            : RecoveryMode.LOG_SPLITTING);

    EntryBuffers sink = new EntryBuffers(new PipelineController(), 1 * 1024 * 1024);
    for (int i = 0; i < 1000; i++) {
      WAL.Entry entry = createTestLogEntry(i);
      sink.appendEntry(entry);
    }

    assertTrue(sink.totalBuffered > 0);
    long amountInChunk = sink.totalBuffered;
    // Get a chunk
    RegionEntryBuffer chunk = sink.getChunkToWrite();
    assertEquals(chunk.heapSize(), amountInChunk);

    // Make sure it got marked that a thread is "working on this"
    assertTrue(sink.isRegionCurrentlyWriting(TEST_REGION));

    // Insert some more entries
    for (int i = 0; i < 500; i++) {
      WAL.Entry entry = createTestLogEntry(i);
      sink.appendEntry(entry);
    }
    // Asking for another chunk shouldn't work since the first one
    // is still writing
    assertNull(sink.getChunkToWrite());

    // If we say we're done writing the first chunk, then we should be able
    // to get the second
    sink.doneWriting(chunk);

    RegionEntryBuffer chunk2 = sink.getChunkToWrite();
    assertNotNull(chunk2);
    assertNotSame(chunk, chunk2);
    long amountInChunk2 = sink.totalBuffered;
    // The second chunk had fewer rows than the first
    assertTrue(amountInChunk2 < amountInChunk);

    sink.doneWriting(chunk2);
    assertEquals(0, sink.totalBuffered);
  }

  private WAL.Entry createTestLogEntry(int i) {
    long seq = i;
    long now = i * 1000;

    WALEdit edit = new WALEdit();
    edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val"));
    WALKey key = new WALKey(TEST_REGION, TEST_TABLE, seq, now, HConstants.DEFAULT_CLUSTER_ID);
    WAL.Entry entry = new WAL.Entry(key, edit);
    return entry;
  }
}
Example #2
0
@Category(MediumTests.class)
public class TestScannerResource {
  private static final TableName TABLE = TableName.valueOf("TestScannerResource");
  private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
  private static final String CFA = "a";
  private static final String CFB = "b";
  private static final String COLUMN_1 = CFA + ":1";
  private static final String COLUMN_2 = CFB + ":2";

  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility();
  private static Client client;
  private static JAXBContext context;
  private static Marshaller marshaller;
  private static Unmarshaller unmarshaller;
  private static int expectedRows1;
  private static int expectedRows2;
  private static Configuration conf;

  static int insertData(Configuration conf, TableName tableName, String column, double prob)
      throws IOException {
    Random rng = new Random();
    int count = 0;
    HTable table = new HTable(conf, tableName);
    byte[] k = new byte[3];
    byte[][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
    for (byte b1 = 'a'; b1 < 'z'; b1++) {
      for (byte b2 = 'a'; b2 < 'z'; b2++) {
        for (byte b3 = 'a'; b3 < 'z'; b3++) {
          if (rng.nextDouble() < prob) {
            k[0] = b1;
            k[1] = b2;
            k[2] = b3;
            Put put = new Put(k);
            put.setDurability(Durability.SKIP_WAL);
            put.add(famAndQf[0], famAndQf[1], k);
            table.put(put);
            count++;
          }
        }
      }
    }
    table.flushCommits();
    table.close();
    return count;
  }

  static int countCellSet(CellSetModel model) {
    int count = 0;
    Iterator<RowModel> rows = model.getRows().iterator();
    while (rows.hasNext()) {
      RowModel row = rows.next();
      Iterator<CellModel> cells = row.getCells().iterator();
      while (cells.hasNext()) {
        cells.next();
        count++;
      }
    }
    return count;
  }

  private static int fullTableScan(ScannerModel model) throws IOException {
    model.setBatch(100);
    Response response =
        client.put(
            "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
    assertEquals(response.getCode(), 201);
    String scannerURI = response.getLocation();
    assertNotNull(scannerURI);
    int count = 0;
    while (true) {
      response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
      assertTrue(response.getCode() == 200 || response.getCode() == 204);
      if (response.getCode() == 200) {
        assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
        CellSetModel cellSet = new CellSetModel();
        cellSet.getObjectFromMessage(response.getBody());
        Iterator<RowModel> rows = cellSet.getRows().iterator();
        while (rows.hasNext()) {
          RowModel row = rows.next();
          Iterator<CellModel> cells = row.getCells().iterator();
          while (cells.hasNext()) {
            cells.next();
            count++;
          }
        }
      } else {
        break;
      }
    }
    // delete the scanner
    response = client.delete(scannerURI);
    assertEquals(response.getCode(), 200);
    return count;
  }

  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    conf = TEST_UTIL.getConfiguration();
    TEST_UTIL.startMiniCluster();
    REST_TEST_UTIL.startServletContainer(conf);
    client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
    context =
        JAXBContext.newInstance(
            CellModel.class, CellSetModel.class, RowModel.class, ScannerModel.class);
    marshaller = context.createMarshaller();
    unmarshaller = context.createUnmarshaller();
    Admin admin = TEST_UTIL.getHBaseAdmin();
    if (admin.tableExists(TABLE)) {
      return;
    }
    HTableDescriptor htd = new HTableDescriptor(TABLE);
    htd.addFamily(new HColumnDescriptor(CFA));
    htd.addFamily(new HColumnDescriptor(CFB));
    admin.createTable(htd);
    expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
    expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
  }

  @AfterClass
  public static void tearDownAfterClass() throws Exception {
    REST_TEST_UTIL.shutdownServletContainer();
    TEST_UTIL.shutdownMiniCluster();
  }

  @Test
  public void testSimpleScannerXML() throws IOException, JAXBException {
    final int BATCH_SIZE = 5;
    // new scanner
    ScannerModel model = new ScannerModel();
    model.setBatch(BATCH_SIZE);
    model.addColumn(Bytes.toBytes(COLUMN_1));
    StringWriter writer = new StringWriter();
    marshaller.marshal(model, writer);
    byte[] body = Bytes.toBytes(writer.toString());

    // test put operation is forbidden in read-only mode
    conf.set("hbase.rest.readonly", "true");
    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
    assertEquals(response.getCode(), 403);
    String scannerURI = response.getLocation();
    assertNull(scannerURI);

    // recall previous put operation with read-only off
    conf.set("hbase.rest.readonly", "false");
    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
    assertEquals(response.getCode(), 201);
    scannerURI = response.getLocation();
    assertNotNull(scannerURI);

    // get a cell set
    response = client.get(scannerURI, Constants.MIMETYPE_XML);
    assertEquals(response.getCode(), 200);
    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
    CellSetModel cellSet =
        (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
    // confirm batch size conformance
    assertEquals(countCellSet(cellSet), BATCH_SIZE);

    // test delete scanner operation is forbidden in read-only mode
    conf.set("hbase.rest.readonly", "true");
    response = client.delete(scannerURI);
    assertEquals(response.getCode(), 403);

    // recall previous delete scanner operation with read-only off
    conf.set("hbase.rest.readonly", "false");
    response = client.delete(scannerURI);
    assertEquals(response.getCode(), 200);
  }

  @Test
  public void testSimpleScannerPB() throws IOException {
    final int BATCH_SIZE = 10;
    // new scanner
    ScannerModel model = new ScannerModel();
    model.setBatch(BATCH_SIZE);
    model.addColumn(Bytes.toBytes(COLUMN_1));

    // test put operation is forbidden in read-only mode
    conf.set("hbase.rest.readonly", "true");
    Response response =
        client.put(
            "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
    assertEquals(response.getCode(), 403);
    String scannerURI = response.getLocation();
    assertNull(scannerURI);

    // recall previous put operation with read-only off
    conf.set("hbase.rest.readonly", "false");
    response =
        client.put(
            "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
    assertEquals(response.getCode(), 201);
    scannerURI = response.getLocation();
    assertNotNull(scannerURI);

    // get a cell set
    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
    assertEquals(response.getCode(), 200);
    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
    CellSetModel cellSet = new CellSetModel();
    cellSet.getObjectFromMessage(response.getBody());
    // confirm batch size conformance
    assertEquals(countCellSet(cellSet), BATCH_SIZE);

    // test delete scanner operation is forbidden in read-only mode
    conf.set("hbase.rest.readonly", "true");
    response = client.delete(scannerURI);
    assertEquals(response.getCode(), 403);

    // recall previous delete scanner operation with read-only off
    conf.set("hbase.rest.readonly", "false");
    response = client.delete(scannerURI);
    assertEquals(response.getCode(), 200);
  }

  @Test
  public void testSimpleScannerBinary() throws IOException {
    // new scanner
    ScannerModel model = new ScannerModel();
    model.setBatch(1);
    model.addColumn(Bytes.toBytes(COLUMN_1));

    // test put operation is forbidden in read-only mode
    conf.set("hbase.rest.readonly", "true");
    Response response =
        client.put(
            "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
    assertEquals(response.getCode(), 403);
    String scannerURI = response.getLocation();
    assertNull(scannerURI);

    // recall previous put operation with read-only off
    conf.set("hbase.rest.readonly", "false");
    response =
        client.put(
            "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
    assertEquals(response.getCode(), 201);
    scannerURI = response.getLocation();
    assertNotNull(scannerURI);

    // get a cell
    response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
    assertEquals(response.getCode(), 200);
    assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
    // verify that data was returned
    assertTrue(response.getBody().length > 0);
    // verify that the expected X-headers are present
    boolean foundRowHeader = false, foundColumnHeader = false, foundTimestampHeader = false;
    for (Header header : response.getHeaders()) {
      if (header.getName().equals("X-Row")) {
        foundRowHeader = true;
      } else if (header.getName().equals("X-Column")) {
        foundColumnHeader = true;
      } else if (header.getName().equals("X-Timestamp")) {
        foundTimestampHeader = true;
      }
    }
    assertTrue(foundRowHeader);
    assertTrue(foundColumnHeader);
    assertTrue(foundTimestampHeader);

    // test delete scanner operation is forbidden in read-only mode
    conf.set("hbase.rest.readonly", "true");
    response = client.delete(scannerURI);
    assertEquals(response.getCode(), 403);

    // recall previous delete scanner operation with read-only off
    conf.set("hbase.rest.readonly", "false");
    response = client.delete(scannerURI);
    assertEquals(response.getCode(), 200);
  }

  @Test
  public void testFullTableScan() throws IOException {
    ScannerModel model = new ScannerModel();
    model.addColumn(Bytes.toBytes(COLUMN_1));
    assertEquals(fullTableScan(model), expectedRows1);

    model = new ScannerModel();
    model.addColumn(Bytes.toBytes(COLUMN_2));
    assertEquals(fullTableScan(model), expectedRows2);
  }

  @Test
  public void testTableDoesNotExist() throws IOException, JAXBException {
    ScannerModel model = new ScannerModel();
    StringWriter writer = new StringWriter();
    marshaller.marshal(model, writer);
    byte[] body = Bytes.toBytes(writer.toString());
    Response response =
        client.put("/" + NONEXISTENT_TABLE + "/scanner", Constants.MIMETYPE_XML, body);
    assertEquals(response.getCode(), 404);
  }
}