/** Simple testing of a few WAL methods. */ @Category({RegionServerTests.class, SmallTests.class}) public class TestWALMethods { private static final byte[] TEST_REGION = Bytes.toBytes("test_region");; private static final TableName TEST_TABLE = TableName.valueOf("test_table"); private final HBaseTestingUtility util = new HBaseTestingUtility(); /** * Assert that getSplitEditFilesSorted returns files in expected order and that it skips * moved-aside files. * * @throws IOException */ @Test public void testGetSplitEditFilesSorted() throws IOException { FileSystem fs = FileSystem.get(util.getConfiguration()); Path regiondir = util.getDataTestDir("regiondir"); fs.delete(regiondir, true); fs.mkdirs(regiondir); Path recoverededits = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); String first = WALSplitter.formatRecoveredEditsFileName(-1); createFile(fs, recoverededits, first); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(0)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(1)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(11)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(2)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(50)); String last = WALSplitter.formatRecoveredEditsFileName(Long.MAX_VALUE); createFile(fs, recoverededits, last); createFile( fs, recoverededits, Long.toString(Long.MAX_VALUE) + "." + System.currentTimeMillis()); final Configuration walConf = new Configuration(util.getConfiguration()); FSUtils.setRootDir(walConf, regiondir); (new WALFactory(walConf, null, "dummyLogName")).getWAL(new byte[] {}, null); NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regiondir); assertEquals(7, files.size()); assertEquals(files.pollFirst().getName(), first); assertEquals(files.pollLast().getName(), last); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(0)); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(1)); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(2)); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(11)); } private void createFile(final FileSystem fs, final Path testdir, final String name) throws IOException { FSDataOutputStream fdos = fs.create(new Path(testdir, name), true); fdos.close(); } @Test public void testRegionEntryBuffer() throws Exception { WALSplitter.RegionEntryBuffer reb = new WALSplitter.RegionEntryBuffer(TEST_TABLE, TEST_REGION); assertEquals(0, reb.heapSize()); reb.appendEntry(createTestLogEntry(1)); assertTrue(reb.heapSize() > 0); } @Test public void testEntrySink() throws Exception { Configuration conf = new Configuration(); RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING); EntryBuffers sink = new EntryBuffers(new PipelineController(), 1 * 1024 * 1024); for (int i = 0; i < 1000; i++) { WAL.Entry entry = createTestLogEntry(i); sink.appendEntry(entry); } assertTrue(sink.totalBuffered > 0); long amountInChunk = sink.totalBuffered; // Get a chunk RegionEntryBuffer chunk = sink.getChunkToWrite(); assertEquals(chunk.heapSize(), amountInChunk); // Make sure it got marked that a thread is "working on this" assertTrue(sink.isRegionCurrentlyWriting(TEST_REGION)); // Insert some more entries for (int i = 0; i < 500; i++) { WAL.Entry entry = createTestLogEntry(i); sink.appendEntry(entry); } // Asking for another chunk shouldn't work since the first one // is still writing assertNull(sink.getChunkToWrite()); // If we say we're done writing the first chunk, then we should be able // to get the second sink.doneWriting(chunk); RegionEntryBuffer chunk2 = sink.getChunkToWrite(); assertNotNull(chunk2); assertNotSame(chunk, chunk2); long amountInChunk2 = sink.totalBuffered; // The second chunk had fewer rows than the first assertTrue(amountInChunk2 < amountInChunk); sink.doneWriting(chunk2); assertEquals(0, sink.totalBuffered); } private WAL.Entry createTestLogEntry(int i) { long seq = i; long now = i * 1000; WALEdit edit = new WALEdit(); edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val")); WALKey key = new WALKey(TEST_REGION, TEST_TABLE, seq, now, HConstants.DEFAULT_CLUSTER_ID); WAL.Entry entry = new WAL.Entry(key, edit); return entry; } }
@Category(MediumTests.class) public class TestScannerResource { private static final TableName TABLE = TableName.valueOf("TestScannerResource"); private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist"; private static final String CFA = "a"; private static final String CFB = "b"; private static final String COLUMN_1 = CFA + ":1"; private static final String COLUMN_2 = CFB + ":2"; private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); private static Client client; private static JAXBContext context; private static Marshaller marshaller; private static Unmarshaller unmarshaller; private static int expectedRows1; private static int expectedRows2; private static Configuration conf; static int insertData(Configuration conf, TableName tableName, String column, double prob) throws IOException { Random rng = new Random(); int count = 0; HTable table = new HTable(conf, tableName); byte[] k = new byte[3]; byte[][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column)); for (byte b1 = 'a'; b1 < 'z'; b1++) { for (byte b2 = 'a'; b2 < 'z'; b2++) { for (byte b3 = 'a'; b3 < 'z'; b3++) { if (rng.nextDouble() < prob) { k[0] = b1; k[1] = b2; k[2] = b3; Put put = new Put(k); put.setDurability(Durability.SKIP_WAL); put.add(famAndQf[0], famAndQf[1], k); table.put(put); count++; } } } } table.flushCommits(); table.close(); return count; } static int countCellSet(CellSetModel model) { int count = 0; Iterator<RowModel> rows = model.getRows().iterator(); while (rows.hasNext()) { RowModel row = rows.next(); Iterator<CellModel> cells = row.getCells().iterator(); while (cells.hasNext()) { cells.next(); count++; } } return count; } private static int fullTableScan(ScannerModel model) throws IOException { model.setBatch(100); Response response = client.put( "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); assertEquals(response.getCode(), 201); String scannerURI = response.getLocation(); assertNotNull(scannerURI); int count = 0; while (true) { response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF); assertTrue(response.getCode() == 200 || response.getCode() == 204); if (response.getCode() == 200) { assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); CellSetModel cellSet = new CellSetModel(); cellSet.getObjectFromMessage(response.getBody()); Iterator<RowModel> rows = cellSet.getRows().iterator(); while (rows.hasNext()) { RowModel row = rows.next(); Iterator<CellModel> cells = row.getCells().iterator(); while (cells.hasNext()) { cells.next(); count++; } } } else { break; } } // delete the scanner response = client.delete(scannerURI); assertEquals(response.getCode(), 200); return count; } @BeforeClass public static void setUpBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); TEST_UTIL.startMiniCluster(); REST_TEST_UTIL.startServletContainer(conf); client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); context = JAXBContext.newInstance( CellModel.class, CellSetModel.class, RowModel.class, ScannerModel.class); marshaller = context.createMarshaller(); unmarshaller = context.createUnmarshaller(); Admin admin = TEST_UTIL.getHBaseAdmin(); if (admin.tableExists(TABLE)) { return; } HTableDescriptor htd = new HTableDescriptor(TABLE); htd.addFamily(new HColumnDescriptor(CFA)); htd.addFamily(new HColumnDescriptor(CFB)); admin.createTable(htd); expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0); expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5); } @AfterClass public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); } @Test public void testSimpleScannerXML() throws IOException, JAXBException { final int BATCH_SIZE = 5; // new scanner ScannerModel model = new ScannerModel(); model.setBatch(BATCH_SIZE); model.addColumn(Bytes.toBytes(COLUMN_1)); StringWriter writer = new StringWriter(); marshaller.marshal(model, writer); byte[] body = Bytes.toBytes(writer.toString()); // test put operation is forbidden in read-only mode conf.set("hbase.rest.readonly", "true"); Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); assertEquals(response.getCode(), 403); String scannerURI = response.getLocation(); assertNull(scannerURI); // recall previous put operation with read-only off conf.set("hbase.rest.readonly", "false"); response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); assertEquals(response.getCode(), 201); scannerURI = response.getLocation(); assertNotNull(scannerURI); // get a cell set response = client.get(scannerURI, Constants.MIMETYPE_XML); assertEquals(response.getCode(), 200); assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); CellSetModel cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); // confirm batch size conformance assertEquals(countCellSet(cellSet), BATCH_SIZE); // test delete scanner operation is forbidden in read-only mode conf.set("hbase.rest.readonly", "true"); response = client.delete(scannerURI); assertEquals(response.getCode(), 403); // recall previous delete scanner operation with read-only off conf.set("hbase.rest.readonly", "false"); response = client.delete(scannerURI); assertEquals(response.getCode(), 200); } @Test public void testSimpleScannerPB() throws IOException { final int BATCH_SIZE = 10; // new scanner ScannerModel model = new ScannerModel(); model.setBatch(BATCH_SIZE); model.addColumn(Bytes.toBytes(COLUMN_1)); // test put operation is forbidden in read-only mode conf.set("hbase.rest.readonly", "true"); Response response = client.put( "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); assertEquals(response.getCode(), 403); String scannerURI = response.getLocation(); assertNull(scannerURI); // recall previous put operation with read-only off conf.set("hbase.rest.readonly", "false"); response = client.put( "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); assertEquals(response.getCode(), 201); scannerURI = response.getLocation(); assertNotNull(scannerURI); // get a cell set response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF); assertEquals(response.getCode(), 200); assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); CellSetModel cellSet = new CellSetModel(); cellSet.getObjectFromMessage(response.getBody()); // confirm batch size conformance assertEquals(countCellSet(cellSet), BATCH_SIZE); // test delete scanner operation is forbidden in read-only mode conf.set("hbase.rest.readonly", "true"); response = client.delete(scannerURI); assertEquals(response.getCode(), 403); // recall previous delete scanner operation with read-only off conf.set("hbase.rest.readonly", "false"); response = client.delete(scannerURI); assertEquals(response.getCode(), 200); } @Test public void testSimpleScannerBinary() throws IOException { // new scanner ScannerModel model = new ScannerModel(); model.setBatch(1); model.addColumn(Bytes.toBytes(COLUMN_1)); // test put operation is forbidden in read-only mode conf.set("hbase.rest.readonly", "true"); Response response = client.put( "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); assertEquals(response.getCode(), 403); String scannerURI = response.getLocation(); assertNull(scannerURI); // recall previous put operation with read-only off conf.set("hbase.rest.readonly", "false"); response = client.put( "/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); assertEquals(response.getCode(), 201); scannerURI = response.getLocation(); assertNotNull(scannerURI); // get a cell response = client.get(scannerURI, Constants.MIMETYPE_BINARY); assertEquals(response.getCode(), 200); assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type")); // verify that data was returned assertTrue(response.getBody().length > 0); // verify that the expected X-headers are present boolean foundRowHeader = false, foundColumnHeader = false, foundTimestampHeader = false; for (Header header : response.getHeaders()) { if (header.getName().equals("X-Row")) { foundRowHeader = true; } else if (header.getName().equals("X-Column")) { foundColumnHeader = true; } else if (header.getName().equals("X-Timestamp")) { foundTimestampHeader = true; } } assertTrue(foundRowHeader); assertTrue(foundColumnHeader); assertTrue(foundTimestampHeader); // test delete scanner operation is forbidden in read-only mode conf.set("hbase.rest.readonly", "true"); response = client.delete(scannerURI); assertEquals(response.getCode(), 403); // recall previous delete scanner operation with read-only off conf.set("hbase.rest.readonly", "false"); response = client.delete(scannerURI); assertEquals(response.getCode(), 200); } @Test public void testFullTableScan() throws IOException { ScannerModel model = new ScannerModel(); model.addColumn(Bytes.toBytes(COLUMN_1)); assertEquals(fullTableScan(model), expectedRows1); model = new ScannerModel(); model.addColumn(Bytes.toBytes(COLUMN_2)); assertEquals(fullTableScan(model), expectedRows2); } @Test public void testTableDoesNotExist() throws IOException, JAXBException { ScannerModel model = new ScannerModel(); StringWriter writer = new StringWriter(); marshaller.marshal(model, writer); byte[] body = Bytes.toBytes(writer.toString()); Response response = client.put("/" + NONEXISTENT_TABLE + "/scanner", Constants.MIMETYPE_XML, body); assertEquals(response.getCode(), 404); } }