private static void verifyScan(Scan s, long expectedRows, long expectedKeys) throws Exception {
ScannerModel model = ScannerModel.fromScan(s);
model.setBatch(Integer.MAX_VALUE); // fetch it all at once
StringWriter writer = new StringWriter();
marshaller.marshal(model, writer);
LOG.debug(writer.toString());
byte[] body = Bytes.toBytes(writer.toString());
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 201);
String scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
CellSetModel cells =
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
int rows = cells.getRows().size();
assertTrue(
"Scanned too many rows! Only expected " + expectedRows + " total but scanned " + rows,
expectedRows == rows);
for (RowModel row : cells.getRows()) {
int count = row.getCells().size();
assertEquals(
"Expected " + expectedKeys + " keys per row but " + "returned " + count,
expectedKeys,
count);
}
// delete the scanner
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
}
/**
* @param scan the scan specification
* @throws Exception
*/
public static ScannerModel fromScan(Scan scan) throws Exception {
ScannerModel model = new ScannerModel();
model.setStartRow(scan.getStartRow());
model.setEndRow(scan.getStopRow());
Map<byte[], NavigableSet<byte[]>> families = scan.getFamilyMap();
if (families != null) {
for (Map.Entry<byte[], NavigableSet<byte[]>> entry : families.entrySet()) {
if (entry.getValue() != null) {
for (byte[] qualifier : entry.getValue()) {
model.addColumn(Bytes.add(entry.getKey(), COLUMN_DIVIDER, qualifier));
}
} else {
model.addColumn(entry.getKey());
}
}
}
model.setStartTime(scan.getTimeRange().getMin());
model.setEndTime(scan.getTimeRange().getMax());
int caching = scan.getCaching();
if (caching > 0) {
model.setBatch(caching);
}
int maxVersions = scan.getMaxVersions();
if (maxVersions > 0) {
model.setMaxVersions(maxVersions);
}
Filter filter = scan.getFilter();
if (filter != null) {
model.setFilter(stringifyFilter(filter));
}
return model;
}
private static int fullTableScan(ScannerModel model) throws IOException {
model.setBatch(100);
Response response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 201);
String scannerURI = response.getLocation();
assertNotNull(scannerURI);
int count = 0;
while (true) {
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
assertTrue(response.getCode() == 200 || response.getCode() == 204);
if (response.getCode() == 200) {
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
CellSetModel cellSet = new CellSetModel();
cellSet.getObjectFromMessage(response.getBody());
Iterator<RowModel> rows = cellSet.getRows().iterator();
while (rows.hasNext()) {
RowModel row = rows.next();
Iterator<CellModel> cells = row.getCells().iterator();
while (cells.hasNext()) {
cells.next();
count++;
}
}
} else {
break;
}
}
// delete the scanner
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
return count;
}
@Test
public void testSimpleScannerBinary() throws IOException {
// new scanner
ScannerModel model = new ScannerModel();
model.setBatch(1);
model.addColumn(Bytes.toBytes(COLUMN_1));
// test put operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
Response response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 403);
String scannerURI = response.getLocation();
assertNull(scannerURI);
// recall previous put operation with read-only off
conf.set("hbase.rest.readonly", "false");
response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 201);
scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell
response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
// verify that data was returned
assertTrue(response.getBody().length > 0);
// verify that the expected X-headers are present
boolean foundRowHeader = false, foundColumnHeader = false, foundTimestampHeader = false;
for (Header header : response.getHeaders()) {
if (header.getName().equals("X-Row")) {
foundRowHeader = true;
} else if (header.getName().equals("X-Column")) {
foundColumnHeader = true;
} else if (header.getName().equals("X-Timestamp")) {
foundTimestampHeader = true;
}
}
assertTrue(foundRowHeader);
assertTrue(foundColumnHeader);
assertTrue(foundTimestampHeader);
// test delete scanner operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 403);
// recall previous delete scanner operation with read-only off
conf.set("hbase.rest.readonly", "false");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
}
@Test
public void testFullTableScan() throws IOException {
ScannerModel model = new ScannerModel();
model.addColumn(Bytes.toBytes(COLUMN_1));
assertEquals(fullTableScan(model), expectedRows1);
model = new ScannerModel();
model.addColumn(Bytes.toBytes(COLUMN_2));
assertEquals(fullTableScan(model), expectedRows2);
}
private static void verifyScanFull(Scan s, KeyValue[] kvs) throws Exception {
ScannerModel model = ScannerModel.fromScan(s);
model.setBatch(Integer.MAX_VALUE); // fetch it all at once
StringWriter writer = new StringWriter();
marshaller.marshal(model, writer);
LOG.debug(writer.toString());
byte[] body = Bytes.toBytes(writer.toString());
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 201);
String scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
CellSetModel cellSet =
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
// delete the scanner
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
int row = 0;
int idx = 0;
Iterator<RowModel> i = cellSet.getRows().iterator();
for (boolean done = true; done; row++) {
done = i.hasNext();
if (!done) break;
RowModel rowModel = i.next();
List<CellModel> cells = rowModel.getCells();
if (cells.isEmpty()) break;
assertTrue(
"Scanned too many keys! Only expected "
+ kvs.length
+ " total but already scanned "
+ (cells.size() + idx),
kvs.length >= idx + cells.size());
for (CellModel cell : cells) {
assertTrue("Row mismatch", Bytes.equals(rowModel.getKey(), kvs[idx].getRow()));
byte[][] split = KeyValue.parseColumn(cell.getColumn());
assertTrue("Family mismatch", Bytes.equals(split[0], kvs[idx].getFamily()));
assertTrue("Qualifier mismatch", Bytes.equals(split[1], kvs[idx].getQualifier()));
assertTrue("Value mismatch", Bytes.equals(cell.getValue(), kvs[idx].getValue()));
idx++;
}
}
assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, kvs.length, idx);
}
@Test
public void testSimpleScannerPB() throws IOException {
final int BATCH_SIZE = 10;
// new scanner
ScannerModel model = new ScannerModel();
model.setBatch(BATCH_SIZE);
model.addColumn(Bytes.toBytes(COLUMN_1));
// test put operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
Response response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 403);
String scannerURI = response.getLocation();
assertNull(scannerURI);
// recall previous put operation with read-only off
conf.set("hbase.rest.readonly", "false");
response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 201);
scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
CellSetModel cellSet = new CellSetModel();
cellSet.getObjectFromMessage(response.getBody());
// confirm batch size conformance
assertEquals(countCellSet(cellSet), BATCH_SIZE);
// test delete scanner operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 403);
// recall previous delete scanner operation with read-only off
conf.set("hbase.rest.readonly", "false");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
}
private static void verifyScanNoEarlyOut(Scan s, long expectedRows, long expectedKeys)
throws Exception {
ScannerModel model = ScannerModel.fromScan(s);
model.setBatch(Integer.MAX_VALUE); // fetch it all at once
StringWriter writer = new StringWriter();
marshaller.marshal(model, writer);
LOG.debug(writer.toString());
byte[] body = Bytes.toBytes(writer.toString());
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 201);
String scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
CellSetModel cellSet =
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
// delete the scanner
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
Iterator<RowModel> i = cellSet.getRows().iterator();
int j = 0;
for (boolean done = true; done; j++) {
done = i.hasNext();
if (!done) break;
RowModel rowModel = i.next();
List<CellModel> cells = rowModel.getCells();
if (cells.isEmpty()) break;
assertTrue(
"Scanned too many rows! Only expected "
+ expectedRows
+ " total but already scanned "
+ (j + 1),
expectedRows > j);
assertEquals(
"Expected " + expectedKeys + " keys per row but " + "returned " + cells.size(),
expectedKeys,
cells.size());
}
assertEquals("Expected " + expectedRows + " rows but scanned " + j + " rows", expectedRows, j);
}
@Test
public void testSimpleScannerXML() throws IOException, JAXBException {
final int BATCH_SIZE = 5;
// new scanner
ScannerModel model = new ScannerModel();
model.setBatch(BATCH_SIZE);
model.addColumn(Bytes.toBytes(COLUMN_1));
StringWriter writer = new StringWriter();
marshaller.marshal(model, writer);
byte[] body = Bytes.toBytes(writer.toString());
// test put operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 403);
String scannerURI = response.getLocation();
assertNull(scannerURI);
// recall previous put operation with read-only off
conf.set("hbase.rest.readonly", "false");
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 201);
scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
CellSetModel cellSet =
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
// confirm batch size conformance
assertEquals(countCellSet(cellSet), BATCH_SIZE);
// test delete scanner operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 403);
// recall previous delete scanner operation with read-only off
conf.set("hbase.rest.readonly", "false");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
}
