private static int fullTableScan(ScannerModel model) throws IOException {
model.setBatch(100);
Response response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 201);
String scannerURI = response.getLocation();
assertNotNull(scannerURI);
int count = 0;
while (true) {
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
assertTrue(response.getCode() == 200 || response.getCode() == 204);
if (response.getCode() == 200) {
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
CellSetModel cellSet = new CellSetModel();
cellSet.getObjectFromMessage(response.getBody());
Iterator<RowModel> rows = cellSet.getRows().iterator();
while (rows.hasNext()) {
RowModel row = rows.next();
Iterator<CellModel> cells = row.getCells().iterator();
while (cells.hasNext()) {
cells.next();
count++;
}
}
} else {
break;
}
}
// delete the scanner
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
return count;
}
private static void verifyScan(Scan s, long expectedRows, long expectedKeys) throws Exception {
ScannerModel model = ScannerModel.fromScan(s);
model.setBatch(Integer.MAX_VALUE); // fetch it all at once
StringWriter writer = new StringWriter();
marshaller.marshal(model, writer);
LOG.debug(writer.toString());
byte[] body = Bytes.toBytes(writer.toString());
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 201);
String scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
CellSetModel cells =
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
int rows = cells.getRows().size();
assertTrue(
"Scanned too many rows! Only expected " + expectedRows + " total but scanned " + rows,
expectedRows == rows);
for (RowModel row : cells.getRows()) {
int count = row.getCells().size();
assertEquals(
"Expected " + expectedKeys + " keys per row but " + "returned " + count,
expectedKeys,
count);
}
// delete the scanner
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
}
@Test
public void testSimpleScannerBinary() throws IOException {
// new scanner
ScannerModel model = new ScannerModel();
model.setBatch(1);
model.addColumn(Bytes.toBytes(COLUMN_1));
// test put operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
Response response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 403);
String scannerURI = response.getLocation();
assertNull(scannerURI);
// recall previous put operation with read-only off
conf.set("hbase.rest.readonly", "false");
response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 201);
scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell
response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
// verify that data was returned
assertTrue(response.getBody().length > 0);
// verify that the expected X-headers are present
boolean foundRowHeader = false, foundColumnHeader = false, foundTimestampHeader = false;
for (Header header : response.getHeaders()) {
if (header.getName().equals("X-Row")) {
foundRowHeader = true;
} else if (header.getName().equals("X-Column")) {
foundColumnHeader = true;
} else if (header.getName().equals("X-Timestamp")) {
foundTimestampHeader = true;
}
}
assertTrue(foundRowHeader);
assertTrue(foundColumnHeader);
assertTrue(foundTimestampHeader);
// test delete scanner operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 403);
// recall previous delete scanner operation with read-only off
conf.set("hbase.rest.readonly", "false");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
}
private static void verifyScanFull(Scan s, KeyValue[] kvs) throws Exception {
ScannerModel model = ScannerModel.fromScan(s);
model.setBatch(Integer.MAX_VALUE); // fetch it all at once
StringWriter writer = new StringWriter();
marshaller.marshal(model, writer);
LOG.debug(writer.toString());
byte[] body = Bytes.toBytes(writer.toString());
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 201);
String scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
CellSetModel cellSet =
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
// delete the scanner
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
int row = 0;
int idx = 0;
Iterator<RowModel> i = cellSet.getRows().iterator();
for (boolean done = true; done; row++) {
done = i.hasNext();
if (!done) break;
RowModel rowModel = i.next();
List<CellModel> cells = rowModel.getCells();
if (cells.isEmpty()) break;
assertTrue(
"Scanned too many keys! Only expected "
+ kvs.length
+ " total but already scanned "
+ (cells.size() + idx),
kvs.length >= idx + cells.size());
for (CellModel cell : cells) {
assertTrue("Row mismatch", Bytes.equals(rowModel.getKey(), kvs[idx].getRow()));
byte[][] split = KeyValue.parseColumn(cell.getColumn());
assertTrue("Family mismatch", Bytes.equals(split[0], kvs[idx].getFamily()));
assertTrue("Qualifier mismatch", Bytes.equals(split[1], kvs[idx].getQualifier()));
assertTrue("Value mismatch", Bytes.equals(cell.getValue(), kvs[idx].getValue()));
idx++;
}
}
assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, kvs.length, idx);
}
void doTestGetStargateVersionPB() throws IOException {
Response response = client.get("/version", MIMETYPE_PROTOBUF);
assertTrue(response.getCode() == 200);
VersionModel model = new VersionModel();
model.getObjectFromMessage(response.getBody());
validate(model);
LOG.info("success retrieving Stargate version as protobuf");
}
@Test
public void testSimpleScannerPB() throws IOException {
final int BATCH_SIZE = 10;
// new scanner
ScannerModel model = new ScannerModel();
model.setBatch(BATCH_SIZE);
model.addColumn(Bytes.toBytes(COLUMN_1));
// test put operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
Response response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 403);
String scannerURI = response.getLocation();
assertNull(scannerURI);
// recall previous put operation with read-only off
conf.set("hbase.rest.readonly", "false");
response =
client.put(
"/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 201);
scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
CellSetModel cellSet = new CellSetModel();
cellSet.getObjectFromMessage(response.getBody());
// confirm batch size conformance
assertEquals(countCellSet(cellSet), BATCH_SIZE);
// test delete scanner operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 403);
// recall previous delete scanner operation with read-only off
conf.set("hbase.rest.readonly", "false");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
}
void doTestGetStargateVersionXML() throws IOException, JAXBException {
Response response = client.get("/version", MIMETYPE_XML);
assertTrue(response.getCode() == 200);
VersionModel model =
(VersionModel)
context.createUnmarshaller().unmarshal(new ByteArrayInputStream(response.getBody()));
validate(model);
LOG.info("success retrieving Stargate version as XML");
}
private static void verifyScanNoEarlyOut(Scan s, long expectedRows, long expectedKeys)
throws Exception {
ScannerModel model = ScannerModel.fromScan(s);
model.setBatch(Integer.MAX_VALUE); // fetch it all at once
StringWriter writer = new StringWriter();
marshaller.marshal(model, writer);
LOG.debug(writer.toString());
byte[] body = Bytes.toBytes(writer.toString());
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 201);
String scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
CellSetModel cellSet =
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
// delete the scanner
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
Iterator<RowModel> i = cellSet.getRows().iterator();
int j = 0;
for (boolean done = true; done; j++) {
done = i.hasNext();
if (!done) break;
RowModel rowModel = i.next();
List<CellModel> cells = rowModel.getCells();
if (cells.isEmpty()) break;
assertTrue(
"Scanned too many rows! Only expected "
+ expectedRows
+ " total but already scanned "
+ (j + 1),
expectedRows > j);
assertEquals(
"Expected " + expectedKeys + " keys per row but " + "returned " + cells.size(),
expectedKeys,
cells.size());
}
assertEquals("Expected " + expectedRows + " rows but scanned " + j + " rows", expectedRows, j);
}
@Test
public void testSimpleScannerXML() throws IOException, JAXBException {
final int BATCH_SIZE = 5;
// new scanner
ScannerModel model = new ScannerModel();
model.setBatch(BATCH_SIZE);
model.addColumn(Bytes.toBytes(COLUMN_1));
StringWriter writer = new StringWriter();
marshaller.marshal(model, writer);
byte[] body = Bytes.toBytes(writer.toString());
// test put operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 403);
String scannerURI = response.getLocation();
assertNull(scannerURI);
// recall previous put operation with read-only off
conf.set("hbase.rest.readonly", "false");
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 201);
scannerURI = response.getLocation();
assertNotNull(scannerURI);
// get a cell set
response = client.get(scannerURI, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
CellSetModel cellSet =
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
// confirm batch size conformance
assertEquals(countCellSet(cellSet), BATCH_SIZE);
// test delete scanner operation is forbidden in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 403);
// recall previous delete scanner operation with read-only off
conf.set("hbase.rest.readonly", "false");
response = client.delete(scannerURI);
assertEquals(response.getCode(), 200);
}
void doTestGetStorageClusterVersionXML() throws IOException, JAXBException {
Response response = client.get("/version/cluster", MIMETYPE_XML);
assertTrue(response.getCode() == 200);
StorageClusterVersionModel clusterVersionModel =
(StorageClusterVersionModel)
context.createUnmarshaller().unmarshal(new ByteArrayInputStream(response.getBody()));
assertNotNull(clusterVersionModel);
assertNotNull(clusterVersionModel.getVersion());
LOG.info("success retrieving storage cluster version as XML");
}
@Test
public void testTableDoesNotExist() throws IOException, JAXBException {
ScannerModel model = new ScannerModel();
StringWriter writer = new StringWriter();
marshaller.marshal(model, writer);
byte[] body = Bytes.toBytes(writer.toString());
Response response =
client.put("/" + NONEXISTENT_TABLE + "/scanner", Constants.MIMETYPE_XML, body);
assertEquals(response.getCode(), 404);
}
void doTestGetStargateVersionText() throws IOException {
Response response = client.get("/version", MIMETYPE_TEXT);
assertTrue(response.getCode() == 200);
String body = Bytes.toString(response.getBody());
assertTrue(body.length() > 0);
assertTrue(body.contains(RESTServlet.VERSION_STRING));
assertTrue(body.contains(System.getProperty("java.vm.vendor")));
assertTrue(body.contains(System.getProperty("java.version")));
assertTrue(body.contains(System.getProperty("java.vm.version")));
assertTrue(body.contains(System.getProperty("os.name")));
assertTrue(body.contains(System.getProperty("os.version")));
assertTrue(body.contains(System.getProperty("os.arch")));
assertTrue(body.contains(ServletContainer.class.getPackage().getImplementationVersion()));
}
@Test
public void testTableCreateAndDeletePB() throws IOException, JAXBException {
String schemaPath = "/" + TABLE2 + "/schema";
TableSchemaModel model;
Response response;
Admin admin = TEST_UTIL.getHBaseAdmin();
assertFalse(admin.tableExists(TableName.valueOf(TABLE2)));
// create the table
model = testTableSchemaModel.buildTestModel(TABLE2);
testTableSchemaModel.checkModel(model, TABLE2);
response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 201);
// recall the same put operation but in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 403);
// retrieve the schema and validate it
response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
model = new TableSchemaModel();
model.getObjectFromMessage(response.getBody());
testTableSchemaModel.checkModel(model, TABLE2);
// retrieve the schema and validate it with alternate pbuf type
response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF_IETF);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type"));
model = new TableSchemaModel();
model.getObjectFromMessage(response.getBody());
testTableSchemaModel.checkModel(model, TABLE2);
// test delete schema operation is forbidden in read-only mode
response = client.delete(schemaPath);
assertEquals(response.getCode(), 403);
// return read-only setting back to default
conf.set("hbase.rest.readonly", "false");
// delete the table and make sure HBase concurs
response = client.delete(schemaPath);
assertEquals(response.getCode(), 200);
assertFalse(admin.tableExists(TableName.valueOf(TABLE2)));
}
@Test
public void testTableCreateAndDeleteXML() throws IOException, JAXBException {
String schemaPath = "/" + TABLE1 + "/schema";
TableSchemaModel model;
Response response;
Admin admin = TEST_UTIL.getHBaseAdmin();
assertFalse(admin.tableExists(TableName.valueOf(TABLE1)));
// create the table
model = testTableSchemaModel.buildTestModel(TABLE1);
testTableSchemaModel.checkModel(model, TABLE1);
response = client.put(schemaPath, Constants.MIMETYPE_XML, toXML(model));
assertEquals(response.getCode(), 201);
// recall the same put operation but in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.put(schemaPath, Constants.MIMETYPE_XML, toXML(model));
assertEquals(response.getCode(), 403);
// retrieve the schema and validate it
response = client.get(schemaPath, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
model = fromXML(response.getBody());
testTableSchemaModel.checkModel(model, TABLE1);
// with json retrieve the schema and validate it
response = client.get(schemaPath, Constants.MIMETYPE_JSON);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
model = testTableSchemaModel.fromJSON(Bytes.toString(response.getBody()));
testTableSchemaModel.checkModel(model, TABLE1);
// test delete schema operation is forbidden in read-only mode
response = client.delete(schemaPath);
assertEquals(response.getCode(), 403);
// return read-only setting back to default
conf.set("hbase.rest.readonly", "false");
// delete the table and make sure HBase concurs
response = client.delete(schemaPath);
assertEquals(response.getCode(), 200);
assertFalse(admin.tableExists(TableName.valueOf(TABLE1)));
}
void doTestGetStargateVersionJSON() throws IOException {
Response response = client.get("/version", MIMETYPE_JSON);
assertTrue(response.getCode() == 200);
}
void doTestGetStorageClusterVersionJSON() throws IOException {
Response response = client.get("/version/cluster", MIMETYPE_JSON);
assertTrue(response.getCode() == 200);
}
@Override
protected void tearDown() throws Exception {
client.shutdown();
super.tearDown();
}