@Test
public void testTTL() throws Exception {
TableName tableName = TableName.valueOf("testTTL");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(F).setMaxVersions(10).setTimeToLive(1);
desc.addFamily(hcd);
TEST_UTIL.getHBaseAdmin().createTable(desc);
Table t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
long now = EnvironmentEdgeManager.currentTime();
ManualEnvironmentEdge me = new ManualEnvironmentEdge();
me.setValue(now);
EnvironmentEdgeManagerTestHelper.injectEdge(me);
// 2s in the past
long ts = now - 2000;
// Set the TTL override to 3s
Put p = new Put(R);
p.setAttribute("ttl", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(3000L));
t.put(p);
p = new Put(R);
p.add(F, Q, ts, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, ts + 1, Q);
t.put(p);
// these two should be expired but for the override
// (their ts was 2s in the past)
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
// still there?
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still there?
assertEquals(2, r.size());
// roll time forward 2s.
me.setValue(now + 2000);
// now verify that data eventually does expire
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// should be gone now
assertEquals(0, r.size());
t.close();
}
@Test
public void testStartStopRow() throws Exception {
final TableName TABLENAME1 = TableName.valueOf("testStartStopRow1");
final TableName TABLENAME2 = TableName.valueOf("testStartStopRow2");
final byte[] FAMILY = Bytes.toBytes("family");
final byte[] COLUMN1 = Bytes.toBytes("c1");
final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0");
final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
// put rows into the first table
Put p = new Put(ROW0);
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
p = new Put(ROW1);
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
p = new Put(ROW2);
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
CopyTable copy = new CopyTable();
assertEquals(
0,
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2,
"--startrow=\\x01row1",
"--stoprow=\\x01row2",
TABLENAME1.getNameAsString()
}));
// verify the data was copied into table 2
// row1 exist, row0, row2 do not exist
Get g = new Get(ROW1);
Result r = t2.get(g);
assertEquals(1, r.size());
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
g = new Get(ROW0);
r = t2.get(g);
assertEquals(0, r.size());
g = new Get(ROW2);
r = t2.get(g);
assertEquals(0, r.size());
t1.close();
t2.close();
TEST_UTIL.deleteTable(TABLENAME1);
TEST_UTIL.deleteTable(TABLENAME2);
}
@Test
public void testBaseCases() throws Exception {
TableName tableName = TableName.valueOf("baseCases");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
Table t = TEST_UTIL.createTable(tableName, F, 1);
// set the version override to 2
Put p = new Put(R);
p.setAttribute("versions", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(2));
t.put(p);
long now = EnvironmentEdgeManager.currentTime();
// insert 2 versions
p = new Put(R);
p.add(F, Q, now, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, now + 1, Q);
t.put(p);
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
// both version are still visible even after a flush/compaction
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
assertEquals(2, r.size());
// insert a 3rd version
p = new Put(R);
p.add(F, Q, now + 2, Q);
t.put(p);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still only two version visible
assertEquals(2, r.size());
t.close();
}
public Optional<Response> get(Optional<Request> request) {
if (!valid) {
Logger.error("CANNOT GET! NO VALID CONNECTION");
return Optional.empty();
}
Response response = new Response();
if (request.isPresent()) {
Request r = request.get();
response.key = r.key;
response.table = r.table;
try {
final Table htable = connection.getTable(TableName.valueOf(r.table));
Result result = htable.get(new Get(r.key));
if (result == null || result.isEmpty()) {
return Optional.empty();
}
r.columns.forEach(
c ->
response.columns.add(
new Request.Column(
c.family,
c.qualifier,
result.getValue(c.family.getBytes(), c.qualifier.getBytes()))));
} catch (IOException e) {
e.printStackTrace();
}
}
return Optional.of(response);
}
@Test(timeout = 300000)
public void testReplicationWithCellTags() throws Exception {
LOG.info("testSimplePutDelete");
Put put = new Put(ROW);
put.setAttribute("visibility", Bytes.toBytes("myTag3"));
put.add(FAMILY, ROW, ROW);
htable1 = utility1.getConnection().getTable(TABLE_NAME);
htable1.put(put);
Get get = new Get(ROW);
try {
for (int i = 0; i < NB_RETRIES; i++) {
if (i == NB_RETRIES - 1) {
fail("Waited too much time for put replication");
}
Result res = htable2.get(get);
if (res.size() == 0) {
LOG.info("Row not available");
Thread.sleep(SLEEP_TIME);
} else {
assertArrayEquals(res.value(), ROW);
assertEquals(1, TestCoprocessorForTagsAtSink.tags.size());
Tag tag = TestCoprocessorForTagsAtSink.tags.get(0);
assertEquals(TAG_TYPE, tag.getType());
break;
}
}
} finally {
TestCoprocessorForTagsAtSink.tags = null;
}
}
public static int getDynamicTable(Configuration config) {
/** Connection to the cluster. A single connection shared by all application threads. */
Connection connection = null;
/** A lightweight handle to a specific table. Used from a single thread. */
Table table = null;
try {
connection = ConnectionFactory.createConnection(config);
table = connection.getTable(TABLE_NAME1);
Get get = new Get(Bytes.toBytes("cloudera"));
get.addFamily(CF);
get.setMaxVersions(Integer.MAX_VALUE);
Result result = table.get(get);
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = result.getMap();
for (Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> columnFamilyEntry :
map.entrySet()) {
NavigableMap<byte[], NavigableMap<Long, byte[]>> columnMap = columnFamilyEntry.getValue();
for (Entry<byte[], NavigableMap<Long, byte[]>> columnEntry : columnMap.entrySet()) {
NavigableMap<Long, byte[]> cellMap = columnEntry.getValue();
for (Entry<Long, byte[]> cellEntry : cellMap.entrySet()) {
System.out.println(
String.format(
"Key : %s, Value :%s",
Bytes.toString(columnEntry.getKey()), Bytes.toString(cellEntry.getValue())));
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return 0;
}
private static VesselTrackInfo getTrackInfo(Table TrackInfo_Table, String IMO_str)
throws IOException {
Get get = new Get(Bytes.toBytes(IMO_str));
get.addColumn(details, lastlocation);
get.addColumn(details, firstrecordtime);
get.addColumn(details, lastrecordtime);
Result result = TrackInfo_Table.get(get);
byte[] last_location = result.getValue(details, lastlocation);
byte[] fist_recordtime = result.getValue(details, firstrecordtime);
byte[] last_recordtime = result.getValue(details, lastrecordtime);
VesselTrackInfo trackinfo = new VesselTrackInfo();
trackinfo.LastLocation = last_location;
if (fist_recordtime != null) {
trackinfo.FirstRecordTime =
DateTime.parse(Bytes.toString(fist_recordtime), rawformatter).getMillis();
}
if (last_recordtime != null) {
trackinfo.LastRecordTime =
DateTime.parse(Bytes.toString(last_recordtime), rawformatter).getMillis();
}
return trackinfo;
}
public static VesselLocation getLocation(Table VTLocation_Table, String RowKey)
throws IOException {
if (RowKey != null) {
Get get = new Get(Bytes.toBytes(RowKey));
Result result = VTLocation_Table.get(get);
VesselLocation VL = new VesselLocation();
for (Cell cell : result.rawCells()) {
String Qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
String Value = Bytes.toString(CellUtil.cloneValue(cell));
if (Qualifier.equals("coordinates")) {
VL.coordinates = Value;
} else if (Qualifier.equals("speed")) {
VL.speed = Value;
} else if (Qualifier.equals("destination")) {
VL.destination = Value;
} else if (Qualifier.equals("timestamp")) {
VL.recordtime = DateTime.parse(Value, rawformatter).getMillis();
} else if (Qualifier.equals("previouslocation")) {
VL.previouslocation = Value;
} else if (Qualifier.equals("nextlocation")) {
VL.nextlocation = Value;
}
}
return VL;
} else {
return null;
}
}
public Optional<byte[]> get(
Optional<String> table,
Optional<String> family,
Optional<String> qualifier,
Optional<String> key) {
if (!valid) {
Logger.error("CANNOT GET! NO VALID CONNECTION");
return Optional.empty();
}
if (table.isPresent()
&& family.isPresent()
&& qualifier.isPresent()
&& key.isPresent()
&& !key.get().isEmpty()) {
try {
final Table htable = connection.getTable(TableName.valueOf(table.get()));
Result result = htable.get(new Get(key.get().getBytes("UTF8")));
return Optional.ofNullable(
result.getValue(family.get().getBytes("UTF8"), qualifier.get().getBytes("UTF8")));
} catch (IOException e) {
e.printStackTrace();
}
}
return Optional.empty();
}
/**
* Read a record from the database. Each field/value pair from the result will be stored in a
* HashMap.
*
* @param table The name of the table
* @param key The record key of the record to read.
* @param fields The list of fields to read, or null for all of them
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
public Status read(
String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
// if this is a "new" table, init HTable object. Else, use existing one
if (!tableName.equals(table)) {
currentTable = null;
try {
getHTable(table);
tableName = table;
} catch (IOException e) {
System.err.println("Error accessing HBase table: " + e);
return Status.ERROR;
}
}
Result r = null;
try {
if (debug) {
System.out.println("Doing read from HBase columnfamily " + columnFamily);
System.out.println("Doing read for key: " + key);
}
Get g = new Get(Bytes.toBytes(key));
if (fields == null) {
g.addFamily(columnFamilyBytes);
} else {
for (String field : fields) {
g.addColumn(columnFamilyBytes, Bytes.toBytes(field));
}
}
r = currentTable.get(g);
} catch (IOException e) {
if (debug) {
System.err.println("Error doing get: " + e);
}
return Status.ERROR;
} catch (ConcurrentModificationException e) {
// do nothing for now...need to understand HBase concurrency model better
return Status.ERROR;
}
if (r.isEmpty()) {
return Status.NOT_FOUND;
}
while (r.advance()) {
final Cell c = r.current();
result.put(
Bytes.toString(CellUtil.cloneQualifier(c)),
new ByteArrayByteIterator(CellUtil.cloneValue(c)));
if (debug) {
System.out.println(
"Result for field: "
+ Bytes.toString(CellUtil.cloneQualifier(c))
+ " is: "
+ Bytes.toString(CellUtil.cloneValue(c)));
}
}
return Status.OK;
}
public static void main(String[] args) throws Exception {
conf.set("hbase.zookeeper.quorum", "hadoop271.itversity.com");
conf.set("hbase.zookeeper.property.clientPort", "2181");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(TableName.valueOf("demo"));
Scan scan1 = new Scan();
ResultScanner scanner1 = table.getScanner(scan1);
for (Result res : scanner1) {
System.out.println(Bytes.toString(res.getRow()));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
}
scanner1.close();
Put put = new Put("3".getBytes());
put.addColumn("cf1".getBytes(), "column1".getBytes(), "value1".getBytes());
put.addColumn("cf1".getBytes(), "column2".getBytes(), "value2".getBytes());
table.put(put);
Get get = new Get("3".getBytes());
Result getResult = table.get(get);
System.out.println("Printing colunns for rowkey 3");
System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column2".getBytes())));
scanner1 = table.getScanner(scan1);
System.out.println("Before Delete");
for (Result res : scanner1) {
System.out.println(Bytes.toString(res.getRow()));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
}
scanner1.close();
Delete del = new Delete("3".getBytes());
table.delete(del);
System.out.println("After Delete");
scanner1 = table.getScanner(scan1);
for (Result res : scanner1) {
System.out.println(Bytes.toString(res.getRow()));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
}
scanner1.close();
table.close();
connection.close();
}
/** Test copy of table from sourceTable to targetTable all rows from family a */
@Test
public void testRenameFamily() throws Exception {
TableName sourceTable = TableName.valueOf("sourceTable");
TableName targetTable = TableName.valueOf("targetTable");
byte[][] families = {FAMILY_A, FAMILY_B};
Table t = TEST_UTIL.createTable(sourceTable, families);
Table t2 = TEST_UTIL.createTable(targetTable, families);
Put p = new Put(ROW1);
p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data11"));
p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data12"));
p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data13"));
t.put(p);
p = new Put(ROW2);
p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
t.put(p);
long currentTime = System.currentTimeMillis();
String[] args =
new String[] {
"--new.name=" + targetTable,
"--families=a:b",
"--all.cells",
"--starttime=" + (currentTime - 100000),
"--endtime=" + (currentTime + 100000),
"--versions=1",
sourceTable.getNameAsString()
};
assertNull(t2.get(new Get(ROW1)).getRow());
assertTrue(runCopy(args));
assertNotNull(t2.get(new Get(ROW1)).getRow());
Result res = t2.get(new Get(ROW1));
byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
assertEquals("Data13", new String(b1));
assertNotNull(t2.get(new Get(ROW2)).getRow());
res = t2.get(new Get(ROW2));
b1 = res.getValue(FAMILY_A, QUALIFIER);
// Data from the family of B is not copied
assertNull(b1);
}
private static String getVesselType(Table Vessel_Table, String IMO_str) throws IOException {
Get get = new Get(Bytes.toBytes(IMO_str));
get.addColumn(ves, TYPE);
Result result = Vessel_Table.get(get);
byte[] type = result.getValue(ves, TYPE);
return Bytes.toString(type);
}
public Result get(String table, String family, byte[] key) {
final Table htable;
try {
htable = connection.getTable(TableName.valueOf(table));
return htable.get(new Get(key));
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
// Get all events with exit at last location
public static Map<Integer, VesselEvent> getAllEventsStartBeforeEndAfterBeforeLocation(
Table VTEvent_Table, String IMO_str, VesselLocation location) throws IOException {
Scan getAllEventsWithExistAtLastLocation = new Scan();
getAllEventsWithExistAtLastLocation
.setStartRow(
Bytes.toBytes(
IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19) + "0000000000"))
.setStopRow(Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE, 19) + "9999999999"))
.addColumn(details, exittime);
getAllEventsWithExistAtLastLocation.setCaching(100);
Filter ExistTimeValuefilter =
new ValueFilter(
CompareFilter.CompareOp.GREATER_OR_EQUAL,
new BinaryComparator(
Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter))));
getAllEventsWithExistAtLastLocation.setFilter(ExistTimeValuefilter);
ResultScanner Result_event = VTEvent_Table.getScanner(getAllEventsWithExistAtLastLocation);
Map<Integer, VesselEvent> events = new HashMap<Integer, VesselEvent>();
for (Result res : Result_event) {
Get get = new Get(res.getRow());
get.addColumn(details, entrytime);
get.addColumn(details, entrycoordinates);
Result result = VTEvent_Table.get(get);
String rowkey = Bytes.toString(result.getRow());
String polygonid = rowkey.substring(26);
VesselEvent VE = new VesselEvent();
VE.exittime = location.recordtime;
VE.exitcoordinates = location.coordinates;
VE.destination = location.destination;
VE.polygonid = Integer.parseInt(polygonid);
for (Cell cell : result.rawCells()) {
String Qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
String Value = Bytes.toString(CellUtil.cloneValue(cell));
if (Qualifier.equals("entertime")) {
VE.entrytime = DateTime.parse(Value, rawformatter).getMillis();
} else if (Qualifier.equals("entercoordinates")) {
VE.entrycoordinates = Value;
}
}
events.put(VE.polygonid, VE);
}
Result_event.close();
return events;
}
private void doCopyTableTest(boolean bulkload) throws Exception {
final TableName TABLENAME1 = TableName.valueOf("testCopyTable1");
final TableName TABLENAME2 = TableName.valueOf("testCopyTable2");
final byte[] FAMILY = Bytes.toBytes("family");
final byte[] COLUMN1 = Bytes.toBytes("c1");
try (Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY); ) {
// put rows into the first table
for (int i = 0; i < 10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
}
CopyTable copy = new CopyTable();
int code;
if (bulkload) {
code =
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2.getNameAsString(),
"--bulkload",
TABLENAME1.getNameAsString()
});
} else {
code =
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2.getNameAsString(), TABLENAME1.getNameAsString()
});
}
assertEquals("copy job failed", 0, code);
// verify the data was copied into table 2
for (int i = 0; i < 10; i++) {
Get g = new Get(Bytes.toBytes("row" + i));
Result r = t2.get(g);
assertEquals(1, r.size());
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
}
} finally {
TEST_UTIL.deleteTable(TABLENAME1);
TEST_UTIL.deleteTable(TABLENAME2);
}
}
public static void main(String[] args) throws IOException {
Configuration conf = HBaseConfiguration.create();
HBaseHelper helper = HBaseHelper.getHelper(conf);
helper.dropTable("testtable");
helper.createTable("testtable", "colfam1", "colfam2");
System.out.println("Adding rows to table...");
helper.fillTable("testtable", 1, 10, 10, "colfam1", "colfam2");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(TableName.valueOf("testtable"));
// vv SingleColumnValueFilterExample
SingleColumnValueFilter filter =
new SingleColumnValueFilter(
Bytes.toBytes("colfam1"),
Bytes.toBytes("col-5"),
CompareFilter.CompareOp.NOT_EQUAL,
new SubstringComparator("val-5"));
filter.setFilterIfMissing(true);
Scan scan = new Scan();
scan.setFilter(filter);
ResultScanner scanner = table.getScanner(scan);
// ^^ SingleColumnValueFilterExample
System.out.println("Results of scan:");
// vv SingleColumnValueFilterExample
for (Result result : scanner) {
for (Cell cell : result.rawCells()) {
System.out.println(
"Cell: "
+ cell
+ ", Value: "
+ Bytes.toString(
cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
}
scanner.close();
Get get = new Get(Bytes.toBytes("row-6"));
get.setFilter(filter);
Result result = table.get(get);
System.out.println("Result of get: ");
for (Cell cell : result.rawCells()) {
System.out.println(
"Cell: "
+ cell
+ ", Value: "
+ Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
// ^^ SingleColumnValueFilterExample
}
public static void main(String[] args) throws IOException {
Configuration conf = HBaseClientHelper.loadDefaultConfiguration();
Connection connection = ConnectionFactory.createConnection(conf);
try {
Table table = connection.getTable(TableName.valueOf("testtable"));
try {
// 1 Put
Put p = new Put(Bytes.toBytes("row1"));
p.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));
table.put(p);
// 2 Get
Get g = new Get(Bytes.toBytes("row1"));
Result r = table.get(g);
byte[] value = r.getValue(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
String valueStr = Bytes.toString(value);
System.out.println("GET: " + valueStr);
// 3 Scan
Scan s = new Scan();
s.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
ResultScanner scanner = table.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
System.out.println("Found row: " + rr);
}
// The other approach is to use a foreach loop. Scanners are
// iterable!
// for (Result rr : scanner) {
// System.out.println("Found row: " + rr);
// }
} finally {
scanner.close();
}
// Close your table and cluster connection.
} finally {
if (table != null) table.close();
}
} finally {
connection.close();
}
}
@Test
public void testReadModifyWrite() throws Throwable {
prepareTestData();
failures.set(0);
int numThreads = 100;
concurrentExec(new IncrementRunner(), numThreads);
Get get = new Get(ROW);
LOG.debug("row keyvalues:" + stringifyKvs(table.get(get).listCells()));
int finalCounter = incrementCounter(table);
int failureNumber = failures.get();
if (failureNumber > 0) {
LOG.debug("We failed " + failureNumber + " times during test");
}
assertEquals(numThreads + 1 - failureNumber, finalCounter);
}
@Test
public void testDoubleScan() throws Throwable {
prepareTestData();
CoprocessorRpcChannel channel = table.coprocessorService(ROW);
RowProcessorEndpoint.FriendsOfFriendsProcessor processor =
new RowProcessorEndpoint.FriendsOfFriendsProcessor(ROW, A);
RowProcessorService.BlockingInterface service = RowProcessorService.newBlockingStub(channel);
ProcessRequest request = RowProcessorClient.getRowProcessorPB(processor);
ProcessResponse protoResult = service.process(null, request);
FriendsOfFriendsProcessorResponse response =
FriendsOfFriendsProcessorResponse.parseFrom(protoResult.getRowProcessorResult());
Set<String> result = new HashSet<String>();
result.addAll(response.getResultList());
Set<String> expected = new HashSet<String>(Arrays.asList(new String[] {"d", "e", "f", "g"}));
Get get = new Get(ROW);
LOG.debug("row keyvalues:" + stringifyKvs(table.get(get).listCells()));
assertEquals(expected, result);
}
public static int getTable(Configuration config) throws IOException {
/** Connection to the cluster. A single connection shared by all application threads. */
Connection connection = null;
/** A lightweight handle to a specific table. Used from a single thread. */
Table table = null;
try {
// establish the connection to the cluster.
connection = ConnectionFactory.createConnection(config);
// retrieve a handle to the target table.
table = connection.getTable(TABLE_NAME1);
// describe the data we want to write.
Get g = new Get(Bytes.toBytes("row4"));
// send the data.
table.get(g);
} finally {
// close everything down
if (table != null) table.close();
if (connection != null) connection.close();
}
return 0;
}
@Test
public void testAclTableEntries() throws Exception {
String userTestNamespace = "userTestNsp";
Table acl = UTIL.getConnection().getTable(AccessControlLists.ACL_TABLE_NAME);
try {
ListMultimap<String, TablePermission> perms =
AccessControlLists.getNamespacePermissions(conf, TEST_NAMESPACE);
perms = AccessControlLists.getNamespacePermissions(conf, TEST_NAMESPACE);
for (Map.Entry<String, TablePermission> entry : perms.entries()) {
LOG.debug(entry);
}
assertEquals(6, perms.size());
// Grant and check state in ACL table
grantOnNamespace(UTIL, userTestNamespace, TEST_NAMESPACE, Permission.Action.WRITE);
Result result = acl.get(new Get(Bytes.toBytes(userTestNamespace)));
assertTrue(result != null);
perms = AccessControlLists.getNamespacePermissions(conf, TEST_NAMESPACE);
assertEquals(7, perms.size());
List<TablePermission> namespacePerms = perms.get(userTestNamespace);
assertTrue(perms.containsKey(userTestNamespace));
assertEquals(1, namespacePerms.size());
assertEquals(TEST_NAMESPACE, namespacePerms.get(0).getNamespace());
assertEquals(null, namespacePerms.get(0).getFamily());
assertEquals(null, namespacePerms.get(0).getQualifier());
assertEquals(1, namespacePerms.get(0).getActions().length);
assertEquals(Permission.Action.WRITE, namespacePerms.get(0).getActions()[0]);
// Revoke and check state in ACL table
revokeFromNamespace(UTIL, userTestNamespace, TEST_NAMESPACE, Permission.Action.WRITE);
perms = AccessControlLists.getNamespacePermissions(conf, TEST_NAMESPACE);
assertEquals(6, perms.size());
} finally {
acl.close();
}
}
@Test
public void testMultipleRows() throws Throwable {
prepareTestData();
failures.set(0);
int numThreads = 100;
concurrentExec(new SwapRowsRunner(), numThreads);
LOG.debug("row keyvalues:" + stringifyKvs(table.get(new Get(ROW)).listCells()));
LOG.debug("row2 keyvalues:" + stringifyKvs(table.get(new Get(ROW2)).listCells()));
int failureNumber = failures.get();
if (failureNumber > 0) {
LOG.debug("We failed " + failureNumber + " times during test");
}
if (!swapped) {
assertEquals(rowSize, table.get(new Get(ROW)).listCells().size());
assertEquals(row2Size, table.get(new Get(ROW2)).listCells().size());
} else {
assertEquals(rowSize, table.get(new Get(ROW2)).listCells().size());
assertEquals(row2Size, table.get(new Get(ROW)).listCells().size());
}
}
/**
* Unfortunately, the easiest way to test this is to spin up a mini-cluster since we want to do
* the usual compaction mechanism on the region, rather than going through the backdoor to the
* region
*/
@Test
public void testRegionObserverCompactionTimeStacking() throws Exception {
// setup a mini cluster so we can do a real compaction on a region
Configuration conf = UTIL.getConfiguration();
conf.setClass(HConstants.REGION_IMPL, CompactionCompletionNotifyingRegion.class, HRegion.class);
conf.setInt("hbase.hstore.compaction.min", 2);
UTIL.startMiniCluster();
String tableName = "testRegionObserverCompactionTimeStacking";
byte[] ROW = Bytes.toBytes("testRow");
byte[] A = Bytes.toBytes("A");
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
desc.addFamily(new HColumnDescriptor(A));
desc.addCoprocessor(EmptyRegionObsever.class.getName(), null, Coprocessor.PRIORITY_USER, null);
desc.addCoprocessor(
NoDataFromCompaction.class.getName(), null, Coprocessor.PRIORITY_HIGHEST, null);
Admin admin = UTIL.getHBaseAdmin();
admin.createTable(desc);
Table table = UTIL.getConnection().getTable(desc.getTableName());
// put a row and flush it to disk
Put put = new Put(ROW);
put.add(A, A, A);
table.put(put);
HRegionServer rs = UTIL.getRSForFirstRegionInTable(desc.getTableName());
List<HRegion> regions = rs.getOnlineRegions(desc.getTableName());
assertEquals("More than 1 region serving test table with 1 row", 1, regions.size());
HRegion region = regions.get(0);
admin.flushRegion(region.getRegionName());
CountDownLatch latch =
((CompactionCompletionNotifyingRegion) region).getCompactionStateChangeLatch();
// put another row and flush that too
put = new Put(Bytes.toBytes("anotherrow"));
put.add(A, A, A);
table.put(put);
admin.flushRegion(region.getRegionName());
// run a compaction, which normally would should get rid of the data
// wait for the compaction checker to complete
latch.await();
// check both rows to ensure that they aren't there
Get get = new Get(ROW);
Result r = table.get(get);
assertNull(
"Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
+ r,
r.listCells());
get = new Get(Bytes.toBytes("anotherrow"));
r = table.get(get);
assertNull(
"Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor Found: "
+ r,
r.listCells());
table.close();
UTIL.shutdownMiniCluster();
}
