@Test
public void testRegionObserverFlushTimeStacking() throws Exception {
byte[] ROW = Bytes.toBytes("testRow");
byte[] TABLE = Bytes.toBytes(getClass().getName());
byte[] A = Bytes.toBytes("A");
byte[][] FAMILIES = new byte[][] {A};
Configuration conf = HBaseConfiguration.create();
HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES);
RegionCoprocessorHost h = region.getCoprocessorHost();
h.load(NoDataFromFlush.class, Coprocessor.PRIORITY_HIGHEST, conf);
h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf);
// put a row and flush it to disk
Put put = new Put(ROW);
put.add(A, A, A);
region.put(put);
region.flushcache();
Get get = new Get(ROW);
Result r = region.get(get);
assertNull(
"Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
+ r,
r.listCells());
}
/**
* 遍历多行
*
* @param tableName 表名
* @param start_rowkey 开始行键
* @param stop_rowkey 结束行键
* @return 行列表
*/
public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
ResultScanner rowstmp = null;
ArrayList<HbaseRow> rows = null;
try {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(start_rowkey));
scan.setStopRow(Bytes.toBytes(stop_rowkey));
HTable table = new HTable(conf, Bytes.toBytes(tableName));
rowstmp = table.getScanner(scan);
rows = new ArrayList<>();
for (Result rowtmp : rowstmp) {
HbaseRow row = new HbaseRow();
row.rowkey = Bytes.toString(rowtmp.getRow());
for (Cell cell : rowtmp.listCells()) {
HbaseColumn col = new HbaseColumn(cell);
row.cols.add(col);
}
rows.add(row);
}
} catch (Exception e) {
logger.error("scanRows failed", e);
} finally {
rowstmp.close();
}
return rows;
}
@Override
public void emitTuples() {
try {
Scan scan = nextScan();
if (scan == null) return;
ResultScanner resultScanner = store.getTable().getScanner(scan);
while (true) {
Result result = resultScanner.next();
if (result == null) break;
String readRow = Bytes.toString(result.getRow());
if (readRow.equals(lastReadRow)) continue;
Object instance = pojoType.newInstance();
rowSetter.set(instance, readRow);
List<Cell> cells = result.listCells();
for (Cell cell : cells) {
String columnName = Bytes.toString(CellUtil.cloneQualifier(cell));
byte[] value = CellUtil.cloneValue(cell);
fieldValueGenerator.setColumnValue(instance, columnName, value, valueConverter);
}
outputPort.emit(instance);
lastReadRow = readRow;
}
} catch (Exception e) {
throw new RuntimeException(e.getMessage());
}
}
/** Test GroupingTableMapper class */
@Test
public void testGroupingTableMapper() throws Exception {
GroupingTableMapper mapper = new GroupingTableMapper();
Configuration configuration = new Configuration();
configuration.set(GroupingTableMapper.GROUP_COLUMNS, "family1:clm family2:clm");
mapper.setConf(configuration);
Result result = mock(Result.class);
@SuppressWarnings("unchecked")
Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Result>.Context context =
mock(Mapper.Context.class);
context.write(any(ImmutableBytesWritable.class), any(Result.class));
List<Cell> keyValue = new ArrayList<Cell>();
byte[] row = {};
keyValue.add(
new KeyValue(row, Bytes.toBytes("family2"), Bytes.toBytes("clm"), Bytes.toBytes("value1")));
keyValue.add(
new KeyValue(row, Bytes.toBytes("family1"), Bytes.toBytes("clm"), Bytes.toBytes("value2")));
when(result.listCells()).thenReturn(keyValue);
mapper.map(null, result, context);
// template data
byte[][] data = {Bytes.toBytes("value1"), Bytes.toBytes("value2")};
ImmutableBytesWritable ibw = mapper.createGroupKey(data);
verify(context).write(ibw, result);
}
/**
* Looks at every value of the mapreduce output and verifies that indeed the values have been
* reversed.
*
* @param table Table to scan.
* @throws IOException
* @throws NullPointerException if we failed to find a cell value
*/
private void verifyAttempt(final Table table) throws IOException, NullPointerException {
Scan scan = new Scan();
scan.addFamily(INPUT_FAMILY);
scan.addFamily(OUTPUT_FAMILY);
ResultScanner scanner = table.getScanner(scan);
try {
Iterator<Result> itr = scanner.iterator();
assertTrue(itr.hasNext());
while (itr.hasNext()) {
Result r = itr.next();
if (LOG.isDebugEnabled()) {
if (r.size() > 2) {
throw new IOException("Too many results, expected 2 got " + r.size());
}
}
byte[] firstValue = null;
byte[] secondValue = null;
int count = 0;
for (Cell kv : r.listCells()) {
if (count == 0) {
firstValue = CellUtil.cloneValue(kv);
} else if (count == 1) {
secondValue = CellUtil.cloneValue(kv);
} else if (count == 2) {
break;
}
count++;
}
String first = "";
if (firstValue == null) {
throw new NullPointerException(Bytes.toString(r.getRow()) + ": first value is null");
}
first = Bytes.toString(firstValue);
String second = "";
if (secondValue == null) {
throw new NullPointerException(Bytes.toString(r.getRow()) + ": second value is null");
}
byte[] secondReversed = new byte[secondValue.length];
for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
secondReversed[i] = secondValue[j];
}
second = Bytes.toString(secondReversed);
if (first.compareTo(second) != 0) {
if (LOG.isDebugEnabled()) {
LOG.debug(
"second key is not the reverse of first. row="
+ Bytes.toStringBinary(r.getRow())
+ ", first value="
+ first
+ ", second value="
+ second);
}
fail();
}
}
} finally {
scanner.close();
}
}
/**
* Confirm ImportTsv via data in online table.
*
* @param dataAvailable
*/
private static void validateTable(
Configuration conf,
TableName tableName,
String family,
int valueMultiplier,
boolean dataAvailable)
throws IOException {
LOG.debug("Validating table.");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName);
boolean verified = false;
long pause = conf.getLong("hbase.client.pause", 5 * 1000);
int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
for (int i = 0; i < numRetries; i++) {
try {
Scan scan = new Scan();
// Scan entire family.
scan.addFamily(Bytes.toBytes(family));
if (dataAvailable) {
ResultScanner resScanner = table.getScanner(scan);
for (Result res : resScanner) {
LOG.debug("Getting results " + res.size());
assertTrue(res.size() == 2);
List<Cell> kvs = res.listCells();
assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));
assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));
assertTrue(
CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));
assertTrue(
CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));
// Only one result set is expected, so let it loop.
verified = true;
}
} else {
ResultScanner resScanner = table.getScanner(scan);
Result[] next = resScanner.next(2);
assertEquals(0, next.length);
verified = true;
}
break;
} catch (NullPointerException e) {
// If here, a cell was empty. Presume its because updates came in
// after the scanner had been opened. Wait a while and retry.
}
try {
Thread.sleep(pause);
} catch (InterruptedException e) {
// continue
}
}
table.close();
connection.close();
assertTrue(verified);
}
/** Return the number of rows in the given table. */
public static int countMobRows(final Table table) throws IOException {
Scan scan = new Scan();
ResultScanner results = table.getScanner(scan);
int count = 0;
for (Result res : results) {
count++;
List<Cell> cells = res.listCells();
for (Cell cell : cells) {
// Verify the value
Assert.assertTrue(CellUtil.cloneValue(cell).length > 0);
}
}
results.close();
return count;
}
/**
* 查询列
*
* @param tableName 表名
* @param rowKey 行键
* @param familyName 列族名
* @param columnName 列名
* @return 列
*/
public HbaseColumn getColumn(
String tableName, String rowKey, String familyName, String columnName) {
HbaseColumn col = null;
try {
HTable table = new HTable(conf, Bytes.toBytes(tableName));
Get get = new Get(Bytes.toBytes(rowKey));
get.addColumn(Bytes.toBytes(familyName), Bytes.toBytes(columnName)); // 获取指定列族和列修饰符对应的列
Result rowtmp = table.get(get);
col = new HbaseColumn();
for (Cell cell : rowtmp.listCells()) {
col.CopyFromCell(cell);
}
} catch (Exception e) {
logger.error("getColumn failed", e);
}
return col;
}
/**
* 查询单行
*
* @param tableName 表名
* @param rowKey 行键名
* @return 行
*/
public HbaseRow getRow(String tableName, String rowKey) {
HbaseRow row = null;
try {
Get get = new Get(Bytes.toBytes(rowKey));
HTable table = new HTable(conf, Bytes.toBytes(tableName)); // 获取表
Result rowtmp = table.get(get);
row = new HbaseRow();
row.rowkey = Bytes.toString(rowtmp.getRow());
for (Cell cell : rowtmp.listCells()) {
HbaseColumn col = new HbaseColumn(cell);
row.cols.add(col);
}
} catch (Exception e) {
logger.error("getRow failed", e);
}
return row;
}
private String resultToString(Result result) {
StringBuilder sb = new StringBuilder();
sb.append("cells=");
if (result.isEmpty()) {
sb.append("NONE");
return sb.toString();
}
sb.append("{");
boolean moreThanOne = false;
for (Cell cell : result.listCells()) {
if (moreThanOne) {
sb.append(", ");
} else {
moreThanOne = true;
}
sb.append(CellUtil.toString(cell, true));
}
sb.append("}");
return sb.toString();
}
/**
* Extract columns values from the current record. This method returns null if any of the columns
* are not found.
*
* <p>Override this method if you want to deal with nulls differently.
*
* @param r The current values.
* @return Array of byte values.
*/
protected byte[][] extractKeyValues(Result r) {
byte[][] keyVals = null;
ArrayList<byte[]> foundList = new ArrayList<byte[]>();
int numCols = columns.length;
if (numCols > 0) {
for (Cell value : r.listCells()) {
byte[] column =
KeyValue.makeColumn(CellUtil.cloneFamily(value), CellUtil.cloneQualifier(value));
for (int i = 0; i < numCols; i++) {
if (Bytes.equals(column, columns[i])) {
foundList.add(CellUtil.cloneValue(value));
break;
}
}
}
if (foundList.size() == numCols) {
keyVals = foundList.toArray(new byte[numCols][]);
}
}
return keyVals;
}
/**
* 查询列的多版本
*
* @param tableName 表名
* @param rowKey 行键名
* @param familyName 列族名
* @param columnName 列名
* @param num 版本数
* @return 列列表
*/
public ArrayList<HbaseColumn> getColumnVersions(
String tableName, String rowKey, String familyName, String columnName, int num) {
ArrayList<HbaseColumn> cols = null;
try {
HTable table = new HTable(conf, Bytes.toBytes(tableName));
Get get = new Get(Bytes.toBytes(rowKey));
get.addColumn(Bytes.toBytes(familyName), Bytes.toBytes(columnName));
get.setMaxVersions(num);
Result rowtmp = table.get(get);
cols = new ArrayList<>();
for (Cell cell : rowtmp.listCells()) {
HbaseColumn col = new HbaseColumn(cell);
cols.add(col);
}
} catch (Exception e) {
logger.error("getColumnVersions failed", e);
}
return cols;
}
/**
* Unfortunately, the easiest way to test this is to spin up a mini-cluster since we want to do
* the usual compaction mechanism on the region, rather than going through the backdoor to the
* region
*/
@Test
public void testRegionObserverCompactionTimeStacking() throws Exception {
// setup a mini cluster so we can do a real compaction on a region
Configuration conf = UTIL.getConfiguration();
conf.setClass(HConstants.REGION_IMPL, CompactionCompletionNotifyingRegion.class, HRegion.class);
conf.setInt("hbase.hstore.compaction.min", 2);
UTIL.startMiniCluster();
String tableName = "testRegionObserverCompactionTimeStacking";
byte[] ROW = Bytes.toBytes("testRow");
byte[] A = Bytes.toBytes("A");
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
desc.addFamily(new HColumnDescriptor(A));
desc.addCoprocessor(EmptyRegionObsever.class.getName(), null, Coprocessor.PRIORITY_USER, null);
desc.addCoprocessor(
NoDataFromCompaction.class.getName(), null, Coprocessor.PRIORITY_HIGHEST, null);
Admin admin = UTIL.getHBaseAdmin();
admin.createTable(desc);
Table table = UTIL.getConnection().getTable(desc.getTableName());
// put a row and flush it to disk
Put put = new Put(ROW);
put.add(A, A, A);
table.put(put);
HRegionServer rs = UTIL.getRSForFirstRegionInTable(desc.getTableName());
List<HRegion> regions = rs.getOnlineRegions(desc.getTableName());
assertEquals("More than 1 region serving test table with 1 row", 1, regions.size());
HRegion region = regions.get(0);
admin.flushRegion(region.getRegionName());
CountDownLatch latch =
((CompactionCompletionNotifyingRegion) region).getCompactionStateChangeLatch();
// put another row and flush that too
put = new Put(Bytes.toBytes("anotherrow"));
put.add(A, A, A);
table.put(put);
admin.flushRegion(region.getRegionName());
// run a compaction, which normally would should get rid of the data
// wait for the compaction checker to complete
latch.await();
// check both rows to ensure that they aren't there
Get get = new Get(ROW);
Result r = table.get(get);
assertNull(
"Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
+ r,
r.listCells());
get = new Get(Bytes.toBytes("anotherrow"));
r = table.get(get);
assertNull(
"Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor Found: "
+ r,
r.listCells());
table.close();
UTIL.shutdownMiniCluster();
}