public String[] getTags(String objectType, String objectId) throws IOException {
List<String> ret = new ArrayList<String>();
String rowKey = objectType + "_" + objectId;
Scan s = new Scan(rowKey.getBytes(), rowKey.getBytes());
s.setMaxVersions(1);
ResultScanner scanner = htable.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
String localObjectType =
new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
NavigableMap<byte[], byte[]> map = rr.getFamilyMap("tags".getBytes());
Iterator<Entry<byte[], byte[]>> it = map.entrySet().iterator();
while (it.hasNext()) {
Entry<byte[], byte[]> entry = it.next();
String key = new String(entry.getKey());
if (!key.startsWith("OBJECT")) {
int val = Bytes.toInt(entry.getValue());
if (val > 0) ret.add(key);
}
}
}
} finally {
scanner.close();
}
return ret.toArray(new String[] {});
}
public static void main(String[] args) throws Exception {
if (args.length < 2) {
throw new Exception("Table name not specified.");
}
Configuration conf = HBaseConfiguration.create();
HTable table = new HTable(conf, args[0]);
String startKey = args[1];
TimeCounter executeTimer = new TimeCounter();
executeTimer.begin();
executeTimer.enter();
Expression exp =
ExpressionFactory.eq(
ExpressionFactory.toLong(
ExpressionFactory.toString(ExpressionFactory.columnValue("family", "longStr2"))),
ExpressionFactory.constant(Long.parseLong("99")));
ExpressionFilter expressionFilter = new ExpressionFilter(exp);
Scan scan = new Scan(Bytes.toBytes(startKey), expressionFilter);
int count = 0;
ResultScanner scanner = table.getScanner(scan);
Result r = scanner.next();
while (r != null) {
count++;
r = scanner.next();
}
System.out.println("++ Scanning finished with count : " + count + " ++");
scanner.close();
executeTimer.leave();
executeTimer.end();
System.out.println("++ Time cost for scanning: " + executeTimer.getTimeString() + " ++");
}
public boolean fetchReadAllFieldLine(List<Line> lines, LineSender sender) throws IOException {
if (null == this.rs) {
throw new IllegalStateException("HBase Client try to fetch data failed .");
}
for (Result result = rs.next(); result != null; result = rs.next()) {
Get get = new Get(result.getRow());
for (int i = 0; i < this.families.length; i++) {
get.addColumn(this.families[i].getBytes(), this.columns[i].getBytes());
}
gets.add(get);
if (gets.size() > this.BUFFER_LINE) {
Result[] getResults = this.htable.get(gets);
for (Result resu : getResults) {
if (null != resu) {
Line line = sender.createLine();
for (int i = 0; i < this.families.length; i++) {
byte[] value = resu.getValue(this.families[i].getBytes(), this.columns[i].getBytes());
if (null == value) {
line.addField(null);
} else {
line.addField(new String(value, encode));
}
}
line.addField(new String(resu.getRow(), encode));
}
}
return true;
}
}
return false;
}
public static void deleteTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_1");
byte[] stopRow = Bytes.toBytes("rowKey_3");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
List<Delete> delete = new ArrayList<Delete>();
while (result != null) {
Delete del = new Delete(result.getRow());
delete.add(del);
result = scanner.next();
}
table.delete(delete);
System.out.println("delete done");
table.close(); // very important
} catch (IOException e) {
e.printStackTrace();
}
}
public String[] getObjectIDs(String objectType, String... tags) throws IOException {
List<String> ret = new ArrayList<String>();
FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
SingleColumnValueFilter filter1 =
new SingleColumnValueFilter(
"tags".getBytes(), "OBJECTTYPE".getBytes(), CompareOp.EQUAL, Bytes.toBytes(objectType));
list.addFilter(filter1);
for (String tag : tags) {
SingleColumnValueFilter filter2 =
new SingleColumnValueFilter(
"tags".getBytes(), tag.toUpperCase().getBytes(), CompareOp.EQUAL, Bytes.toBytes(1));
filter2.setFilterIfMissing(true);
list.addFilter(filter2);
}
Scan s = new Scan();
s.setFilter(list);
s.setMaxVersions(1);
ResultScanner scanner = htable.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
String localObjectType =
new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
ret.add(localObjectId);
}
} finally {
scanner.close();
}
return ret.toArray(new String[] {});
}
@TimeDepend
@Test
public void testScan_ts_same() throws Exception {
recreateTable();
Date ts = parse("2000-01-01", "yyyy-MM-dd");
Put put = new Put(rowKey_ForTest);
put.add(ColumnFamilyNameBytes, QName1, ts.getTime(), Bytes.toBytes("a"));
table.put(put);
Set<String> resultRowKeys = new HashSet<String>();
Scan scan = new Scan(rowKey_ForTest, rowKey_ForTest);
scan.setTimeRange(ts.getTime(), ts.getTime());
ResultScanner resultScanner = table.getScanner(scan);
for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
resultRowKeys.add(Bytes.toString(result.getRow()));
}
close(resultScanner);
Assert.assertTrue(resultRowKeys.size() == 0);
recreateTable();
}
public static void main(String[] args) throws IOException {
Configuration conf = HBaseClientHelper.loadDefaultConfiguration();
Connection connection = ConnectionFactory.createConnection(conf);
try {
Table table = connection.getTable(TableName.valueOf("testtable"));
try {
// 1 Put
Put p = new Put(Bytes.toBytes("row1"));
p.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));
table.put(p);
// 2 Get
Get g = new Get(Bytes.toBytes("row1"));
Result r = table.get(g);
byte[] value = r.getValue(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
String valueStr = Bytes.toString(value);
System.out.println("GET: " + valueStr);
// 3 Scan
Scan s = new Scan();
s.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
ResultScanner scanner = table.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
System.out.println("Found row: " + rr);
}
// The other approach is to use a foreach loop. Scanners are
// iterable!
// for (Result rr : scanner) {
// System.out.println("Found row: " + rr);
// }
} finally {
scanner.close();
}
// Close your table and cluster connection.
} finally {
if (table != null) table.close();
}
} finally {
connection.close();
}
}
@Test
public void testCheckpointRollback() throws Exception {
// start a transaction, using checkpoints between writes
transactionContext.start();
transactionAwareHTable.put(
new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
transactionContext.checkpoint();
transactionAwareHTable.put(
new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
transactionContext.checkpoint();
transactionAwareHTable.put(
new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
transactionContext.abort();
transactionContext.start();
verifyRow(transactionAwareHTable, TestBytes.row, null);
verifyRow(transactionAwareHTable, TestBytes.row2, null);
verifyRow(transactionAwareHTable, TestBytes.row3, null);
Scan scan = new Scan();
ResultScanner scanner = transactionAwareHTable.getScanner(scan);
assertNull(scanner.next());
scanner.close();
transactionContext.finish();
}
/**
* Performs a full scan of a catalog table.
*
* @param catalogTracker
* @param visitor Visitor invoked against each row.
* @param startrow Where to start the scan. Pass null if want to begin scan at first row.
* @param scanRoot True if we are to scan <code>-ROOT-</code> rather than <code>.META.</code>, the
* default (pass false to scan .META.)
* @throws IOException
*/
static void fullScan(
CatalogTracker catalogTracker,
final Visitor visitor,
final byte[] startrow,
final boolean scanRoot)
throws IOException {
Scan scan = new Scan();
if (startrow != null) scan.setStartRow(startrow);
if (startrow == null && !scanRoot) {
int caching =
catalogTracker
.getConnection()
.getConfiguration()
.getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
scan.setCaching(caching);
}
scan.addFamily(HConstants.CATALOG_FAMILY);
HTable metaTable = scanRoot ? getRootHTable(catalogTracker) : getMetaHTable(catalogTracker);
ResultScanner scanner = metaTable.getScanner(scan);
try {
Result data;
while ((data = scanner.next()) != null) {
if (data.isEmpty()) continue;
// Break if visit returns false.
if (!visitor.visit(data)) break;
}
} finally {
scanner.close();
metaTable.close();
}
return;
}
@Override
public void emitTuples() {
try {
Scan scan = nextScan();
if (scan == null) return;
ResultScanner resultScanner = store.getTable().getScanner(scan);
while (true) {
Result result = resultScanner.next();
if (result == null) break;
String readRow = Bytes.toString(result.getRow());
if (readRow.equals(lastReadRow)) continue;
Object instance = pojoType.newInstance();
rowSetter.set(instance, readRow);
List<Cell> cells = result.listCells();
for (Cell cell : cells) {
String columnName = Bytes.toString(CellUtil.cloneQualifier(cell));
byte[] value = CellUtil.cloneValue(cell);
fieldValueGenerator.setColumnValue(instance, columnName, value, valueConverter);
}
outputPort.emit(instance);
lastReadRow = readRow;
}
} catch (Exception e) {
throw new RuntimeException(e.getMessage());
}
}
/*
* Check current row has a HRegionInfo. Skip to next row if HRI is empty.
* @return A Map of the row content else null if we are off the end.
* @throws IOException
*/
private Result getMetaRow() throws IOException {
Result currentRow = metaScanner.next();
boolean foundResult = false;
while (currentRow != null) {
LOG.info("Row: <" + Bytes.toStringBinary(currentRow.getRow()) + ">");
byte[] regionInfoValue =
currentRow.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
if (regionInfoValue == null || regionInfoValue.length == 0) {
currentRow = metaScanner.next();
continue;
}
foundResult = true;
break;
}
return foundResult ? currentRow : null;
}
/*
* Add to each of the regions in .META. a value. Key is the startrow of the
* region (except its 'aaa' for first region). Actual value is the row name.
* @param expected
* @return
* @throws IOException
*/
private static int addToEachStartKey(final int expected) throws IOException {
HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
HTable meta = new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
int rows = 0;
Scan scan = new Scan();
scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
ResultScanner s = meta.getScanner(scan);
for (Result r = null; (r = s.next()) != null; ) {
byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
if (b == null || b.length <= 0) break;
HRegionInfo hri = Writables.getHRegionInfo(b);
// If start key, add 'aaa'.
byte[] row = getStartKey(hri);
Put p = new Put(row);
p.setWriteToWAL(false);
p.add(getTestFamily(), getTestQualifier(), row);
t.put(p);
rows++;
}
s.close();
Assert.assertEquals(expected, rows);
t.close();
meta.close();
return rows;
}
/* (non-Javadoc)
* @see com.hazelcast.core.MapLoader#loadAllKeys()
*/
@Override
public Set<String> loadAllKeys() {
Set<String> keySet = null;
if (allowLoadAll) {
keySet = new HashSet<String>();
HTableInterface hti = null;
try {
hti = pool.getTable(tableName);
Scan s = new Scan();
s.addColumn(family, qualifier);
ResultScanner rs = hti.getScanner(s);
Result r = null;
while ((r = rs.next()) != null) {
String k = new String(r.getRow());
keySet.add(k);
}
} catch (IOException e) {
LOG.error("IOException while loading all keys", e);
} finally {
if (hti != null) {
pool.putTable(hti);
}
}
}
return keySet;
}
@Test
// HBase-3583
public void testHBase3583() throws IOException {
TableName tableName = TableName.valueOf("testHBase3583");
util.createTable(tableName, new byte[][] {A, B, C});
util.waitUntilAllRegionsAssigned(tableName);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled"},
tableName,
new Boolean[] {false, false, false, false});
HTable table = new HTable(util.getConfiguration(), tableName);
Put put = new Put(ROW);
put.add(A, A, A);
table.put(put);
Get get = new Get(ROW);
get.addColumn(A, A);
table.get(get);
// verify that scannerNext and scannerClose upcalls won't be invoked
// when we perform get().
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled"},
tableName,
new Boolean[] {true, true, false, false});
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {}
} finally {
scanner.close();
}
// now scanner hooks should be invoked.
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"wasScannerNextCalled", "wasScannerCloseCalled"},
tableName,
new Boolean[] {true, true});
util.deleteTable(tableName);
table.close();
}
private void verifyRowCount(Table table, int expectedRegionNum) throws IOException {
ResultScanner scanner = table.getScanner(new Scan());
int rowCount = 0;
while (scanner.next() != null) {
rowCount++;
}
assertEquals(expectedRegionNum, rowCount);
scanner.close();
}
public void checkHbase() throws IOException {
List<RequiredEmail> temp = new LinkedList<RequiredEmail>(requiredEmails);
HTableInterface table = hbase.getTestingTable();
Scan scan =
createScan(
hbase.getFamily(), new String[] {"Subject", "Sender", "Bcc", "Cc", "To", "Body"});
ResultScanner scanner = table.getScanner(scan);
int countInHBase = 0;
List<Result> extraResults = new LinkedList<Result>();
for (Result result = scanner.next(); result != null; result = scanner.next()) {
countInHBase++;
if (!removeResult(temp, result)) {
extraResults.add(result);
}
}
assertEmpty(temp);
assertNoExtraResults(extraResults);
}
@Override
public Tuple next() throws SQLException {
try {
Result result = scanner.next();
return result == null ? null : new ResultTuple(result);
} catch (IOException e) {
throw new PhoenixIOException(e);
}
}
/**
* Confirm ImportTsv via data in online table.
*
* @param dataAvailable
*/
private static void validateTable(
Configuration conf,
TableName tableName,
String family,
int valueMultiplier,
boolean dataAvailable)
throws IOException {
LOG.debug("Validating table.");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName);
boolean verified = false;
long pause = conf.getLong("hbase.client.pause", 5 * 1000);
int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
for (int i = 0; i < numRetries; i++) {
try {
Scan scan = new Scan();
// Scan entire family.
scan.addFamily(Bytes.toBytes(family));
if (dataAvailable) {
ResultScanner resScanner = table.getScanner(scan);
for (Result res : resScanner) {
LOG.debug("Getting results " + res.size());
assertTrue(res.size() == 2);
List<Cell> kvs = res.listCells();
assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));
assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));
assertTrue(
CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));
assertTrue(
CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));
// Only one result set is expected, so let it loop.
verified = true;
}
} else {
ResultScanner resScanner = table.getScanner(scan);
Result[] next = resScanner.next(2);
assertEquals(0, next.length);
verified = true;
}
break;
} catch (NullPointerException e) {
// If here, a cell was empty. Presume its because updates came in
// after the scanner had been opened. Wait a while and retry.
}
try {
Thread.sleep(pause);
} catch (InterruptedException e) {
// continue
}
}
table.close();
connection.close();
assertTrue(verified);
}
public static void scanTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_0");
byte[] stopRow = Bytes.toBytes("rowKey_6");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
int count = 0;
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
while (result != null) {
String rowKey = Bytes.toString(result.getRow());
NavigableMap<byte[], byte[]> m = result.getFamilyMap(family);
if (m == null || m.isEmpty()) {
System.err.println("Empty." + m);
return;
}
for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
String qualifier = Bytes.toString(entry.getKey());
String value = Bytes.toString(entry.getValue());
System.out.println(rowKey + ":" + qualifier + ":" + value);
}
result = scanner.next();
count++;
System.out.println("-----------------------------");
}
table.close(); // very important
System.out.println("count:" + count);
} catch (IOException e) {
e.printStackTrace();
}
}
@Test
// HBase-3758
public void testHBase3758() throws IOException {
TableName tableName = TableName.valueOf("testHBase3758");
util.createTable(tableName, new byte[][] {A, B, C});
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"hadDeleted", "wasScannerOpenCalled"},
tableName,
new Boolean[] {false, false});
HTable table = new HTable(util.getConfiguration(), tableName);
Put put = new Put(ROW);
put.add(A, A, A);
table.put(put);
Delete delete = new Delete(ROW);
table.delete(delete);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"hadDeleted", "wasScannerOpenCalled"},
tableName,
new Boolean[] {true, false});
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {}
} finally {
scanner.close();
}
// now scanner hooks should be invoked.
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"wasScannerOpenCalled"},
tableName,
new Boolean[] {true});
util.deleteTable(tableName);
table.close();
}
public void scan(
Consumer<Result> callback, String tableName, String columnFamily, String... qualifiers) {
if (callback != null && tableName != null && columnFamily != null && qualifiers != null) {
Scan s = new Scan();
byte[] family = Bytes.toBytes(columnFamily);
for (String qualifier : qualifiers) {
s.addColumn(family, Bytes.toBytes(qualifier));
}
try {
final Table table = connection.getTable(TableName.valueOf(tableName));
ResultScanner scanner = table.getScanner(s);
for (Result r = scanner.next(); r != null; r = scanner.next()) {
callback.accept(r);
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
public String[] getAll() throws DaoException {
try {
List<String> ret = new ArrayList<String>();
Scan s = new Scan();
s.setMaxVersions(1);
ResultScanner scanner = htable.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
ret.add(localObjectId);
}
} finally {
scanner.close();
}
return ret.toArray(new String[] {});
} catch (IOException ex) {
throw new DaoException(ex);
}
}
/*
* @return Count of rows in TABLENAME
* @throws IOException
*/
private static int count() throws IOException {
HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
int rows = 0;
Scan scan = new Scan();
ResultScanner s = t.getScanner(scan);
for (Result r = null; (r = s.next()) != null; ) {
rows++;
}
s.close();
LOG.info("Counted=" + rows);
return rows;
}
public List<D> next(int nbRows) throws IOException {
Result[] rows = rs.next(nbRows);
List<D> resultSets = new ArrayList<D>();
for (Result r : rows) {
D ds = client.toEntity(r);
if (ds != null) {
resultSets.add(ds);
}
}
return resultSets;
}
/**
* Insert a whole batch of entries
*
* @throws Exception
*/
@Test
public void testBatchSink() throws Exception {
List<WALEntry> entries = new ArrayList<WALEntry>(BATCH_SIZE);
List<Cell> cells = new ArrayList<Cell>();
for (int i = 0; i < BATCH_SIZE; i++) {
entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
}
SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
Scan scan = new Scan();
ResultScanner scanRes = table1.getScanner(scan);
assertEquals(BATCH_SIZE, scanRes.next(BATCH_SIZE).length);
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
// Start up our mini cluster on top of an 0.92 root.dir that has data from
// a 0.92 hbase run -- it has a table with 100 rows in it -- and see if
// we can migrate from 0.92
TEST_UTIL.startMiniZKCluster();
TEST_UTIL.startMiniDFSCluster(1);
Path testdir = TEST_UTIL.getDataTestDir("TestMetaMigrationConvertToPB");
// Untar our test dir.
File untar = untar(new File(testdir.toString()));
// Now copy the untar up into hdfs so when we start hbase, we'll run from it.
Configuration conf = TEST_UTIL.getConfiguration();
FsShell shell = new FsShell(conf);
FileSystem fs = FileSystem.get(conf);
// find where hbase will root itself, so we can copy filesystem there
Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
if (!fs.isDirectory(hbaseRootDir.getParent())) {
// mkdir at first
fs.mkdirs(hbaseRootDir.getParent());
}
doFsCommand(shell, new String[] {"-put", untar.toURI().toString(), hbaseRootDir.toString()});
// windows fix: tgz file has .META. directory renamed as -META- since the original is an illegal
// name under windows. So we rename it back. See
// src/test/data//TestMetaMigrationConvertingToPB.README and
// https://issues.apache.org/jira/browse/HBASE-6821
doFsCommand(
shell,
new String[] {
"-mv",
new Path(hbaseRootDir, "-META-").toString(),
new Path(hbaseRootDir, ".META.").toString()
});
// See whats in minihdfs.
doFsCommand(shell, new String[] {"-lsr", "/"});
TEST_UTIL.startMiniHBaseCluster(1, 1);
// Assert we are running against the copied-up filesystem. The copied-up
// rootdir should have had a table named 'TestTable' in it. Assert it
// present.
HTable t = new HTable(TEST_UTIL.getConfiguration(), TESTTABLE);
ResultScanner scanner = t.getScanner(new Scan());
int count = 0;
while (scanner.next() != null) {
count++;
}
// Assert that we find all 100 rows that are in the data we loaded. If
// so then we must have migrated it from 0.90 to 0.92.
Assert.assertEquals(ROW_COUNT, count);
scanner.close();
t.close();
}
/** Delte all the data in table. */
private void deleteData() throws Exception {
try {
// full scan.
Scan scan = new Scan();
ResultScanner resultScanner = table.getScanner(scan);
List<byte[]> rows = new LinkedList<byte[]>();
for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
rows.add(result.getRow());
}
close(resultScanner);
for (byte[] row : rows) {
table.delete(new Delete(row));
log.info("delete " + Bytes.toString(row));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public List<RowLogMessage> next(String subscription, Long minimalTimestamp, boolean problematic)
throws RowLogException {
byte[] rowPrefix;
byte[] subscriptionBytes = Bytes.toBytes(subscription);
if (problematic) {
rowPrefix = PROBLEMATIC_MARKER;
rowPrefix = Bytes.add(rowPrefix, subscriptionBytes);
} else {
rowPrefix = subscriptionBytes;
}
byte[] startRow = rowPrefix;
if (minimalTimestamp != null) startRow = Bytes.add(startRow, Bytes.toBytes(minimalTimestamp));
try {
List<RowLogMessage> rowLogMessages = new ArrayList<RowLogMessage>();
Scan scan = new Scan(startRow);
if (minimalTimestamp != null) scan.setTimeRange(minimalTimestamp, Long.MAX_VALUE);
scan.addColumn(MESSAGES_CF, MESSAGE_COLUMN);
ResultScanner scanner = table.getScanner(scan);
boolean keepScanning = problematic;
do {
Result[] results = scanner.next(batchSize);
if (results.length == 0) {
keepScanning = false;
}
for (Result next : results) {
byte[] rowKey = next.getRow();
if (!Bytes.startsWith(rowKey, rowPrefix)) {
keepScanning = false;
break; // There were no messages for this subscription
}
if (problematic) {
rowKey = Bytes.tail(rowKey, rowKey.length - PROBLEMATIC_MARKER.length);
}
byte[] value = next.getValue(MESSAGES_CF, MESSAGE_COLUMN);
byte[] messageId = Bytes.tail(rowKey, rowKey.length - subscriptionBytes.length);
rowLogMessages.add(decodeMessage(messageId, value));
}
} while (keepScanning);
// The scanner is not closed in a finally block, since when we get an IOException from
// HBase, it is likely that closing the scanner will give problems too. Not closing
// the scanner is not fatal since HBase will expire it after a while.
Closer.close(scanner);
return rowLogMessages;
} catch (IOException e) {
throw new RowLogException("Failed to fetch next message from RowLogShard", e);
}
}
@Test
public void testPassiveVisibility() throws Exception {
// No values should be filtered regardless of authorization if we are passive
try (Table t =
createTableAndWriteDataWithLabels(
TableName.valueOf(TEST_NAME.getMethodName()),
SECRET,
PRIVATE,
SECRET + "|" + CONFIDENTIAL,
PRIVATE + "|" + CONFIDENTIAL)) {
Scan s = new Scan();
s.setAuthorizations(new Authorizations());
try (ResultScanner scanner = t.getScanner(s)) {
Result[] next = scanner.next(10);
assertEquals(next.length, 4);
}
s = new Scan();
s.setAuthorizations(new Authorizations(SECRET));
try (ResultScanner scanner = t.getScanner(s)) {
Result[] next = scanner.next(10);
assertEquals(next.length, 4);
}
s = new Scan();
s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
try (ResultScanner scanner = t.getScanner(s)) {
Result[] next = scanner.next(10);
assertEquals(next.length, 4);
}
s = new Scan();
s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
try (ResultScanner scanner = t.getScanner(s)) {
Result[] next = scanner.next(10);
assertEquals(next.length, 4);
}
}
}
private void putDataAndVerify(
HTable table, String row, byte[] family, String value, int verifyNum) throws IOException {
System.out.println("=========Putting data :" + row);
Put put = new Put(Bytes.toBytes(row));
put.add(family, Bytes.toBytes("q1"), Bytes.toBytes(value));
table.put(put);
ResultScanner resultScanner = table.getScanner(new Scan());
List<Result> results = new ArrayList<Result>();
while (true) {
Result r = resultScanner.next();
if (r == null) break;
results.add(r);
}
resultScanner.close();
if (results.size() != verifyNum) {
System.out.println(results);
}
assertEquals(verifyNum, results.size());
}