/*
* Add to each of the regions in .META. a value. Key is the startrow of the
* region (except its 'aaa' for first region). Actual value is the row name.
* @param expected
* @return
* @throws IOException
*/
private static int addToEachStartKey(final int expected) throws IOException {
HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
HTable meta = new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
int rows = 0;
Scan scan = new Scan();
scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
ResultScanner s = meta.getScanner(scan);
for (Result r = null; (r = s.next()) != null; ) {
byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
if (b == null || b.length <= 0) break;
HRegionInfo hri = Writables.getHRegionInfo(b);
// If start key, add 'aaa'.
byte[] row = getStartKey(hri);
Put p = new Put(row);
p.setWriteToWAL(false);
p.add(getTestFamily(), getTestQualifier(), row);
t.put(p);
rows++;
}
s.close();
Assert.assertEquals(expected, rows);
t.close();
meta.close();
return rows;
}
public Blog(String blogid) throws IOException {
Configuration conf = HBaseConfiguration.create();
table = new HTable(conf, "blogs");
// 1. Get the row whose row key is blogid from above
Get g = new Get(Bytes.toBytes(blogid));
Result r = table.get(g);
// 2. Extract the rowkey, blog text (column "body") and blog title
// (column "meta:title")
key = r.getRow();
keyStr = Bytes.toString(key);
blogText = Bytes.toString(r.getValue(Bytes.toBytes("body"), Bytes.toBytes("")));
blogTitle = Bytes.toString(r.getValue(Bytes.toBytes("meta"), Bytes.toBytes("title")));
Long reverseTimestamp = Long.parseLong(keyStr.substring(4));
Long epoch = Math.abs(reverseTimestamp - Long.MAX_VALUE);
dateOfPost = new Date(epoch);
// Get an iterator for the comments
Scan s = new Scan();
s.addFamily(Bytes.toBytes("comment"));
// Use a PrefixFilter
PrefixFilter filter = new PrefixFilter(key);
s.setFilter(filter);
scanner = table.getScanner(s);
resultIterator = scanner.iterator();
}
public String[] getTags(String objectType, String objectId) throws IOException {
List<String> ret = new ArrayList<String>();
String rowKey = objectType + "_" + objectId;
Scan s = new Scan(rowKey.getBytes(), rowKey.getBytes());
s.setMaxVersions(1);
ResultScanner scanner = htable.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
String localObjectType =
new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
NavigableMap<byte[], byte[]> map = rr.getFamilyMap("tags".getBytes());
Iterator<Entry<byte[], byte[]>> it = map.entrySet().iterator();
while (it.hasNext()) {
Entry<byte[], byte[]> entry = it.next();
String key = new String(entry.getKey());
if (!key.startsWith("OBJECT")) {
int val = Bytes.toInt(entry.getValue());
if (val > 0) ret.add(key);
}
}
}
} finally {
scanner.close();
}
return ret.toArray(new String[] {});
}
/**
* 遍历多行
*
* @param tableName 表名
* @param start_rowkey 开始行键
* @param stop_rowkey 结束行键
* @return 行列表
*/
public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
ResultScanner rowstmp = null;
ArrayList<HbaseRow> rows = null;
try {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(start_rowkey));
scan.setStopRow(Bytes.toBytes(stop_rowkey));
HTable table = new HTable(conf, Bytes.toBytes(tableName));
rowstmp = table.getScanner(scan);
rows = new ArrayList<>();
for (Result rowtmp : rowstmp) {
HbaseRow row = new HbaseRow();
row.rowkey = Bytes.toString(rowtmp.getRow());
for (Cell cell : rowtmp.listCells()) {
HbaseColumn col = new HbaseColumn(cell);
row.cols.add(col);
}
rows.add(row);
}
} catch (Exception e) {
logger.error("scanRows failed", e);
} finally {
rowstmp.close();
}
return rows;
}
/**
* Performs a full scan of a catalog table.
*
* @param catalogTracker
* @param visitor Visitor invoked against each row.
* @param startrow Where to start the scan. Pass null if want to begin scan at first row.
* @param scanRoot True if we are to scan <code>-ROOT-</code> rather than <code>.META.</code>, the
* default (pass false to scan .META.)
* @throws IOException
*/
static void fullScan(
CatalogTracker catalogTracker,
final Visitor visitor,
final byte[] startrow,
final boolean scanRoot)
throws IOException {
Scan scan = new Scan();
if (startrow != null) scan.setStartRow(startrow);
if (startrow == null && !scanRoot) {
int caching =
catalogTracker
.getConnection()
.getConfiguration()
.getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
scan.setCaching(caching);
}
scan.addFamily(HConstants.CATALOG_FAMILY);
HTable metaTable = scanRoot ? getRootHTable(catalogTracker) : getMetaHTable(catalogTracker);
ResultScanner scanner = metaTable.getScanner(scan);
try {
Result data;
while ((data = scanner.next()) != null) {
if (data.isEmpty()) continue;
// Break if visit returns false.
if (!visitor.visit(data)) break;
}
} finally {
scanner.close();
metaTable.close();
}
return;
}
public static void main(String[] args) throws Exception {
if (args.length < 2) {
throw new Exception("Table name not specified.");
}
Configuration conf = HBaseConfiguration.create();
HTable table = new HTable(conf, args[0]);
String startKey = args[1];
TimeCounter executeTimer = new TimeCounter();
executeTimer.begin();
executeTimer.enter();
Expression exp =
ExpressionFactory.eq(
ExpressionFactory.toLong(
ExpressionFactory.toString(ExpressionFactory.columnValue("family", "longStr2"))),
ExpressionFactory.constant(Long.parseLong("99")));
ExpressionFilter expressionFilter = new ExpressionFilter(exp);
Scan scan = new Scan(Bytes.toBytes(startKey), expressionFilter);
int count = 0;
ResultScanner scanner = table.getScanner(scan);
Result r = scanner.next();
while (r != null) {
count++;
r = scanner.next();
}
System.out.println("++ Scanning finished with count : " + count + " ++");
scanner.close();
executeTimer.leave();
executeTimer.end();
System.out.println("++ Time cost for scanning: " + executeTimer.getTimeString() + " ++");
}
public String[] getObjectIDs(String objectType, String... tags) throws IOException {
List<String> ret = new ArrayList<String>();
FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
SingleColumnValueFilter filter1 =
new SingleColumnValueFilter(
"tags".getBytes(), "OBJECTTYPE".getBytes(), CompareOp.EQUAL, Bytes.toBytes(objectType));
list.addFilter(filter1);
for (String tag : tags) {
SingleColumnValueFilter filter2 =
new SingleColumnValueFilter(
"tags".getBytes(), tag.toUpperCase().getBytes(), CompareOp.EQUAL, Bytes.toBytes(1));
filter2.setFilterIfMissing(true);
list.addFilter(filter2);
}
Scan s = new Scan();
s.setFilter(list);
s.setMaxVersions(1);
ResultScanner scanner = htable.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
String localObjectType =
new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
ret.add(localObjectId);
}
} finally {
scanner.close();
}
return ret.toArray(new String[] {});
}
/**
* rowFilter的使用
*
* @param tableName
* @param reg
* @throws Exception
*/
public void getRowFilter(String tableName, String reg) throws Exception {
Connection conn = null;
HTable table = null;
try {
conn = ConnectionFactory.createConnection(conf);
table = (HTable) conn.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
// Filter
RowFilter rowFilter = new RowFilter(CompareOp.NOT_EQUAL, new RegexStringComparator(reg));
scan.setFilter(rowFilter);
ResultScanner scanner = table.getScanner(scan);
for (Result result : scanner) {
System.out.println(new String(result.getRow()));
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (null != table) {
try {
table.close();
} catch (IOException e) {
logger.error("HTable close exception, errMsg:{}", e.getMessage());
}
}
if (null != conn) {
try {
conn.close();
} catch (Exception e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
}
public static List<String> getBooksbyPrice(String min, String max) throws IOException {
List list = new ArrayList<String>();
Scan scan = new Scan();
scan.setMaxVersions();
FilterList filterList = new FilterList();
Filter maxFilter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.GREATER_OR_EQUAL,
Bytes.toBytes(min));
Filter minFilter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.LESS_OR_EQUAL,
Bytes.toBytes(max));
filterList.addFilter(maxFilter);
filterList.addFilter(minFilter);
scan.setFilter(filterList);
ResultScanner rs = table.getScanner(scan);
for (Result r : rs) {
String str = "title: " + Bytes.toString(r.getRow());
for (KeyValue kv : r.raw()) {
str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
str += Bytes.toString(kv.getValue());
}
System.out.println(str);
list.add(str);
}
return list;
}
public static List<String> getFreeBooks() throws IOException {
List list = new ArrayList<String>();
Scan scan = new Scan();
scan.setMaxVersions();
Filter filter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.EQUAL,
Bytes.toBytes("0.00"));
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
for (Result r : rs) {
String str = "title: " + Bytes.toString(r.getRow());
for (KeyValue kv : r.raw()) {
str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
str += Bytes.toString(kv.getValue());
}
/*
* for (KeyValue kv : r.raw()) { System.out.println(String.format(
* "row:%s, family:%s, qualifier:%s, qualifiervalue:%s, timestamp:%s."
* , Bytes.toString(kv.getRow()), Bytes.toString(kv.getFamily()),
* Bytes.toString(kv.getQualifier()), Bytes.toString(kv.getValue()),
* kv.getTimestamp())); }
*/
System.out.println(str);
list.add(str);
}
return list;
}
OnlineMerger(Configuration conf, FileSystem fs, final byte[] tableName) throws IOException {
super(conf, fs, tableName);
this.tableName = tableName;
this.table = new HTable(conf, HConstants.META_TABLE_NAME);
this.metaScanner =
table.getScanner(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
this.latestRegion = null;
}
public static void main(String[] args) throws IOException {
Configuration conf = HBaseConfiguration.create();
HBaseHelper helper = HBaseHelper.getHelper(conf);
helper.dropTable("testtable");
helper.createTable("testtable", "colfam1", "colfam2", "colfam3", "colfam4");
System.out.println("Adding rows to table...");
helper.fillTable("testtable", 1, 10, 2, "colfam1", "colfam2", "colfam3", "colfam4");
HTable table = new HTable(conf, "testtable");
// vv FamilyFilterExample
Filter filter1 =
new FamilyFilter(
CompareFilter.CompareOp
.LESS, // co FamilyFilterExample-1-Filter Create filter, while specifying the
// comparison operator and comparator.
new BinaryComparator(Bytes.toBytes("colfam3")));
Scan scan = new Scan();
scan.setFilter(filter1);
ResultScanner scanner =
table.getScanner(
scan); // co FamilyFilterExample-2-Scan Scan over table while applying the filter.
// ^^ FamilyFilterExample
System.out.println("Scanning table... ");
// vv FamilyFilterExample
for (Result result : scanner) {
System.out.println(result);
}
scanner.close();
Get get1 = new Get(Bytes.toBytes("row-5"));
get1.setFilter(filter1);
Result result1 =
table.get(get1); // co FamilyFilterExample-3-Get Get a row while applying the same filter.
System.out.println("Result of get(): " + result1);
Filter filter2 =
new FamilyFilter(
CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("colfam3")));
Get get2 =
new Get(
Bytes.toBytes(
"row-5")); // co FamilyFilterExample-4-Mismatch Create a filter on one column family
// while trying to retrieve another.
get2.addFamily(Bytes.toBytes("colfam1"));
get2.setFilter(filter2);
Result result2 =
table.get(
get2); // co FamilyFilterExample-5-Get2 Get the same row while applying the new filter,
// this will return "NONE".
System.out.println("Result of get(): " + result2);
// ^^ FamilyFilterExample
}
/*
* @return Count of rows in TABLENAME
* @throws IOException
*/
private static int count() throws IOException {
HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
int rows = 0;
Scan scan = new Scan();
ResultScanner s = t.getScanner(scan);
for (Result r = null; (r = s.next()) != null; ) {
rows++;
}
s.close();
LOG.info("Counted=" + rows);
return rows;
}
/**
* Insert a whole batch of entries
*
* @throws Exception
*/
@Test
public void testBatchSink() throws Exception {
List<WALEntry> entries = new ArrayList<WALEntry>(BATCH_SIZE);
List<Cell> cells = new ArrayList<Cell>();
for (int i = 0; i < BATCH_SIZE; i++) {
entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
}
SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
Scan scan = new Scan();
ResultScanner scanRes = table1.getScanner(scan);
assertEquals(BATCH_SIZE, scanRes.next(BATCH_SIZE).length);
}
@Override
public void tweet(User user, String tweetText) throws IOException {
final long epoch = System.currentTimeMillis();
final long tranposeEpoch = Long.MAX_VALUE - epoch;
final byte[] epochBytes = Bytes.toBytes(epoch);
final byte[] tweetBytes = Bytes.toBytes(tweetText);
byte[] nameBytes = Bytes.toBytes(user.getName());
/** put tweet into tweets */
Put tweetRowPut = new Put(generateTweetId(user));
tweetRowPut.add(_DEFAULT, _NAME, nameBytes);
tweetRowPut.add(_DEFAULT, _MAIL, Bytes.toBytes(user.getEmail()));
tweetRowPut.add(_DEFAULT, _TWEET, tweetBytes);
tweetRowPut.add(_DEFAULT, _TIME, epochBytes);
tweetsTable.put(tweetRowPut);
/** put tweets for followers */
Scan followerScan = new Scan();
followerScan.setStartRow(Bytes.toBytes(user.getUserId() + "-"));
followerScan.setStopRow(Bytes.toBytes((user.getUserId() + 1) + "-"));
ResultScanner followerRS = followersTable.getScanner(followerScan);
/** put users on tweet to her own tweetline */
Put put =
new Put(Bytes.toBytes(user.getUserId() + "-" + tranposeEpoch + "-" + user.getUserId()));
put.add(_DEFAULT, _NAME, nameBytes);
put.add(_DEFAULT, _TWEET, tweetBytes);
put.add(_DEFAULT, _TIME, epochBytes);
List<Row> puts = new ArrayList<Row>();
puts.add(put);
for (Result result : followerRS) {
Long followerid = Bytes.toLong(result.getColumnLatest(_DEFAULT, _USERID).getValue());
put = new Put(Bytes.toBytes(followerid + "-" + tranposeEpoch + "-" + user.getUserId()));
put.add(_DEFAULT, _NAME, nameBytes);
put.add(_DEFAULT, _TWEET, tweetBytes);
put.add(_DEFAULT, _TIME, epochBytes);
puts.add(put);
}
followerRS.close();
try {
tweetlineTable.batch(puts);
} catch (InterruptedException e) {
e.printStackTrace(); // @TODO log and handle properly.
}
}
/**
* Get data according to tableName and filter list.
*
* @param tableName
* @param filterList
* @return
*/
public static ResultScanner getData(String tableName, FilterList filterList) {
if (tableName == null) {
return null;
}
ResultScanner scanner = null;
try {
// Instantiating HTable class
HTable table = new HTable(config, tableName);
// Instantiating the Scan class
Scan scan = new Scan();
scan.setFilter(filterList);
scanner = table.getScanner(scan);
ResultScanner results = table.getScanner(scan);
return results;
} catch (Exception e) {
e.printStackTrace();
return null;
} finally {
// closing the scanner
scanner.close();
}
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
// Start up our mini cluster on top of an 0.92 root.dir that has data from
// a 0.92 hbase run -- it has a table with 100 rows in it -- and see if
// we can migrate from 0.92
TEST_UTIL.startMiniZKCluster();
TEST_UTIL.startMiniDFSCluster(1);
Path testdir = TEST_UTIL.getDataTestDir("TestMetaMigrationConvertToPB");
// Untar our test dir.
File untar = untar(new File(testdir.toString()));
// Now copy the untar up into hdfs so when we start hbase, we'll run from it.
Configuration conf = TEST_UTIL.getConfiguration();
FsShell shell = new FsShell(conf);
FileSystem fs = FileSystem.get(conf);
// find where hbase will root itself, so we can copy filesystem there
Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
if (!fs.isDirectory(hbaseRootDir.getParent())) {
// mkdir at first
fs.mkdirs(hbaseRootDir.getParent());
}
doFsCommand(shell, new String[] {"-put", untar.toURI().toString(), hbaseRootDir.toString()});
// windows fix: tgz file has .META. directory renamed as -META- since the original is an illegal
// name under windows. So we rename it back. See
// src/test/data//TestMetaMigrationConvertingToPB.README and
// https://issues.apache.org/jira/browse/HBASE-6821
doFsCommand(
shell,
new String[] {
"-mv",
new Path(hbaseRootDir, "-META-").toString(),
new Path(hbaseRootDir, ".META.").toString()
});
// See whats in minihdfs.
doFsCommand(shell, new String[] {"-lsr", "/"});
TEST_UTIL.startMiniHBaseCluster(1, 1);
// Assert we are running against the copied-up filesystem. The copied-up
// rootdir should have had a table named 'TestTable' in it. Assert it
// present.
HTable t = new HTable(TEST_UTIL.getConfiguration(), TESTTABLE);
ResultScanner scanner = t.getScanner(new Scan());
int count = 0;
while (scanner.next() != null) {
count++;
}
// Assert that we find all 100 rows that are in the data we loaded. If
// so then we must have migrated it from 0.90 to 0.92.
Assert.assertEquals(ROW_COUNT, count);
scanner.close();
t.close();
}
public void loadMetaRecords(List<MetaRecord> records) throws Exception {
MetaRecord rec;
for (Result r : table.getScanner(new Scan())) {
if (r.isEmpty()) continue;
rec = getMetaRecord(r);
records.add(rec);
if (!tracker.contains(rec.getId()))
ZKUtil.createNodeIfNotExistsAndWatch(
watcher,
ZKUtil.joinZNode(H2MetaTableTracker.NODE_NAME, Integer.toString(rec.getId())),
EMPTY_BYTE_ARRAY);
}
}
/**
* Insert to 2 different tables
*
* @throws Exception
*/
@Test
public void testMixedPutTables() throws Exception {
List<WALEntry> entries = new ArrayList<WALEntry>(BATCH_SIZE / 2);
List<Cell> cells = new ArrayList<Cell>();
for (int i = 0; i < BATCH_SIZE; i++) {
entries.add(createEntry(i % 2 == 0 ? TABLE_NAME2 : TABLE_NAME1, i, KeyValue.Type.Put, cells));
}
SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
Scan scan = new Scan();
ResultScanner scanRes = table2.getScanner(scan);
for (Result res : scanRes) {
assertTrue(Bytes.toInt(res.getRow()) % 2 == 0);
}
}
public static void main(String[] args) throws IOException, SolrServerException {
final Configuration conf;
HttpSolrServer solrServer = new HttpSolrServer("http://c1master:8983/solr");
conf = HBaseConfiguration.create();
// Define Hbase Table Name
HTable table = new HTable(conf, "test_global_shop");
Scan scan = new Scan();
// Define Hbase Column Family
scan.addFamily(Bytes.toBytes("shop"));
scan.setCaching(1000);
scan.setCacheBlocks(false);
ResultScanner ss = table.getScanner(scan);
System.out.println("start Storing...");
int i = 0;
try {
for (Result r : ss) {
SolrInputDocument solrDoc = new SolrInputDocument();
solrDoc.addField("key", new String(r.getRow()));
for (KeyValue kv : r.raw()) {
String fieldName = new String(kv.getQualifier());
String fieldValue = new String(kv.getValue());
if (fieldName.equalsIgnoreCase("address")
|| fieldName.equalsIgnoreCase("category")
|| fieldName.equalsIgnoreCase("name")
|| fieldName.equalsIgnoreCase("province")
|| fieldName.equalsIgnoreCase("tel")) {
solrDoc.addField(fieldName, fieldValue);
}
}
solrServer.add(solrDoc);
solrServer.commit(true, true, true);
i = i + 1;
System.out.println("Already Succcess " + i + " number data");
}
ss.close();
table.close();
System.out.println("done !");
} catch (IOException e) {
} finally {
ss.close();
table.close();
System.out.println("error !");
}
}
public static void main(String[] args) throws Exception, ZooKeeperConnectionException {
// 初始化hbase配置
Configuration conf = HBaseConfiguration.create();
// 指定zk集群
conf.set("hbase.zookeeper.quorum", "yarn1,yarn2,yarn3");
// 操作元数据的类，对表进行操作
HBaseAdmin admin = new HBaseAdmin(conf);
// 1.创建新表
HTableDescriptor tableDescriptor = new HTableDescriptor("user2".getBytes());
// 添加列族
tableDescriptor.addFamily(new HColumnDescriptor("info"));
tableDescriptor.addFamily(new HColumnDescriptor("address"));
admin.createTable(tableDescriptor);
// 表对象
HTable table = new HTable(conf, "user2");
// 2.Htable.put()插入数据,rowone1为行键
Put putRow1 = new Put("jiangxiaoyue".getBytes());
putRow1.add("info".getBytes(), "age".getBytes(), "25".getBytes());
putRow1.add("info".getBytes(), "sex".getBytes(), "male".getBytes());
putRow1.add("address".getBytes(), "prevince".getBytes(), "anhui".getBytes());
putRow1.add("address".getBytes(), "city".getBytes(), "hefei".getBytes());
table.put(putRow1);
System.out.println("add row2");
Put putRow2 = new Put("xiaoli".getBytes()); // row2为行键
putRow2.add("info".getBytes(), "age".getBytes(), "21".getBytes());
putRow2.add("info".getBytes(), "sex".getBytes(), "female".getBytes());
table.put(putRow2);
// 4.htable.getScanner()获得数据，该方法返回Result类.
ResultScanner resultScanner = table.getScanner("info".getBytes());
for (Result row : resultScanner) {
System.out.format("Row\t%s\n", new String(row.getRow()));
for (Map.Entry<byte[], byte[]> entry : row.getFamilyMap("info".getBytes()).entrySet()) {
String column = new String(entry.getKey());
String value = new String(entry.getValue());
System.out.format("COLUMN\tinfo:%s\t%s\n", column, value);
}
}
// 5.删除表
/*System.out.println("删除表 lcytab");
admin.disableTable("lcytab");
admin.deleteTable("lcytab");*/
}
@Test
// HBase-3583
public void testHBase3583() throws IOException {
TableName tableName = TableName.valueOf("testHBase3583");
util.createTable(tableName, new byte[][] {A, B, C});
util.waitUntilAllRegionsAssigned(tableName);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled"},
tableName,
new Boolean[] {false, false, false, false});
HTable table = new HTable(util.getConfiguration(), tableName);
Put put = new Put(ROW);
put.add(A, A, A);
table.put(put);
Get get = new Get(ROW);
get.addColumn(A, A);
table.get(get);
// verify that scannerNext and scannerClose upcalls won't be invoked
// when we perform get().
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled"},
tableName,
new Boolean[] {true, true, false, false});
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {}
} finally {
scanner.close();
}
// now scanner hooks should be invoked.
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"wasScannerNextCalled", "wasScannerCloseCalled"},
tableName,
new Boolean[] {true, true});
util.deleteTable(tableName);
table.close();
}
@SuppressWarnings({"rawtypes", "unchecked"})
public void Read() throws IOException {
List list = new ArrayList();
Scan scan = new Scan();
scan.setBatch(0);
scan.setCaching(10000);
scan.setMaxVersions();
scan.addColumn(Bytes.toBytes("cf1"), Bytes.toBytes("total"));
ResultScanner rsScanner = table.getScanner(scan);
for (Result rs : rsScanner) {
String date = Bytes.toString(rs.getRow());
String total = Bytes.toString(rs.getValue(Bytes.toBytes("cf1"), Bytes.toBytes("total")));
list.add(date + "\t" + total);
}
for (int i = 0; i < 7; i++)
System.out.println((String) list.get(i) + "\t" + (String) list.get(i + 7));
}
@Test
// HBase-3758
public void testHBase3758() throws IOException {
TableName tableName = TableName.valueOf("testHBase3758");
util.createTable(tableName, new byte[][] {A, B, C});
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"hadDeleted", "wasScannerOpenCalled"},
tableName,
new Boolean[] {false, false});
HTable table = new HTable(util.getConfiguration(), tableName);
Put put = new Put(ROW);
put.add(A, A, A);
table.put(put);
Delete delete = new Delete(ROW);
table.delete(delete);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"hadDeleted", "wasScannerOpenCalled"},
tableName,
new Boolean[] {true, false});
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {}
} finally {
scanner.close();
}
// now scanner hooks should be invoked.
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"wasScannerOpenCalled"},
tableName,
new Boolean[] {true});
util.deleteTable(tableName);
table.close();
}
private void putDataAndVerify(
HTable table, String row, byte[] family, String value, int verifyNum) throws IOException {
System.out.println("=========Putting data :" + row);
Put put = new Put(Bytes.toBytes(row));
put.add(family, Bytes.toBytes("q1"), Bytes.toBytes(value));
table.put(put);
ResultScanner resultScanner = table.getScanner(new Scan());
List<Result> results = new ArrayList<Result>();
while (true) {
Result r = resultScanner.next();
if (r == null) break;
results.add(r);
}
resultScanner.close();
if (results.size() != verifyNum) {
System.out.println(results);
}
assertEquals(verifyNum, results.size());
}
/**
* Insert then do different types of deletes
*
* @throws Exception
*/
@Test
public void testMixedDeletes() throws Exception {
List<WALEntry> entries = new ArrayList<WALEntry>(3);
List<Cell> cells = new ArrayList<Cell>();
for (int i = 0; i < 3; i++) {
entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
}
SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
entries = new ArrayList<WALEntry>(3);
cells = new ArrayList<Cell>();
entries.add(createEntry(TABLE_NAME1, 0, KeyValue.Type.DeleteColumn, cells));
entries.add(createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily, cells));
entries.add(createEntry(TABLE_NAME1, 2, KeyValue.Type.DeleteColumn, cells));
SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
Scan scan = new Scan();
ResultScanner scanRes = table1.getScanner(scan);
assertEquals(0, scanRes.next(3).length);
}
public static void scanNum(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
Scan scan = new Scan();
scan.setCaching(5000);
Filter f = new FirstKeyOnlyFilter();
scan.setFilter(f);
HTable htable = new HTable(conf, Bytes.toBytes(tableStr));
ResultScanner scanner = htable.getScanner(scan);
int count = 0;
for (Result scannerRst : scanner) {
count++;
}
System.out.println(tableStr + ":" + count);
scanner.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public String[] getAll() throws DaoException {
try {
List<String> ret = new ArrayList<String>();
Scan s = new Scan();
s.setMaxVersions(1);
ResultScanner scanner = htable.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
ret.add(localObjectId);
}
} finally {
scanner.close();
}
return ret.toArray(new String[] {});
} catch (IOException ex) {
throw new DaoException(ex);
}
}
/*
* 遍历查询hbase表
*
* @tableName 表名
*/
public static void getResultScann(String tableName) throws IOException {
Scan scan = new Scan();
ResultScanner rs = null;
HTable table = new HTable(conf, Bytes.toBytes(tableName));
try {
rs = table.getScanner(scan);
for (Result r : rs) {
for (KeyValue kv : r.list()) {
System.out.println("row:" + Bytes.toString(kv.getRow()));
System.out.println("family:" + Bytes.toString(kv.getFamily()));
System.out.println("qualifier:" + Bytes.toString(kv.getQualifier()));
System.out.println("value:" + Bytes.toString(kv.getValue()));
System.out.println("timestamp:" + kv.getTimestamp());
System.out.println("-------------------------------------------");
}
}
} finally {
rs.close();
}
}
/**
* Deletes the specified table with all its columns. ATTENTION: Invoking this method will delete
* the table if it exists and therefore causes data loss.
*/
@Override
public void clearStorage() throws StorageException {
HBaseAdmin adm = getAdminInterface();
try { // first of all, check if table exists, if not - we are done
if (!adm.tableExists(tableName)) {
logger.debug("clearStorage() called before table {} was created, skipping.", tableName);
return;
}
} catch (IOException e) {
throw new TemporaryStorageException(e);
}
HTable table = null;
try {
table = new HTable(hconf, tableName);
Scan scan = new Scan();
scan.setBatch(100);
scan.setCacheBlocks(false);
scan.setCaching(2000);
ResultScanner scanner = null;
try {
scanner = table.getScanner(scan);
for (Result res : scanner) {
table.delete(new Delete(res.getRow()));
}
} finally {
IOUtils.closeQuietly(scanner);
}
} catch (IOException e) {
throw new TemporaryStorageException(e);
} finally {
IOUtils.closeQuietly(table);
}
}
