private Scan createScan(String applicationName, Range range, boolean scanBackward) {
Scan scan = new Scan();
scan.setCaching(this.scanCacheSize);
byte[] bApplicationName = Bytes.toBytes(applicationName);
byte[] traceIndexStartKey = SpanUtils.getTraceIndexRowKey(bApplicationName, range.getFrom());
byte[] traceIndexEndKey = SpanUtils.getTraceIndexRowKey(bApplicationName, range.getTo());
if (scanBackward) {
// start key is replaced by end key because key has been reversed
scan.setStartRow(traceIndexEndKey);
scan.setStopRow(traceIndexStartKey);
} else {
scan.setReversed(true);
scan.setStartRow(traceIndexStartKey);
scan.setStopRow(traceIndexEndKey);
}
scan.addFamily(HBaseTables.APPLICATION_TRACE_INDEX_CF_TRACE);
scan.setId("ApplicationTraceIndexScan");
// toString() method of Scan converts a message to json format so it is slow for the first time.
logger.trace("create scan:{}", scan);
return scan;
}
public static ResultScanner getAllFamilyList(
String startRowRange, String stopRowRange, byte[][] cfs, String ctableName) {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(startRowRange));
for (byte[] family : cfs) {
scan.addFamily(family);
}
if (stopRowRange != null) {
scan.setStopRow(Bytes.toBytes(stopRowRange));
}
ResultScanner resultScanner = null;
try {
resultScanner = tblMngr.getTable(ctableName).getScanner(scan);
} catch (Exception e) {
}
return resultScanner;
}
/**
* Gets an hbase scanner.
*
* @return An hbase scanner.
* @throws IOException Error accessing hbase.
*/
@Override
protected Scan getScanner() throws IOException {
Scan scan = super.getScanner();
if (this.query != null) {
if (this.query.getStartKey() != null) {
scan.setStartRow(this.query.getStartKey());
}
if (this.query.getEndKey() != null) {
scan.setStopRow(padWithMaxUnicode(this.query.getEndKey()));
}
if (this.query.getStartDate() != null && this.query.getEndDate() != null) {
scan.setTimeRange(this.query.getStartDate().getTime(), this.query.getEndDate().getTime());
}
if (this.query.getWord() != null || this.query.getOperator().isUnary()) {
WritableByteArrayComparable comparator;
switch (this.query.getOperator()) {
case Contains:
comparator = new SubstringComparator(this.query.getWord());
break;
case StartsWith:
comparator = new RegexStringComparator(String.format("^%s.*", this.query.getWord()));
break;
case EndsWith:
comparator = new RegexStringComparator(String.format(".*%s$", this.query.getWord()));
break;
case Less:
case LessOrEqual:
case Equal:
case NotEqual:
case GreaterOrEqual:
case Greater:
comparator = new BinaryComparator(this.query.getWordAsByteArray());
break;
case IsNull:
case IsNotNull:
comparator = new BinaryComparator(EMPTY_BYTES_ARRAY);
break;
default:
throw new IllegalArgumentException(
String.format(
"The specified operator type '%s' is not supported.",
this.query.getOperator()));
}
scan.setFilter(
new SingleColumnValueFilter(
Bytes.toBytesBinary(this.query.getFamily()),
Bytes.toBytesBinary(this.query.getColumn()),
this.query.getOperator().toFilter(),
comparator));
}
}
return scan;
}
/**
* 遍历多行
*
* @param tableName 表名
* @param start_rowkey 开始行键
* @param stop_rowkey 结束行键
* @return 行列表
*/
public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
ResultScanner rowstmp = null;
ArrayList<HbaseRow> rows = null;
try {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(start_rowkey));
scan.setStopRow(Bytes.toBytes(stop_rowkey));
HTable table = new HTable(conf, Bytes.toBytes(tableName));
rowstmp = table.getScanner(scan);
rows = new ArrayList<>();
for (Result rowtmp : rowstmp) {
HbaseRow row = new HbaseRow();
row.rowkey = Bytes.toString(rowtmp.getRow());
for (Cell cell : rowtmp.listCells()) {
HbaseColumn col = new HbaseColumn(cell);
row.cols.add(col);
}
rows.add(row);
}
} catch (Exception e) {
logger.error("scanRows failed", e);
} finally {
rowstmp.close();
}
return rows;
}
/**
* Get a collection of Scans, one per region, that cover the range of the table having the given
* key prefix. Thes will be used as the map task input splits.
*/
public static List<Scan> scansThisCubeOnly(byte[] keyPrefix, byte[][] splitKeys)
throws IOException {
Scan copyScan = new Scan();
copyScan.setCaching(5000);
copyScan.setCacheBlocks(false);
// Hack: generate a key that probably comes after all this cube's keys but doesn't include any
// keys not belonging to this cube.
byte[] keyAfterCube = ArrayUtils.addAll(keyPrefix, fiftyBytesFF);
List<Scan> scans = new ArrayList<Scan>();
Scan scanUnderConstruction = new Scan(copyScan);
for (byte[] splitKey : splitKeys) {
scanUnderConstruction.setStopRow(splitKey);
// Coerce scan to only touch keys belonging to this cube
Scan truncated = truncateScan(scanUnderConstruction, keyPrefix, keyAfterCube);
if (truncated != null) {
scans.add(truncated);
}
scanUnderConstruction = new Scan(copyScan);
scanUnderConstruction.setStartRow(splitKey);
}
// There's another region from last split key to the end of the table.
Scan truncated = truncateScan(scanUnderConstruction, keyPrefix, keyAfterCube);
if (truncated != null) {
scans.add(truncated);
}
return scans;
}
/**
* Given a scan and a key range, return a new Scan whose range is truncated to only include keys
* in that range. Returns null if the Scan does not overlap the given range.
*/
private static final Scan truncateScan(Scan scan, byte[] rangeStart, byte[] rangeEnd) {
byte[] scanStart = scan.getStartRow();
byte[] scanEnd = scan.getStopRow();
if (scanEnd.length > 0 && bytesCompare(scanEnd, rangeStart) <= 0) {
// The entire scan range is before the entire cube key range
return null;
} else if (scanStart.length > 0 && bytesCompare(scanStart, rangeEnd) >= 0) {
// The entire scan range is after the entire cube key range
return null;
} else {
// Now we now that the scan range at least partially overlaps the cube key range.
Scan truncated;
try {
truncated = new Scan(scan); // make a copy, don't modify input scan
} catch (IOException e) {
throw new RuntimeException(); // This is not plausible
}
if (scanStart.length == 0 || bytesCompare(rangeStart, scanStart) > 0) {
// The scan includes extra keys at the beginning that are not part of the cube. Move
// the scan start point so that it only touches keys belonging to the cube.
truncated.setStartRow(rangeStart);
}
if (scanEnd.length == 0 || bytesCompare(rangeEnd, scanEnd) < 0) {
// The scan includes extra keys at the end that are not part of the cube. Move the
// scan end point so it only touches keys belonging to the cube.
truncated.setStopRow(rangeEnd);
}
return truncated;
}
}
public static ResultScanner getList(
String startRowRange,
String stopRowRange,
byte[] cf1,
byte[] cf2,
long limit,
FilterList filterList,
String ctableName) {
Scan scan = new Scan();
scan.addFamily(cf1);
scan.setStartRow(Bytes.toBytes(startRowRange));
if (stopRowRange != null) {
scan.setStopRow(Bytes.toBytes(stopRowRange));
}
if (limit != 0) {
filterList.addFilter(new PageFilter(limit));
} else {
filterList.addFilter(new PageFilter(100));
}
scan.setFilter(filterList);
ResultScanner resultScanner = null;
try {
resultScanner = tblMngr.getTable(ctableName).getScanner(scan);
} catch (Exception e) {
}
return resultScanner;
}
public Scan generateScan(
String[] rowRange, FilterList filterList, String[] family, String[] columns, int maxVersion)
throws Exception {
if (table == null) throw new Exception("No table handler");
if (cacheSize < 0) throw new Exception("should set cache size before scanning");
Scan scan = null;
try {
scan = new Scan();
scan.setCaching(this.cacheSize);
scan.setCacheBlocks(this.blockCached);
scan.setFilter(filterList);
if (maxVersion > 0) scan.setMaxVersions(maxVersion);
if (rowRange != null) {
scan.setStartRow(rowRange[0].getBytes());
if (rowRange.length == 2) scan.setStopRow(rowRange[1].getBytes());
}
if (columns != null) {
for (int i = 0; i < columns.length; i++) {
scan.addColumn(family[0].getBytes(), columns[i].getBytes());
// System.out.println(family[i]+";"+columns[i]);
}
} else {
scan.addFamily(family[0].getBytes());
}
} catch (Exception e) {
e.printStackTrace();
}
return scan;
}
// 获取扫描器对象
private static Scan getScan(String startRow, String stopRow) {
Scan scan = new Scan();
if (startRow != null && stopRow != null) {
scan.setStartRow(getBytes(startRow));
scan.setStopRow(getBytes(stopRow));
}
return scan;
}
public HBaseSecondaryIndexCursor(
HBaseSecondaryIndex index, TableFilter filter, byte[] startKey, byte[] endKey) {
defaultColumnFamilyName =
Bytes.toBytes(((HBaseTable) filter.getTable()).getDefaultColumnFamilyName());
secondaryIndex = index;
session = (HBaseSession) filter.getSession();
Prepared p = filter.getPrepared();
if (p instanceof WithWhereClause) {
regionName = Bytes.toBytes(((WithWhereClause) p).getWhereClauseSupport().getRegionName());
}
if (regionName == null) throw new RuntimeException("regionName is null");
fetchSize = filter.getPrepared().getFetchSize();
// 非查询的操作一般不设置fetchSize，此时fetchSize为0，所以要设置一个默认值
// if (fetchSize < 1 && !filter.getPrepared().isQuery())
if (fetchSize < 1) fetchSize = SysProperties.SERVER_RESULT_SET_FETCH_SIZE;
if (filter.getSelect() != null) columns = filter.getSelect().getColumns(filter);
else columns = Arrays.asList(filter.getTable().getColumns());
if (startKey == null) startKey = HConstants.EMPTY_BYTE_ARRAY;
if (endKey == null) endKey = HConstants.EMPTY_BYTE_ARRAY;
Scan scan = new Scan();
scan.setMaxVersions(1);
try {
HRegionInfo info = session.getRegionServer().getRegionInfo(regionName);
if (Bytes.compareTo(startKey, info.getStartKey()) >= 0) scan.setStartRow(startKey);
else scan.setStartRow(info.getStartKey());
if (Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)) scan.setStopRow(info.getEndKey());
else if (Bytes.compareTo(endKey, info.getEndKey()) < 0) scan.setStopRow(endKey);
else scan.setStopRow(info.getEndKey());
scan.addColumn(HBaseSecondaryIndex.PSEUDO_FAMILY, HBaseSecondaryIndex.PSEUDO_COLUMN);
scannerId = session.getRegionServer().openScanner(regionName, scan);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* This method creates a Scan object that will only scan catalog rows that belong to the specified
* table. It doesn't specify any columns. This is a better alternative to just using a start row
* and scan until it hits a new table since that requires parsing the HRI to get the table name.
*
* @param tableName bytes of table's name
* @return configured Scan object
*/
public static Scan getScanForTableName(byte[] tableName) {
String strName = Bytes.toString(tableName);
// Start key is just the table name with delimiters
byte[] startKey = Bytes.toBytes(strName + ",,");
// Stop key appends the smallest possible char to the table name
byte[] stopKey = Bytes.toBytes(strName + " ,,");
Scan scan = new Scan(startKey);
scan.setStopRow(stopKey);
return scan;
}
@Test
public void testStdWithValidRange2WithNoCQ() throws Throwable {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addFamily(TEST_FAMILY);
scan.setStartRow(ROWS[6]);
scan.setStopRow(ROWS[7]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
double std = aClient.std(TEST_TABLE, ci, scan);
assertEquals(0, std, 0);
}
/** @throws Throwable */
@Test
public void testStdWithValidRange2() throws Throwable {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[5]);
scan.setStopRow(ROWS[15]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
double std = aClient.std(TEST_TABLE, ci, scan);
assertEquals(2.87, std, 0.05d);
}
@Test
public void testMinWithValidRangeWithNoCQ() throws Throwable {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addFamily(TEST_FAMILY);
scan.setStartRow(HConstants.EMPTY_START_ROW);
scan.setStopRow(HConstants.EMPTY_END_ROW);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
long min = aClient.min(TEST_TABLE, ci, scan);
assertEquals(0, min);
}
/**
* This will test rowcount with a valid range, i.e., a subset of rows. It will be the most common
* use case.
*
* @throws Throwable
*/
@Test
public void testRowCountWithValidRange() throws Throwable {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[2]);
scan.setStopRow(ROWS[14]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
long rowCount = aClient.rowCount(TEST_TABLE, ci, scan);
assertEquals(12, rowCount);
}
@Override
public void tweet(User user, String tweetText) throws IOException {
final long epoch = System.currentTimeMillis();
final long tranposeEpoch = Long.MAX_VALUE - epoch;
final byte[] epochBytes = Bytes.toBytes(epoch);
final byte[] tweetBytes = Bytes.toBytes(tweetText);
byte[] nameBytes = Bytes.toBytes(user.getName());
/** put tweet into tweets */
Put tweetRowPut = new Put(generateTweetId(user));
tweetRowPut.add(_DEFAULT, _NAME, nameBytes);
tweetRowPut.add(_DEFAULT, _MAIL, Bytes.toBytes(user.getEmail()));
tweetRowPut.add(_DEFAULT, _TWEET, tweetBytes);
tweetRowPut.add(_DEFAULT, _TIME, epochBytes);
tweetsTable.put(tweetRowPut);
/** put tweets for followers */
Scan followerScan = new Scan();
followerScan.setStartRow(Bytes.toBytes(user.getUserId() + "-"));
followerScan.setStopRow(Bytes.toBytes((user.getUserId() + 1) + "-"));
ResultScanner followerRS = followersTable.getScanner(followerScan);
/** put users on tweet to her own tweetline */
Put put =
new Put(Bytes.toBytes(user.getUserId() + "-" + tranposeEpoch + "-" + user.getUserId()));
put.add(_DEFAULT, _NAME, nameBytes);
put.add(_DEFAULT, _TWEET, tweetBytes);
put.add(_DEFAULT, _TIME, epochBytes);
List<Row> puts = new ArrayList<Row>();
puts.add(put);
for (Result result : followerRS) {
Long followerid = Bytes.toLong(result.getColumnLatest(_DEFAULT, _USERID).getValue());
put = new Put(Bytes.toBytes(followerid + "-" + tranposeEpoch + "-" + user.getUserId()));
put.add(_DEFAULT, _NAME, nameBytes);
put.add(_DEFAULT, _TWEET, tweetBytes);
put.add(_DEFAULT, _TIME, epochBytes);
puts.add(put);
}
followerRS.close();
try {
tweetlineTable.batch(puts);
} catch (InterruptedException e) {
e.printStackTrace(); // @TODO log and handle properly.
}
}
/**
* Configure the scan according to the row options.
*
* @param scan The HBase scan descriptor to configure.
* @throws IOException If there is an error.
*/
private void configureScanWithRowOptions(Scan scan) throws IOException {
if (null != mRowOptions.getStartRow()) {
scan.setStartRow(mRowOptions.getStartRow().getHBaseRowKey());
}
if (null != mRowOptions.getLimitRow()) {
scan.setStopRow(mRowOptions.getLimitRow().getHBaseRowKey());
}
if (null != mRowOptions.getRowFilter()) {
KijiTableLayout tableLayout = mInputTable.getLayout();
KijiSchemaTable schemaTable = mInputTable.getKiji().getSchemaTable();
new KijiRowFilterApplicator(mRowOptions.getRowFilter(), tableLayout, schemaTable)
.applyTo(scan);
}
}
public <V> List<V> list(
String tableName, byte[] startKey, byte[] stopKey, Filter filter, HbaseMapper<V> mapper)
throws Exception {
HTableInterface htable = dataSource.getConnection(tableName);
Scan scan = new Scan();
scan.setStartRow(startKey);
scan.setStopRow(stopKey);
scan.setFilter(filter);
log.info("scan is -[" + scan + "]");
ResultScanner resultScanner = htable.getScanner(scan);
List<V> result = trasfer(resultScanner, mapper);
htable.close();
log.info("list " + result.size() + " objects from " + tableName);
return result;
}
/** This should return a 0 */
@Test
public void testRowCountWithNullCF() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.setStartRow(ROWS[5]);
scan.setStopRow(ROWS[15]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
long rowCount = -1;
try {
rowCount = aClient.rowCount(TEST_TABLE, ci, scan);
} catch (Throwable e) {
rowCount = 0;
}
assertEquals(0, rowCount);
}
@Test
public void testAvgWithInvalidRange() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[5]);
scan.setStopRow(ROWS[1]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
Double avg = null;
try {
avg = aClient.avg(TEST_TABLE, ci, scan);
} catch (Throwable e) {
}
assertEquals(null, avg); // control should go to the catch block
}
@Test
public void testStdWithValidRangeWithNullCF() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.setStartRow(ROWS[6]);
scan.setStopRow(ROWS[17]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
Double std = null;
try {
std = aClient.std(TEST_TABLE, ci, scan);
} catch (Throwable e) {
}
assertEquals(null, std); // CP will throw an IOException about the
// null column family, and max will be set to 0
}
@Test
public void testStdWithInvalidRange() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addFamily(TEST_FAMILY);
scan.setStartRow(ROWS[6]);
scan.setStopRow(ROWS[1]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
Double std = null;
try {
std = aClient.std(TEST_TABLE, ci, scan);
} catch (Throwable e) {
}
assertEquals(null, std); // control should go to the catch block
}
@Test
public void testMaxWithInvalidRange2() throws Throwable {
long max = Long.MIN_VALUE;
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[4]);
scan.setStopRow(ROWS[4]);
try {
AggregationClient aClient = new AggregationClient(conf);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
max = aClient.max(TEST_TABLE, ci, scan);
} catch (Exception e) {
max = 0;
}
assertEquals(0, max); // control should go to the catch block
}
@Override
public int run(String[] strings) throws Exception {
Configuration conf = new Configuration();
// String inputFileName = "/cluster/gmm.seq";
String outputFileName = "/cluster/matrix_intermediate_" + level + ".seq";
int result;
System.out.println("level:" + level);
conf.set("level", level + "");
String table = "ClusterDatabase";
// String seqFileName = "/cluster/gmm.seq";
Scan scan = new Scan();
scan.setStartRow((level + "|").getBytes());
scan.setStopRow(
Bytes.add((level + "|").getBytes(), Bytes.toBytes("ffffffffffffffffffffffffffffffff")));
scan.addColumn("Cluster".getBytes(), "GMM".getBytes());
// try (FileSystem fileSystem = FileSystem.get(conf)) {
FileSystem fileSystem = FileSystem.get(conf);
Path outputpath = new Path(outputFileName);
if (fileSystem.exists(outputpath)) {
fileSystem.delete(outputpath, true);
}
Job job = new Job(conf, "Matrix Creation I From HBase");
job.setJarByClass(MatrixCreationI.class);
TableMapReduceUtil.initTableMapperJob(
table, scan, MatrixMapper.class, IntWritable.class, Text.class, job);
job.setReducerClass(MatrixReducer.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
// job.setInputFormatClass(TableInputFormat.class);
// job.setOutputFormatClass(TextOutputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setNumReduceTasks(1);
// FileInputFormat.addInputPath(job, new Path(inputFileName + "/part*"));
FileOutputFormat.setOutputPath(job, outputpath);
result = job.waitForCompletion(true) ? 0 : 1;
// }
return result;
}
@Override
public Result getCurrentRowState(Mutation m, Collection<? extends ColumnReference> columns)
throws IOException {
byte[] row = m.getRow();
// need to use a scan here so we can get raw state, which Get doesn't provide.
Scan s = IndexManagementUtil.newLocalStateScan(Collections.singletonList(columns));
s.setStartRow(row);
s.setStopRow(row);
HRegion region = this.env.getRegion();
RegionScanner scanner = region.getScanner(s);
List<KeyValue> kvs = new ArrayList<KeyValue>(1);
boolean more = scanner.next(kvs);
assert !more : "Got more than one result when scanning" + " a single row in the primary table!";
Result r = new Result(kvs);
scanner.close();
return r;
}
/**
* This will test the row count with startrow > endrow. The result should be -1.
*
* @throws Throwable
*/
@Test
public void testRowCountWithInvalidRange1() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[5]);
scan.setStopRow(ROWS[2]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
long rowCount = -1;
try {
rowCount = aClient.rowCount(TEST_TABLE, ci, scan);
} catch (Throwable e) {
myLog.error("Exception thrown in the invalidRange method" + e.getStackTrace());
}
assertEquals(-1, rowCount);
}
// Start/stop row must be swapped if scan is being done in reverse
public static void setupReverseScan(Scan scan) {
if (isReversed(scan)) {
byte[] startRow = scan.getStartRow();
byte[] stopRow = scan.getStopRow();
byte[] newStartRow = startRow;
byte[] newStopRow = stopRow;
if (startRow.length != 0) {
/*
* Must get previous key because this is going from an inclusive start key to an exclusive stop key, and
* we need the start key to be included. We get the previous key by decrementing the last byte by one.
* However, with variable length data types, we need to fill with the max byte value, otherwise, if the
* start key is 'ab', we lower it to 'aa' which would cause 'aab' to be included (which isn't correct).
* So we fill with a 0xFF byte to prevent this. A single 0xFF would be enough for our primitive types (as
* that byte wouldn't occur), but for an arbitrary VARBINARY key we can't know how many bytes to tack
* on. It's lame of HBase to force us to do this.
*/
newStartRow =
Arrays.copyOf(startRow, startRow.length + MAX_FILL_LENGTH_FOR_PREVIOUS_KEY.length);
if (ByteUtil.previousKey(newStartRow, startRow.length)) {
System.arraycopy(
MAX_FILL_LENGTH_FOR_PREVIOUS_KEY,
0,
newStartRow,
startRow.length,
MAX_FILL_LENGTH_FOR_PREVIOUS_KEY.length);
} else {
newStartRow = HConstants.EMPTY_START_ROW;
}
}
if (stopRow.length != 0) {
// Must add null byte because we need the start to be exclusive while it was inclusive
newStopRow = ByteUtil.concat(stopRow, QueryConstants.SEPARATOR_BYTE_ARRAY);
}
scan.setStartRow(newStopRow);
scan.setStopRow(newStartRow);
scan.setReversed(true);
}
}
@Override
public List<Tweet> loadTweets(User user) throws IOException {
Scan tweetlineScan = new Scan();
tweetlineScan.setStartRow(Bytes.toBytes(user.getUserId() + "-"));
tweetlineScan.setStopRow(Bytes.toBytes((user.getUserId() + 1) + "-"));
ResultScanner tweetlineRS = tweetlineTable.getScanner(tweetlineScan);
List<Tweet> tweets = new ArrayList();
for (Result result : tweetlineRS) {
Tweet tweet = new Tweet();
tweet.setMsg(Bytes.toString(result.getValue(_DEFAULT, _TWEET)));
tweet.setUserName(Bytes.toString(result.getValue(_DEFAULT, _NAME)));
tweet.setTime(new Date(Bytes.toLong(result.getValue(_DEFAULT, _TIME))));
tweets.add(tweet);
}
tweetlineRS.close();
return tweets;
}
/*
* 遍历查询hbase表
*
* @tableName 表名
*/
public static void getResultScann(String tableName, String start_rowkey, String stop_rowkey)
throws IOException {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(start_rowkey));
scan.setStopRow(Bytes.toBytes(stop_rowkey));
ResultScanner rs = null;
HTable table = new HTable(conf, Bytes.toBytes(tableName));
try {
rs = table.getScanner(scan);
for (Result r : rs) {
for (KeyValue kv : r.list()) {
System.out.println("row:" + Bytes.toString(kv.getRow()));
System.out.println("family:" + Bytes.toString(kv.getFamily()));
System.out.println("qualifier:" + Bytes.toString(kv.getQualifier()));
System.out.println("value:" + Bytes.toString(kv.getValue()));
System.out.println("timestamp:" + kv.getTimestamp());
System.out.println("-------------------------------------------");
}
}
} finally {
rs.close();
}
}
/**
* It should be noticed that the stop row in scan is not included as default
*
* @param rowRange
* @param filterList
* @param family
* @param columns
* @param maxVersion
* @return
* @throws Exception
*/
public ResultScanner getResultSet(
String[] rowRange, FilterList filterList, String[] family, String[] columns, int maxVersion)
throws Exception {
if (table == null) throw new Exception("No table handler");
if (cacheSize < 0) throw new Exception("should set cache size before scanning");
Scan scan = null;
ResultScanner rscanner = null;
try {
scan = new Scan();
scan.setCaching(this.cacheSize);
scan.setCacheBlocks(blockCached);
scan.setFilter(filterList);
if (maxVersion > 0) scan.setMaxVersions(maxVersion);
// scan exclude the stop row directly, so have to make a little difference of the stop row
if (rowRange != null) {
scan.setStartRow(rowRange[0].getBytes());
if (rowRange.length == 2 && rowRange[1] != null) scan.setStopRow((rowRange[1]).getBytes());
}
if (columns != null) {
for (int i = 0; i < columns.length; i++) {
scan.addColumn(family[0].getBytes(), columns[i].getBytes());
}
}
rscanner = this.table.getScanner(scan);
} catch (Exception e) {
e.printStackTrace();
}
return rscanner;
}