/**
* Creates the scanner for flushing snapshot. Also calls coprocessors.
*
* @param snapshotScanner
* @param smallestReadPoint
* @return The scanner; null if coprocessor is canceling the flush.
*/
protected InternalScanner createScanner(KeyValueScanner snapshotScanner, long smallestReadPoint)
throws IOException {
InternalScanner scanner = null;
if (store.getCoprocessorHost() != null) {
scanner = store.getCoprocessorHost().preFlushScannerOpen(store, snapshotScanner);
}
if (scanner == null) {
Scan scan = new Scan();
scan.setMaxVersions(store.getScanInfo().getMaxVersions());
scanner =
new StoreScanner(
store,
store.getScanInfo(),
scan,
Collections.singletonList(snapshotScanner),
ScanType.COMPACT_RETAIN_DELETES,
smallestReadPoint,
HConstants.OLDEST_TIMESTAMP);
}
assert scanner != null;
if (store.getCoprocessorHost() != null) {
try {
return store.getCoprocessorHost().preFlush(store, scanner);
} catch (IOException ioe) {
scanner.close();
throw ioe;
}
}
return scanner;
}
/** Ensure that expired delete family markers don't override valid puts */
public void testExpiredDeleteFamily() throws Exception {
long now = System.currentTimeMillis();
KeyValue[] kvs =
new KeyValue[] {
new KeyValue(
Bytes.toBytes("R1"),
Bytes.toBytes("cf"),
null,
now - 1000,
KeyValue.Type.DeleteFamily),
KeyValueTestUtil.create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"),
};
List<KeyValueScanner> scanners = scanFixture(kvs);
Scan scan = new Scan();
scan.setMaxVersions(1);
// scanner with ttl equal to 500
ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, KeyValue.COMPARATOR);
ScanType scanType = ScanType.USER_SCAN;
StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, null, scanners);
List<KeyValue> results = new ArrayList<KeyValue>();
assertEquals(true, scanner.next(results));
assertEquals(1, results.size());
assertEquals(kvs[1], results.get(0));
results.clear();
assertEquals(false, scanner.next(results));
}
public String[] getObjectIDs(String objectType, String... tags) throws IOException {
List<String> ret = new ArrayList<String>();
FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
SingleColumnValueFilter filter1 =
new SingleColumnValueFilter(
"tags".getBytes(), "OBJECTTYPE".getBytes(), CompareOp.EQUAL, Bytes.toBytes(objectType));
list.addFilter(filter1);
for (String tag : tags) {
SingleColumnValueFilter filter2 =
new SingleColumnValueFilter(
"tags".getBytes(), tag.toUpperCase().getBytes(), CompareOp.EQUAL, Bytes.toBytes(1));
filter2.setFilterIfMissing(true);
list.addFilter(filter2);
}
Scan s = new Scan();
s.setFilter(list);
s.setMaxVersions(1);
ResultScanner scanner = htable.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
String localObjectType =
new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
ret.add(localObjectId);
}
} finally {
scanner.close();
}
return ret.toArray(new String[] {});
}
public List<String> getStationsNearPoint_Schema2(double lat, double lon) throws IOException {
Scan scan = new Scan();
scan.addFamily(BixiConstant.SCHEMA2_CLUSTER_FAMILY_NAME.getBytes());
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
boolean hasMoreResult = false;
List<KeyValue> res = new ArrayList<KeyValue>();
List<String> result = new ArrayList<String>();
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String clusterId = Bytes.toString(kv.getRow());
String[] parts = clusterId.split(":");
double cLat = Double.parseDouble(parts[0]);
double cLon = Double.parseDouble(parts[1]);
double dx = Double.parseDouble(parts[2]);
double dy = Double.parseDouble(parts[3]);
double distx = lat - cLat;
double disty = lon - cLon;
if (distx >= 0 && distx <= dx && disty >= 0 && disty <= dy) {
// get stations in cluster
result.add(Bytes.toString(kv.getQualifier()));
}
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
/*
* (non-Javadoc)
*
* @see com.impetus.kunderahbase.dao.user.UserDao#findAllByUserName(int)
*/
@Override
public void findAllByUserName(int count) {
Filter filter =
new SingleColumnValueFilter(
Bytes.toBytes("user_name"),
Bytes.toBytes("user_name"),
CompareOp.EQUAL,
Bytes.toBytes("Amry"));
Scan scan = new Scan();
scan.setFilter(filter);
scan.addColumn(Bytes.toBytes("user_name"), Bytes.toBytes("user_name"));
scan.addColumn(Bytes.toBytes("user_nameCnt"), Bytes.toBytes("user_nameCnt"));
scan.addColumn(Bytes.toBytes("password"), Bytes.toBytes("password"));
scan.addColumn(Bytes.toBytes("relation"), Bytes.toBytes("relation"));
ResultScanner scanner = null;
int counter = 0;
try {
scanner = hTablePool.getTable("User").getScanner(scan);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
assert scanner != null;
for (Result result : scanner) {
counter++;
assert result != null;
}
assert counter != 0;
assert counter == count;
}
/**
* Given a scan and a key range, return a new Scan whose range is truncated to only include keys
* in that range. Returns null if the Scan does not overlap the given range.
*/
private static final Scan truncateScan(Scan scan, byte[] rangeStart, byte[] rangeEnd) {
byte[] scanStart = scan.getStartRow();
byte[] scanEnd = scan.getStopRow();
if (scanEnd.length > 0 && bytesCompare(scanEnd, rangeStart) <= 0) {
// The entire scan range is before the entire cube key range
return null;
} else if (scanStart.length > 0 && bytesCompare(scanStart, rangeEnd) >= 0) {
// The entire scan range is after the entire cube key range
return null;
} else {
// Now we now that the scan range at least partially overlaps the cube key range.
Scan truncated;
try {
truncated = new Scan(scan); // make a copy, don't modify input scan
} catch (IOException e) {
throw new RuntimeException(); // This is not plausible
}
if (scanStart.length == 0 || bytesCompare(rangeStart, scanStart) > 0) {
// The scan includes extra keys at the beginning that are not part of the cube. Move
// the scan start point so that it only touches keys belonging to the cube.
truncated.setStartRow(rangeStart);
}
if (scanEnd.length == 0 || bytesCompare(rangeEnd, scanEnd) < 0) {
// The scan includes extra keys at the end that are not part of the cube. Move the
// scan end point so it only touches keys belonging to the cube.
truncated.setStopRow(rangeEnd);
}
return truncated;
}
}
@Override
public void process(long now, HRegion region, List<Mutation> mutations, WALEdit walEdit)
throws IOException {
List<Cell> kvs = new ArrayList<Cell>();
{ // First scan to get friends of the person
Scan scan = new Scan(row, row);
scan.addColumn(FAM, person);
doScan(region, scan, kvs);
}
// Second scan to get friends of friends
Scan scan = new Scan(row, row);
for (Cell kv : kvs) {
byte[] friends = CellUtil.cloneValue(kv);
for (byte f : friends) {
scan.addColumn(FAM, new byte[] {f});
}
}
doScan(region, scan, kvs);
// Collect result
result.clear();
for (Cell kv : kvs) {
for (byte b : CellUtil.cloneValue(kv)) {
result.add((char) b + "");
}
}
}
public static ResultScanner getAllFamilyList(
String startRowRange, String stopRowRange, byte[][] cfs, String ctableName) {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(startRowRange));
for (byte[] family : cfs) {
scan.addFamily(family);
}
if (stopRowRange != null) {
scan.setStopRow(Bytes.toBytes(stopRowRange));
}
ResultScanner resultScanner = null;
try {
resultScanner = tblMngr.getTable(ctableName).getScanner(scan);
} catch (Exception e) {
}
return resultScanner;
}
@Test
public void testInclusiveStopFilter() throws Exception {
// Grab rows from group one
// If we just use start/stop row, we get total/2 - 1 rows
long expectedRows = (numRows / 2) - 1;
long expectedKeys = colsPerRow;
Scan s = new Scan(Bytes.toBytes("testRowOne-0"), Bytes.toBytes("testRowOne-3"));
verifyScan(s, expectedRows, expectedKeys);
// Now use start row with inclusive stop filter
expectedRows = numRows / 2;
s = new Scan(Bytes.toBytes("testRowOne-0"));
s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-3")));
verifyScan(s, expectedRows, expectedKeys);
// Grab rows from group two
// If we just use start/stop row, we get total/2 - 1 rows
expectedRows = (numRows / 2) - 1;
expectedKeys = colsPerRow;
s = new Scan(Bytes.toBytes("testRowTwo-0"), Bytes.toBytes("testRowTwo-3"));
verifyScan(s, expectedRows, expectedKeys);
// Now use start row with inclusive stop filter
expectedRows = numRows / 2;
s = new Scan(Bytes.toBytes("testRowTwo-0"));
s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-3")));
verifyScan(s, expectedRows, expectedKeys);
}
/**
* Sets up the actual job.
*
* @param conf The current configuration.
* @param args The command line parameters.
* @return The newly created job.
* @throws IOException When setting up the job fails.
*/
public static Job createSubmittableJob(Configuration conf, String[] args) throws IOException {
String tableName = args[0];
Job job = new Job(conf, NAME + "_" + tableName);
job.setJarByClass(CachingRowCounter.class);
// Columns are space delimited
StringBuilder sb = new StringBuilder();
final int columnoffset = 1;
for (int i = columnoffset; i < args.length; i++) {
if (i > columnoffset) {
sb.append(" ");
}
sb.append(args[i]);
}
Scan scan = new Scan();
scan.setFilter(new FirstKeyOnlyFilter());
if (sb.length() > 0) {
for (String columnName : sb.toString().split(" ")) {
String[] fields = columnName.split(":");
if (fields.length == 1) {
scan.addFamily(Bytes.toBytes(fields[0]));
} else {
scan.addColumn(Bytes.toBytes(fields[0]), Bytes.toBytes(fields[1]));
}
}
}
scan.setCaching(100);
// Second argument is the table name.
job.setOutputFormatClass(NullOutputFormat.class);
TableMapReduceUtil.initTableMapperJob(
tableName, scan, RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
job.setNumReduceTasks(0);
return job;
}
public static ResultScanner getList(
String startRowRange,
String stopRowRange,
byte[] cf1,
byte[] cf2,
long limit,
FilterList filterList,
String ctableName) {
Scan scan = new Scan();
scan.addFamily(cf1);
scan.setStartRow(Bytes.toBytes(startRowRange));
if (stopRowRange != null) {
scan.setStopRow(Bytes.toBytes(stopRowRange));
}
if (limit != 0) {
filterList.addFilter(new PageFilter(limit));
} else {
filterList.addFilter(new PageFilter(100));
}
scan.setFilter(filterList);
ResultScanner resultScanner = null;
try {
resultScanner = tblMngr.getTable(ctableName).getScanner(scan);
} catch (Exception e) {
}
return resultScanner;
}
@Test
public void shouldRunMapReduce() throws Exception {
// given
Job job = new Job(configuration, "Average Rating");
job.setJarByClass(AverageRatingMapper.class);
Scan scan = new Scan();
scan.setCaching(500);
scan.setCacheBlocks(false);
scan.addFamily(Bytes.toBytes(LoadMovieRatingData.FAMILY_NAME));
TableMapReduceUtil.initTableMapperJob(
LoadMovieRatingData.TABLE_NAME,
scan,
AverageRatingMapper.class,
Text.class,
DoubleWritable.class,
job);
job.setReducerClass(RatingExportReducer.class);
job.setNumReduceTasks(1);
FileOutputFormat.setOutputPath(
job, new Path("/tmp/mr/mySummaryFile_" + System.currentTimeMillis()));
// when
boolean succeeded = job.waitForCompletion(true);
// then
assertThat(succeeded).isTrue();
}
@Override
protected List<RawResource> getAllResources(String rangeStart, String rangeEnd)
throws IOException {
byte[] startRow = Bytes.toBytes(rangeStart);
byte[] endRow = plusZero(Bytes.toBytes(rangeEnd));
Scan scan = new Scan(startRow, endRow);
scan.addColumn(B_FAMILY, B_COLUMN_TS);
scan.addColumn(B_FAMILY, B_COLUMN);
HTableInterface table = getConnection().getTable(getAllInOneTableName());
List<RawResource> result = Lists.newArrayList();
try {
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
result.add(new RawResource(getInputStream(Bytes.toString(r.getRow()), r), getTimestamp(r)));
}
} catch (IOException e) {
for (RawResource rawResource : result) {
IOUtils.closeQuietly(rawResource.resource);
}
throw e;
} finally {
IOUtils.closeQuietly(table);
}
return result;
}
@Override
protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
assert resPath.startsWith("/");
String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
byte[] startRow = Bytes.toBytes(lookForPrefix);
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
ArrayList<String> result = new ArrayList<String>();
HTableInterface table = getConnection().getTable(getAllInOneTableName());
Scan scan = new Scan(startRow, endRow);
scan.setFilter(new KeyOnlyFilter());
try {
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
String path = Bytes.toString(r.getRow());
assert path.startsWith(lookForPrefix);
int cut = path.indexOf('/', lookForPrefix.length());
String child = cut < 0 ? path : path.substring(0, cut);
if (result.contains(child) == false) result.add(child);
}
} finally {
IOUtils.closeQuietly(table);
}
// return null to indicate not a folder
return result.isEmpty() ? null : result;
}
/**
* 遍历多行
*
* @param tableName 表名
* @param start_rowkey 开始行键
* @param stop_rowkey 结束行键
* @return 行列表
*/
public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
ResultScanner rowstmp = null;
ArrayList<HbaseRow> rows = null;
try {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(start_rowkey));
scan.setStopRow(Bytes.toBytes(stop_rowkey));
HTable table = new HTable(conf, Bytes.toBytes(tableName));
rowstmp = table.getScanner(scan);
rows = new ArrayList<>();
for (Result rowtmp : rowstmp) {
HbaseRow row = new HbaseRow();
row.rowkey = Bytes.toString(rowtmp.getRow());
for (Cell cell : rowtmp.listCells()) {
HbaseColumn col = new HbaseColumn(cell);
row.cols.add(col);
}
rows.add(row);
}
} catch (Exception e) {
logger.error("scanRows failed", e);
} finally {
rowstmp.close();
}
return rows;
}
@Test
public void testFilterList() throws Exception {
// Test getting a single row, single key using Row, Qualifier, and Value
// regular expression and substring filters
// Use must pass all
List<Filter> filters = new ArrayList<Filter>();
filters.add(new RowFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2")));
filters.add(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2")));
filters.add(new ValueFilter(CompareOp.EQUAL, new SubstringComparator("One")));
Filter f = new FilterList(Operator.MUST_PASS_ALL, filters);
Scan s = new Scan();
s.addFamily(FAMILIES[0]);
s.setFilter(f);
KeyValue[] kvs = {new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0])};
verifyScanFull(s, kvs);
// Test getting everything with a MUST_PASS_ONE filter including row, qf,
// val, regular expression and substring filters
filters.clear();
filters.add(new RowFilter(CompareOp.EQUAL, new RegexStringComparator(".+Two.+")));
filters.add(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2")));
filters.add(new ValueFilter(CompareOp.EQUAL, new SubstringComparator("One")));
f = new FilterList(Operator.MUST_PASS_ONE, filters);
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, numRows, colsPerRow);
}
/* (non-Javadoc)
* @see com.hazelcast.core.MapLoader#loadAllKeys()
*/
@Override
public Set<String> loadAllKeys() {
Set<String> keySet = null;
if (allowLoadAll) {
keySet = new HashSet<String>();
HTableInterface hti = null;
try {
hti = pool.getTable(tableName);
Scan s = new Scan();
s.addColumn(family, qualifier);
ResultScanner rs = hti.getScanner(s);
Result r = null;
while ((r = rs.next()) != null) {
String k = new String(r.getRow());
keySet.add(k);
}
} catch (IOException e) {
LOG.error("IOException while loading all keys", e);
} finally {
if (hti != null) {
pool.putTable(hti);
}
}
}
return keySet;
}
/*
* Add to each of the regions in .META. a value. Key is the startrow of the
* region (except its 'aaa' for first region). Actual value is the row name.
* @param expected
* @return
* @throws IOException
*/
private static int addToEachStartKey(final int expected) throws IOException {
HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
HTable meta = new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
int rows = 0;
Scan scan = new Scan();
scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
ResultScanner s = meta.getScanner(scan);
for (Result r = null; (r = s.next()) != null; ) {
byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
if (b == null || b.length <= 0) break;
HRegionInfo hri = Writables.getHRegionInfo(b);
// If start key, add 'aaa'.
byte[] row = getStartKey(hri);
Put p = new Put(row);
p.setWriteToWAL(false);
p.add(getTestFamily(), getTestQualifier(), row);
t.put(p);
rows++;
}
s.close();
Assert.assertEquals(expected, rows);
t.close();
meta.close();
return rows;
}
@Override
public void process(long now, HRegion region, List<Mutation> mutations, WALEdit walEdit)
throws IOException {
// Scan current counter
List<Cell> kvs = new ArrayList<Cell>();
Scan scan = new Scan(row, row);
scan.addColumn(FAM, COUNTER);
doScan(region, scan, kvs);
counter = kvs.size() == 0 ? 0 : Bytes.toInt(CellUtil.cloneValue(kvs.iterator().next()));
// Assert counter value
assertEquals(expectedCounter, counter);
// Increment counter and send it to both memstore and wal edit
counter += 1;
expectedCounter += 1;
Put p = new Put(row);
KeyValue kv = new KeyValue(row, FAM, COUNTER, now, Bytes.toBytes(counter));
p.add(kv);
mutations.add(p);
walEdit.add(kv);
// We can also inject some meta data to the walEdit
KeyValue metaKv =
new KeyValue(
row,
WALEdit.METAFAMILY,
Bytes.toBytes("I just increment counter"),
Bytes.toBytes(counter));
walEdit.add(metaKv);
}
public static void main(String[] args) throws Exception {
Configuration con = new Configuration();
String[] otherArgs = new GenericOptionsParser(con, args).getRemainingArgs();
HBaseConfiguration conf = new HBaseConfiguration();
Job job = new Job(conf, "AverageCalc");
job.setJarByClass(AverageCalculator.class);
Scan scan = new Scan();
scan.setCaching(500);
scan.setCacheBlocks(false);
scan.addFamily(Bytes.toBytes("Post"));
FilterList li = new FilterList(FilterList.Operator.MUST_PASS_ALL);
SingleColumnValueFilter filter =
new SingleColumnValueFilter(
Bytes.toBytes("Post"),
Bytes.toBytes("PostTypeId"),
CompareOp.EQUAL,
Bytes.toBytes("1"));
li.addFilter(filter);
scan.setFilter(li);
FileOutputFormat.setOutputPath(job, new Path(otherArgs[0]));
job.setOutputKeyClass(Text.class);
TableMapReduceUtil.initTableMapperJob(
"bigd24-hbase-sample", scan, Mapper1.class, Text.class, IntWritable.class, job);
job.setReducerClass(Reducer1.class);
job.setOutputValueClass(FloatWritable.class);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
/** Job configuration. */
public static Job configureJob(Configuration conf, String[] args) throws IOException {
String tableName = args[0];
String columnFamily = args[1];
String outputPath = args[2];
String rowKeyType = args[3];
conf.set("row.key.type", rowKeyType);
conf.set("table.name", tableName);
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(columnFamily));
scan.setBatch(ConstantsTruthy.TRUTHY_TABLE_SCAN_BATCH);
conf.set("mapred.map.tasks.speculative.execution", "false");
conf.set("mapred.reduce.tasks.speculative.execution", "false");
Job job =
Job.getInstance(
conf, "Count the column count and indexRecordSize for each row in " + tableName);
job.setJarByClass(TruthyIndexFeatureCounter.class);
TableMapReduceUtil.initTableMapperJob(
tableName, scan, TfcMapper.class, Text.class, Text.class, job, true);
job.setNumReduceTasks(0);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileOutputFormat.setOutputPath(job, new Path(outputPath));
TableMapReduceUtil.addDependencyJars(job);
return job;
}
@TimeDepend
@Test
public void testScan_ts_same() throws Exception {
recreateTable();
Date ts = parse("2000-01-01", "yyyy-MM-dd");
Put put = new Put(rowKey_ForTest);
put.add(ColumnFamilyNameBytes, QName1, ts.getTime(), Bytes.toBytes("a"));
table.put(put);
Set<String> resultRowKeys = new HashSet<String>();
Scan scan = new Scan(rowKey_ForTest, rowKey_ForTest);
scan.setTimeRange(ts.getTime(), ts.getTime());
ResultScanner resultScanner = table.getScanner(scan);
for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
resultRowKeys.add(Bytes.toString(result.getRow()));
}
close(resultScanner);
Assert.assertTrue(resultRowKeys.size() == 0);
recreateTable();
}
/*
* (non-Javadoc)
*
* @see
* com.impetus.kunderahbase.dao.user.UserDao#findUserByUserName(java.lang
* .String, boolean, java.util.List)
*/
@Override
public void findUserByUserName(String userName, boolean isBulk, List<UserHBaseDTO> users) {
for (UserHBaseDTO user : users) {
Filter filter =
new SingleColumnValueFilter(
Bytes.toBytes("user_name"),
Bytes.toBytes("user_name"),
CompareOp.EQUAL,
Bytes.toBytes(user.getUserNameCounter()));
Scan scan = new Scan();
scan.setFilter(filter);
scan.addColumn(Bytes.toBytes("user_name"), Bytes.toBytes("user_name"));
scan.addColumn(Bytes.toBytes("user_nameCnt"), Bytes.toBytes("user_nameCnt"));
scan.addColumn(Bytes.toBytes("password"), Bytes.toBytes("password"));
scan.addColumn(Bytes.toBytes("relation"), Bytes.toBytes("relation"));
ResultScanner scanner = null;
int counter = 0;
try {
scanner = hTablePool.getTable("User").getScanner(scan);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
assert scanner != null;
for (Result result : scanner) {
counter++;
assert result != null;
}
assert counter != 0;
}
}
private void runTestOnTable(Table table)
throws IOException, InterruptedException, ClassNotFoundException {
Job job = null;
try {
LOG.info("Before map/reduce startup");
job = new Job(table.getConfiguration(), "process column contents");
job.setNumReduceTasks(1);
Scan scan = new Scan();
scan.addFamily(INPUT_FAMILY);
TableMapReduceUtil.initTableMapperJob(
table.getName(),
scan,
MultithreadedTableMapper.class,
ImmutableBytesWritable.class,
Put.class,
job);
MultithreadedTableMapper.setMapperClass(job, ProcessContentsMapper.class);
MultithreadedTableMapper.setNumberOfThreads(job, NUMBER_OF_THREADS);
TableMapReduceUtil.initTableReducerJob(
table.getName().getNameAsString(), IdentityTableReducer.class, job);
FileOutputFormat.setOutputPath(job, new Path("test"));
LOG.info("Started " + table.getName());
assertTrue(job.waitForCompletion(true));
LOG.info("After map/reduce completion");
// verify map-reduce results
verify(table.getName());
} finally {
table.close();
if (job != null) {
FileUtil.fullyDelete(new File(job.getConfiguration().get("hadoop.tmp.dir")));
}
}
}
public static List<Delete> GetDeleteEventsBetween(
Table VTEvent_Table, String imo_str, long first_timestamp, long last_timestamp)
throws IOException {
// scan
// 'cdb_vessel:vessel_event',{FILTER=>"(PrefixFilter('0000003162')"}
Scan GetEventsBetween = new Scan();
GetEventsBetween.setStartRow(
Bytes.toBytes(imo_str + LpadNum(Long.MAX_VALUE - last_timestamp, 19) + "0000000000"))
.setStopRow(
Bytes.toBytes(
imo_str + LpadNum(Long.MAX_VALUE - first_timestamp + 1, 19) + "9999999999"))
.addColumn(details, exittime);
GetEventsBetween.setCaching(100);
Filter ExistTimeValuefilter =
new ValueFilter(
CompareFilter.CompareOp.LESS_OR_EQUAL,
new BinaryComparator(
Bytes.toBytes(new DateTime(last_timestamp).toString(rawformatter))));
GetEventsBetween.setFilter(ExistTimeValuefilter);
ResultScanner Result_ExistingEvents = VTEvent_Table.getScanner(GetEventsBetween);
List<Delete> deletes = new ArrayList<Delete>();
for (Result res : Result_ExistingEvents) {
deletes.add(new Delete(res.getRow()));
}
Result_ExistingEvents.close();
return deletes;
}
private static void projectAllColumnFamilies(PTable table, Scan scan) {
// Will project all known/declared column families
scan.getFamilyMap().clear();
for (PColumnFamily family : table.getColumnFamilies()) {
scan.addFamily(family.getName().getBytes());
}
}
public String[] getTags(String objectType, String objectId) throws IOException {
List<String> ret = new ArrayList<String>();
String rowKey = objectType + "_" + objectId;
Scan s = new Scan(rowKey.getBytes(), rowKey.getBytes());
s.setMaxVersions(1);
ResultScanner scanner = htable.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
String localObjectType =
new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
NavigableMap<byte[], byte[]> map = rr.getFamilyMap("tags".getBytes());
Iterator<Entry<byte[], byte[]>> it = map.entrySet().iterator();
while (it.hasNext()) {
Entry<byte[], byte[]> entry = it.next();
String key = new String(entry.getKey());
if (!key.startsWith("OBJECT")) {
int val = Bytes.toInt(entry.getValue());
if (val > 0) ret.add(key);
}
}
}
} finally {
scanner.close();
}
return ret.toArray(new String[] {});
}
public static void main(String args[]) {
if (args.length == 0) {
System.out.println("JavaHBaseDistributedScan {master} {tableName}");
}
String master = args[0];
String tableName = args[1];
JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseDistributedScan");
jsc.addJar("SparkHBase.jar");
Configuration conf = HBaseConfiguration.create();
conf.addResource(new Path("/etc/hbase/conf/core-site.xml"));
conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml"));
JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);
Scan scan = new Scan();
scan.setCaching(100);
JavaRDD<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> javaRdd =
hbaseContext.hbaseRDD(tableName, scan);
List<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> results = javaRdd.collect();
results.size();
}
public Blog(String blogid) throws IOException {
Configuration conf = HBaseConfiguration.create();
table = new HTable(conf, "blogs");
// 1. Get the row whose row key is blogid from above
Get g = new Get(Bytes.toBytes(blogid));
Result r = table.get(g);
// 2. Extract the rowkey, blog text (column "body") and blog title
// (column "meta:title")
key = r.getRow();
keyStr = Bytes.toString(key);
blogText = Bytes.toString(r.getValue(Bytes.toBytes("body"), Bytes.toBytes("")));
blogTitle = Bytes.toString(r.getValue(Bytes.toBytes("meta"), Bytes.toBytes("title")));
Long reverseTimestamp = Long.parseLong(keyStr.substring(4));
Long epoch = Math.abs(reverseTimestamp - Long.MAX_VALUE);
dateOfPost = new Date(epoch);
// Get an iterator for the comments
Scan s = new Scan();
s.addFamily(Bytes.toBytes("comment"));
// Use a PrefixFilter
PrefixFilter filter = new PrefixFilter(key);
s.setFilter(filter);
scanner = table.getScanner(s);
resultIterator = scanner.iterator();
}
private Scan createScan(final String columnFamily, final String[] headers) {
Scan scan = new Scan();
for (String header : headers) {
scan.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(header));
}
return scan;
}