private Scan createScan(String applicationName, Range range, boolean scanBackward) {
Scan scan = new Scan();
scan.setCaching(this.scanCacheSize);
byte[] bApplicationName = Bytes.toBytes(applicationName);
byte[] traceIndexStartKey = SpanUtils.getTraceIndexRowKey(bApplicationName, range.getFrom());
byte[] traceIndexEndKey = SpanUtils.getTraceIndexRowKey(bApplicationName, range.getTo());
if (scanBackward) {
// start key is replaced by end key because key has been reversed
scan.setStartRow(traceIndexEndKey);
scan.setStopRow(traceIndexStartKey);
} else {
scan.setReversed(true);
scan.setStartRow(traceIndexStartKey);
scan.setStopRow(traceIndexEndKey);
}
scan.addFamily(HBaseTables.APPLICATION_TRACE_INDEX_CF_TRACE);
scan.setId("ApplicationTraceIndexScan");
// toString() method of Scan converts a message to json format so it is slow for the first time.
logger.trace("create scan:{}", scan);
return scan;
}
public static ResultScanner getAllFamilyList(
String startRowRange, String stopRowRange, byte[][] cfs, String ctableName) {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(startRowRange));
for (byte[] family : cfs) {
scan.addFamily(family);
}
if (stopRowRange != null) {
scan.setStopRow(Bytes.toBytes(stopRowRange));
}
ResultScanner resultScanner = null;
try {
resultScanner = tblMngr.getTable(ctableName).getScanner(scan);
} catch (Exception e) {
}
return resultScanner;
}
/**
* Performs a full scan of a catalog table.
*
* @param catalogTracker
* @param visitor Visitor invoked against each row.
* @param startrow Where to start the scan. Pass null if want to begin scan at first row.
* @param scanRoot True if we are to scan <code>-ROOT-</code> rather than <code>.META.</code>, the
* default (pass false to scan .META.)
* @throws IOException
*/
static void fullScan(
CatalogTracker catalogTracker,
final Visitor visitor,
final byte[] startrow,
final boolean scanRoot)
throws IOException {
Scan scan = new Scan();
if (startrow != null) scan.setStartRow(startrow);
if (startrow == null && !scanRoot) {
int caching =
catalogTracker
.getConnection()
.getConfiguration()
.getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
scan.setCaching(caching);
}
scan.addFamily(HConstants.CATALOG_FAMILY);
HTable metaTable = scanRoot ? getRootHTable(catalogTracker) : getMetaHTable(catalogTracker);
ResultScanner scanner = metaTable.getScanner(scan);
try {
Result data;
while ((data = scanner.next()) != null) {
if (data.isEmpty()) continue;
// Break if visit returns false.
if (!visitor.visit(data)) break;
}
} finally {
scanner.close();
metaTable.close();
}
return;
}
/**
* 遍历多行
*
* @param tableName 表名
* @param start_rowkey 开始行键
* @param stop_rowkey 结束行键
* @return 行列表
*/
public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
ResultScanner rowstmp = null;
ArrayList<HbaseRow> rows = null;
try {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(start_rowkey));
scan.setStopRow(Bytes.toBytes(stop_rowkey));
HTable table = new HTable(conf, Bytes.toBytes(tableName));
rowstmp = table.getScanner(scan);
rows = new ArrayList<>();
for (Result rowtmp : rowstmp) {
HbaseRow row = new HbaseRow();
row.rowkey = Bytes.toString(rowtmp.getRow());
for (Cell cell : rowtmp.listCells()) {
HbaseColumn col = new HbaseColumn(cell);
row.cols.add(col);
}
rows.add(row);
}
} catch (Exception e) {
logger.error("scanRows failed", e);
} finally {
rowstmp.close();
}
return rows;
}
/**
* Gets an hbase scanner.
*
* @return An hbase scanner.
* @throws IOException Error accessing hbase.
*/
@Override
protected Scan getScanner() throws IOException {
Scan scan = super.getScanner();
if (this.query != null) {
if (this.query.getStartKey() != null) {
scan.setStartRow(this.query.getStartKey());
}
if (this.query.getEndKey() != null) {
scan.setStopRow(padWithMaxUnicode(this.query.getEndKey()));
}
if (this.query.getStartDate() != null && this.query.getEndDate() != null) {
scan.setTimeRange(this.query.getStartDate().getTime(), this.query.getEndDate().getTime());
}
if (this.query.getWord() != null || this.query.getOperator().isUnary()) {
WritableByteArrayComparable comparator;
switch (this.query.getOperator()) {
case Contains:
comparator = new SubstringComparator(this.query.getWord());
break;
case StartsWith:
comparator = new RegexStringComparator(String.format("^%s.*", this.query.getWord()));
break;
case EndsWith:
comparator = new RegexStringComparator(String.format(".*%s$", this.query.getWord()));
break;
case Less:
case LessOrEqual:
case Equal:
case NotEqual:
case GreaterOrEqual:
case Greater:
comparator = new BinaryComparator(this.query.getWordAsByteArray());
break;
case IsNull:
case IsNotNull:
comparator = new BinaryComparator(EMPTY_BYTES_ARRAY);
break;
default:
throw new IllegalArgumentException(
String.format(
"The specified operator type '%s' is not supported.",
this.query.getOperator()));
}
scan.setFilter(
new SingleColumnValueFilter(
Bytes.toBytesBinary(this.query.getFamily()),
Bytes.toBytesBinary(this.query.getColumn()),
this.query.getOperator().toFilter(),
comparator));
}
}
return scan;
}
/**
* Given a scan and a key range, return a new Scan whose range is truncated to only include keys
* in that range. Returns null if the Scan does not overlap the given range.
*/
private static final Scan truncateScan(Scan scan, byte[] rangeStart, byte[] rangeEnd) {
byte[] scanStart = scan.getStartRow();
byte[] scanEnd = scan.getStopRow();
if (scanEnd.length > 0 && bytesCompare(scanEnd, rangeStart) <= 0) {
// The entire scan range is before the entire cube key range
return null;
} else if (scanStart.length > 0 && bytesCompare(scanStart, rangeEnd) >= 0) {
// The entire scan range is after the entire cube key range
return null;
} else {
// Now we now that the scan range at least partially overlaps the cube key range.
Scan truncated;
try {
truncated = new Scan(scan); // make a copy, don't modify input scan
} catch (IOException e) {
throw new RuntimeException(); // This is not plausible
}
if (scanStart.length == 0 || bytesCompare(rangeStart, scanStart) > 0) {
// The scan includes extra keys at the beginning that are not part of the cube. Move
// the scan start point so that it only touches keys belonging to the cube.
truncated.setStartRow(rangeStart);
}
if (scanEnd.length == 0 || bytesCompare(rangeEnd, scanEnd) < 0) {
// The scan includes extra keys at the end that are not part of the cube. Move the
// scan end point so it only touches keys belonging to the cube.
truncated.setStopRow(rangeEnd);
}
return truncated;
}
}
/**
* Get a collection of Scans, one per region, that cover the range of the table having the given
* key prefix. Thes will be used as the map task input splits.
*/
public static List<Scan> scansThisCubeOnly(byte[] keyPrefix, byte[][] splitKeys)
throws IOException {
Scan copyScan = new Scan();
copyScan.setCaching(5000);
copyScan.setCacheBlocks(false);
// Hack: generate a key that probably comes after all this cube's keys but doesn't include any
// keys not belonging to this cube.
byte[] keyAfterCube = ArrayUtils.addAll(keyPrefix, fiftyBytesFF);
List<Scan> scans = new ArrayList<Scan>();
Scan scanUnderConstruction = new Scan(copyScan);
for (byte[] splitKey : splitKeys) {
scanUnderConstruction.setStopRow(splitKey);
// Coerce scan to only touch keys belonging to this cube
Scan truncated = truncateScan(scanUnderConstruction, keyPrefix, keyAfterCube);
if (truncated != null) {
scans.add(truncated);
}
scanUnderConstruction = new Scan(copyScan);
scanUnderConstruction.setStartRow(splitKey);
}
// There's another region from last split key to the end of the table.
Scan truncated = truncateScan(scanUnderConstruction, keyPrefix, keyAfterCube);
if (truncated != null) {
scans.add(truncated);
}
return scans;
}
public static List<Delete> GetDeleteEventsBetween(
Table VTEvent_Table, String imo_str, long first_timestamp, long last_timestamp)
throws IOException {
// scan
// 'cdb_vessel:vessel_event',{FILTER=>"(PrefixFilter('0000003162')"}
Scan GetEventsBetween = new Scan();
GetEventsBetween.setStartRow(
Bytes.toBytes(imo_str + LpadNum(Long.MAX_VALUE - last_timestamp, 19) + "0000000000"))
.setStopRow(
Bytes.toBytes(
imo_str + LpadNum(Long.MAX_VALUE - first_timestamp + 1, 19) + "9999999999"))
.addColumn(details, exittime);
GetEventsBetween.setCaching(100);
Filter ExistTimeValuefilter =
new ValueFilter(
CompareFilter.CompareOp.LESS_OR_EQUAL,
new BinaryComparator(
Bytes.toBytes(new DateTime(last_timestamp).toString(rawformatter))));
GetEventsBetween.setFilter(ExistTimeValuefilter);
ResultScanner Result_ExistingEvents = VTEvent_Table.getScanner(GetEventsBetween);
List<Delete> deletes = new ArrayList<Delete>();
for (Result res : Result_ExistingEvents) {
deletes.add(new Delete(res.getRow()));
}
Result_ExistingEvents.close();
return deletes;
}
public static ResultScanner getList(
String startRowRange,
String stopRowRange,
byte[] cf1,
byte[] cf2,
long limit,
FilterList filterList,
String ctableName) {
Scan scan = new Scan();
scan.addFamily(cf1);
scan.setStartRow(Bytes.toBytes(startRowRange));
if (stopRowRange != null) {
scan.setStopRow(Bytes.toBytes(stopRowRange));
}
if (limit != 0) {
filterList.addFilter(new PageFilter(limit));
} else {
filterList.addFilter(new PageFilter(100));
}
scan.setFilter(filterList);
ResultScanner resultScanner = null;
try {
resultScanner = tblMngr.getTable(ctableName).getScanner(scan);
} catch (Exception e) {
}
return resultScanner;
}
public Scan generateScan(
String[] rowRange, FilterList filterList, String[] family, String[] columns, int maxVersion)
throws Exception {
if (table == null) throw new Exception("No table handler");
if (cacheSize < 0) throw new Exception("should set cache size before scanning");
Scan scan = null;
try {
scan = new Scan();
scan.setCaching(this.cacheSize);
scan.setCacheBlocks(this.blockCached);
scan.setFilter(filterList);
if (maxVersion > 0) scan.setMaxVersions(maxVersion);
if (rowRange != null) {
scan.setStartRow(rowRange[0].getBytes());
if (rowRange.length == 2) scan.setStopRow(rowRange[1].getBytes());
}
if (columns != null) {
for (int i = 0; i < columns.length; i++) {
scan.addColumn(family[0].getBytes(), columns[i].getBytes());
// System.out.println(family[i]+";"+columns[i]);
}
} else {
scan.addFamily(family[0].getBytes());
}
} catch (Exception e) {
e.printStackTrace();
}
return scan;
}
// 获取扫描器对象
private static Scan getScan(String startRow, String stopRow) {
Scan scan = new Scan();
if (startRow != null && stopRow != null) {
scan.setStartRow(getBytes(startRow));
scan.setStopRow(getBytes(stopRow));
}
return scan;
}
// Get all events with exit at last location
public static Map<Integer, VesselEvent> getAllEventsStartBeforeEndAfterBeforeLocation(
Table VTEvent_Table, String IMO_str, VesselLocation location) throws IOException {
Scan getAllEventsWithExistAtLastLocation = new Scan();
getAllEventsWithExistAtLastLocation
.setStartRow(
Bytes.toBytes(
IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19) + "0000000000"))
.setStopRow(Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE, 19) + "9999999999"))
.addColumn(details, exittime);
getAllEventsWithExistAtLastLocation.setCaching(100);
Filter ExistTimeValuefilter =
new ValueFilter(
CompareFilter.CompareOp.GREATER_OR_EQUAL,
new BinaryComparator(
Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter))));
getAllEventsWithExistAtLastLocation.setFilter(ExistTimeValuefilter);
ResultScanner Result_event = VTEvent_Table.getScanner(getAllEventsWithExistAtLastLocation);
Map<Integer, VesselEvent> events = new HashMap<Integer, VesselEvent>();
for (Result res : Result_event) {
Get get = new Get(res.getRow());
get.addColumn(details, entrytime);
get.addColumn(details, entrycoordinates);
Result result = VTEvent_Table.get(get);
String rowkey = Bytes.toString(result.getRow());
String polygonid = rowkey.substring(26);
VesselEvent VE = new VesselEvent();
VE.exittime = location.recordtime;
VE.exitcoordinates = location.coordinates;
VE.destination = location.destination;
VE.polygonid = Integer.parseInt(polygonid);
for (Cell cell : result.rawCells()) {
String Qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
String Value = Bytes.toString(CellUtil.cloneValue(cell));
if (Qualifier.equals("entertime")) {
VE.entrytime = DateTime.parse(Value, rawformatter).getMillis();
} else if (Qualifier.equals("entercoordinates")) {
VE.entrycoordinates = Value;
}
}
events.put(VE.polygonid, VE);
}
Result_event.close();
return events;
}
public HBaseSecondaryIndexCursor(
HBaseSecondaryIndex index, TableFilter filter, byte[] startKey, byte[] endKey) {
defaultColumnFamilyName =
Bytes.toBytes(((HBaseTable) filter.getTable()).getDefaultColumnFamilyName());
secondaryIndex = index;
session = (HBaseSession) filter.getSession();
Prepared p = filter.getPrepared();
if (p instanceof WithWhereClause) {
regionName = Bytes.toBytes(((WithWhereClause) p).getWhereClauseSupport().getRegionName());
}
if (regionName == null) throw new RuntimeException("regionName is null");
fetchSize = filter.getPrepared().getFetchSize();
// 非查询的操作一般不设置fetchSize，此时fetchSize为0，所以要设置一个默认值
// if (fetchSize < 1 && !filter.getPrepared().isQuery())
if (fetchSize < 1) fetchSize = SysProperties.SERVER_RESULT_SET_FETCH_SIZE;
if (filter.getSelect() != null) columns = filter.getSelect().getColumns(filter);
else columns = Arrays.asList(filter.getTable().getColumns());
if (startKey == null) startKey = HConstants.EMPTY_BYTE_ARRAY;
if (endKey == null) endKey = HConstants.EMPTY_BYTE_ARRAY;
Scan scan = new Scan();
scan.setMaxVersions(1);
try {
HRegionInfo info = session.getRegionServer().getRegionInfo(regionName);
if (Bytes.compareTo(startKey, info.getStartKey()) >= 0) scan.setStartRow(startKey);
else scan.setStartRow(info.getStartKey());
if (Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)) scan.setStopRow(info.getEndKey());
else if (Bytes.compareTo(endKey, info.getEndKey()) < 0) scan.setStopRow(endKey);
else scan.setStopRow(info.getEndKey());
scan.addColumn(HBaseSecondaryIndex.PSEUDO_FAMILY, HBaseSecondaryIndex.PSEUDO_COLUMN);
scannerId = session.getRegionServer().openScanner(regionName, scan);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Test
public void testStdWithValidRange2WithNoCQ() throws Throwable {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addFamily(TEST_FAMILY);
scan.setStartRow(ROWS[6]);
scan.setStopRow(ROWS[7]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
double std = aClient.std(TEST_TABLE, ci, scan);
assertEquals(0, std, 0);
}
/** @throws Throwable */
@Test
public void testStdWithValidRange2() throws Throwable {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[5]);
scan.setStopRow(ROWS[15]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
double std = aClient.std(TEST_TABLE, ci, scan);
assertEquals(2.87, std, 0.05d);
}
@Test
public void testMinWithValidRangeWithNoCQ() throws Throwable {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addFamily(TEST_FAMILY);
scan.setStartRow(HConstants.EMPTY_START_ROW);
scan.setStopRow(HConstants.EMPTY_END_ROW);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
long min = aClient.min(TEST_TABLE, ci, scan);
assertEquals(0, min);
}
@Override
public void tweet(User user, String tweetText) throws IOException {
final long epoch = System.currentTimeMillis();
final long tranposeEpoch = Long.MAX_VALUE - epoch;
final byte[] epochBytes = Bytes.toBytes(epoch);
final byte[] tweetBytes = Bytes.toBytes(tweetText);
byte[] nameBytes = Bytes.toBytes(user.getName());
/** put tweet into tweets */
Put tweetRowPut = new Put(generateTweetId(user));
tweetRowPut.add(_DEFAULT, _NAME, nameBytes);
tweetRowPut.add(_DEFAULT, _MAIL, Bytes.toBytes(user.getEmail()));
tweetRowPut.add(_DEFAULT, _TWEET, tweetBytes);
tweetRowPut.add(_DEFAULT, _TIME, epochBytes);
tweetsTable.put(tweetRowPut);
/** put tweets for followers */
Scan followerScan = new Scan();
followerScan.setStartRow(Bytes.toBytes(user.getUserId() + "-"));
followerScan.setStopRow(Bytes.toBytes((user.getUserId() + 1) + "-"));
ResultScanner followerRS = followersTable.getScanner(followerScan);
/** put users on tweet to her own tweetline */
Put put =
new Put(Bytes.toBytes(user.getUserId() + "-" + tranposeEpoch + "-" + user.getUserId()));
put.add(_DEFAULT, _NAME, nameBytes);
put.add(_DEFAULT, _TWEET, tweetBytes);
put.add(_DEFAULT, _TIME, epochBytes);
List<Row> puts = new ArrayList<Row>();
puts.add(put);
for (Result result : followerRS) {
Long followerid = Bytes.toLong(result.getColumnLatest(_DEFAULT, _USERID).getValue());
put = new Put(Bytes.toBytes(followerid + "-" + tranposeEpoch + "-" + user.getUserId()));
put.add(_DEFAULT, _NAME, nameBytes);
put.add(_DEFAULT, _TWEET, tweetBytes);
put.add(_DEFAULT, _TIME, epochBytes);
puts.add(put);
}
followerRS.close();
try {
tweetlineTable.batch(puts);
} catch (InterruptedException e) {
e.printStackTrace(); // @TODO log and handle properly.
}
}
/**
* This will test rowcount with a valid range, i.e., a subset of rows. It will be the most common
* use case.
*
* @throws Throwable
*/
@Test
public void testRowCountWithValidRange() throws Throwable {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[2]);
scan.setStopRow(ROWS[14]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
long rowCount = aClient.rowCount(TEST_TABLE, ci, scan);
assertEquals(12, rowCount);
}
/**
* Configure the scan according to the row options.
*
* @param scan The HBase scan descriptor to configure.
* @throws IOException If there is an error.
*/
private void configureScanWithRowOptions(Scan scan) throws IOException {
if (null != mRowOptions.getStartRow()) {
scan.setStartRow(mRowOptions.getStartRow().getHBaseRowKey());
}
if (null != mRowOptions.getLimitRow()) {
scan.setStopRow(mRowOptions.getLimitRow().getHBaseRowKey());
}
if (null != mRowOptions.getRowFilter()) {
KijiTableLayout tableLayout = mInputTable.getLayout();
KijiSchemaTable schemaTable = mInputTable.getKiji().getSchemaTable();
new KijiRowFilterApplicator(mRowOptions.getRowFilter(), tableLayout, schemaTable)
.applyTo(scan);
}
}
public <V> List<V> list(
String tableName, byte[] startKey, byte[] stopKey, Filter filter, HbaseMapper<V> mapper)
throws Exception {
HTableInterface htable = dataSource.getConnection(tableName);
Scan scan = new Scan();
scan.setStartRow(startKey);
scan.setStopRow(stopKey);
scan.setFilter(filter);
log.info("scan is -[" + scan + "]");
ResultScanner resultScanner = htable.getScanner(scan);
List<V> result = trasfer(resultScanner, mapper);
htable.close();
log.info("list " + result.size() + " objects from " + tableName);
return result;
}
/** This should return a 0 */
@Test
public void testRowCountWithNullCF() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.setStartRow(ROWS[5]);
scan.setStopRow(ROWS[15]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
long rowCount = -1;
try {
rowCount = aClient.rowCount(TEST_TABLE, ci, scan);
} catch (Throwable e) {
rowCount = 0;
}
assertEquals(0, rowCount);
}
@Test
public void testAvgWithInvalidRange() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[5]);
scan.setStopRow(ROWS[1]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
Double avg = null;
try {
avg = aClient.avg(TEST_TABLE, ci, scan);
} catch (Throwable e) {
}
assertEquals(null, avg); // control should go to the catch block
}
@Test
public void testStdWithValidRangeWithNullCF() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.setStartRow(ROWS[6]);
scan.setStopRow(ROWS[17]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
Double std = null;
try {
std = aClient.std(TEST_TABLE, ci, scan);
} catch (Throwable e) {
}
assertEquals(null, std); // CP will throw an IOException about the
// null column family, and max will be set to 0
}
@Test
public void testStdWithInvalidRange() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addFamily(TEST_FAMILY);
scan.setStartRow(ROWS[6]);
scan.setStopRow(ROWS[1]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
Double std = null;
try {
std = aClient.std(TEST_TABLE, ci, scan);
} catch (Throwable e) {
}
assertEquals(null, std); // control should go to the catch block
}
public static void updateExistingEventsToEndAtLastLocation(
Table VTEvent_Table, long imo, VesselLocation lastlocation) throws IOException {
// update existing events that started BEFORE the first new location and end after the first
// to end as the last location
// Find existing events that started BEFORE the first new location and end after the first
Scan getEventStartedBeforeAndEndAfter = new Scan();
;
getEventStartedBeforeAndEndAfter
.setStartRow(
Bytes.toBytes(
LpadNum(imo, 7)
+ LpadNum(Long.MAX_VALUE - lastlocation.recordtime, 19)
+ "0000000000"))
.setStopRow(Bytes.toBytes(LpadNum(imo, 7) + LpadNum(Long.MAX_VALUE, 19) + "9999999999"))
.addColumn(details, exittime);
getEventStartedBeforeAndEndAfter.setCaching(100);
Filter ExistTimeValuefilter =
new ValueFilter(
CompareFilter.CompareOp.GREATER,
new BinaryComparator(
Bytes.toBytes(new DateTime(lastlocation.recordtime).toString(rawformatter))));
getEventStartedBeforeAndEndAfter.setFilter(ExistTimeValuefilter);
ResultScanner Result_eventcross = VTEvent_Table.getScanner(getEventStartedBeforeAndEndAfter);
List<Put> puts = new ArrayList<Put>();
for (Result res : Result_eventcross) {
// vessel event table
// rowkey: imo(7)+timestamp(19 desc)+polygonid(8)
// qualifier:entrytime,entrycoordinates,exittime,exitcoordinates,destination
byte[] rowkey = res.getRow();
Put updateevent = new Put(rowkey);
updateevent.addColumn(
details,
exittime,
Bytes.toBytes(new DateTime(lastlocation.recordtime).toString(rawformatter)));
updateevent.addColumn(details, coordinates, Bytes.toBytes(lastlocation.coordinates));
updateevent.addColumn(details, destination, Bytes.toBytes(lastlocation.destination));
puts.add(updateevent);
}
Result_eventcross.close();
VTEvent_Table.put(puts);
}
@Test
public void testMaxWithInvalidRange2() throws Throwable {
long max = Long.MIN_VALUE;
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[4]);
scan.setStopRow(ROWS[4]);
try {
AggregationClient aClient = new AggregationClient(conf);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
max = aClient.max(TEST_TABLE, ci, scan);
} catch (Exception e) {
max = 0;
}
assertEquals(0, max); // control should go to the catch block
}
@Override
public int run(String[] strings) throws Exception {
Configuration conf = new Configuration();
// String inputFileName = "/cluster/gmm.seq";
String outputFileName = "/cluster/matrix_intermediate_" + level + ".seq";
int result;
System.out.println("level:" + level);
conf.set("level", level + "");
String table = "ClusterDatabase";
// String seqFileName = "/cluster/gmm.seq";
Scan scan = new Scan();
scan.setStartRow((level + "|").getBytes());
scan.setStopRow(
Bytes.add((level + "|").getBytes(), Bytes.toBytes("ffffffffffffffffffffffffffffffff")));
scan.addColumn("Cluster".getBytes(), "GMM".getBytes());
// try (FileSystem fileSystem = FileSystem.get(conf)) {
FileSystem fileSystem = FileSystem.get(conf);
Path outputpath = new Path(outputFileName);
if (fileSystem.exists(outputpath)) {
fileSystem.delete(outputpath, true);
}
Job job = new Job(conf, "Matrix Creation I From HBase");
job.setJarByClass(MatrixCreationI.class);
TableMapReduceUtil.initTableMapperJob(
table, scan, MatrixMapper.class, IntWritable.class, Text.class, job);
job.setReducerClass(MatrixReducer.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
// job.setInputFormatClass(TableInputFormat.class);
// job.setOutputFormatClass(TextOutputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setNumReduceTasks(1);
// FileInputFormat.addInputPath(job, new Path(inputFileName + "/part*"));
FileOutputFormat.setOutputPath(job, outputpath);
result = job.waitForCompletion(true) ? 0 : 1;
// }
return result;
}
@Override
public Result getCurrentRowState(Mutation m, Collection<? extends ColumnReference> columns)
throws IOException {
byte[] row = m.getRow();
// need to use a scan here so we can get raw state, which Get doesn't provide.
Scan s = IndexManagementUtil.newLocalStateScan(Collections.singletonList(columns));
s.setStartRow(row);
s.setStopRow(row);
HRegion region = this.env.getRegion();
RegionScanner scanner = region.getScanner(s);
List<KeyValue> kvs = new ArrayList<KeyValue>(1);
boolean more = scanner.next(kvs);
assert !more : "Got more than one result when scanning" + " a single row in the primary table!";
Result r = new Result(kvs);
scanner.close();
return r;
}
/**
* Fully scan a given region, on a given server starting with given row.
*
* @param hRegionInterface region server
* @param visitor visitor
* @param regionName name of region
* @param startrow start row
* @throws IOException
* @deprecated Does not retry; use fullScan xxx instead. x
*/
public static void fullScan(
HRegionInterface hRegionInterface, Visitor visitor, final byte[] regionName, byte[] startrow)
throws IOException {
if (hRegionInterface == null) return;
Scan scan = new Scan();
if (startrow != null) scan.setStartRow(startrow);
scan.addFamily(HConstants.CATALOG_FAMILY);
long scannerid = hRegionInterface.openScanner(regionName, scan);
try {
Result data;
while ((data = hRegionInterface.next(scannerid)) != null) {
if (!data.isEmpty()) visitor.visit(data);
}
} finally {
hRegionInterface.close(scannerid);
}
return;
}
/**
* This will test the row count with startrow > endrow. The result should be -1.
*
* @throws Throwable
*/
@Test
public void testRowCountWithInvalidRange1() {
AggregationClient aClient = new AggregationClient(conf);
Scan scan = new Scan();
scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
scan.setStartRow(ROWS[5]);
scan.setStopRow(ROWS[2]);
final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
new LongColumnInterpreter();
long rowCount = -1;
try {
rowCount = aClient.rowCount(TEST_TABLE, ci, scan);
} catch (Throwable e) {
myLog.error("Exception thrown in the invalidRange method" + e.getStackTrace());
}
assertEquals(-1, rowCount);
}