/**
* Given a scan and a key range, return a new Scan whose range is truncated to only include keys
* in that range. Returns null if the Scan does not overlap the given range.
*/
private static final Scan truncateScan(Scan scan, byte[] rangeStart, byte[] rangeEnd) {
byte[] scanStart = scan.getStartRow();
byte[] scanEnd = scan.getStopRow();
if (scanEnd.length > 0 && bytesCompare(scanEnd, rangeStart) <= 0) {
// The entire scan range is before the entire cube key range
return null;
} else if (scanStart.length > 0 && bytesCompare(scanStart, rangeEnd) >= 0) {
// The entire scan range is after the entire cube key range
return null;
} else {
// Now we now that the scan range at least partially overlaps the cube key range.
Scan truncated;
try {
truncated = new Scan(scan); // make a copy, don't modify input scan
} catch (IOException e) {
throw new RuntimeException(); // This is not plausible
}
if (scanStart.length == 0 || bytesCompare(rangeStart, scanStart) > 0) {
// The scan includes extra keys at the beginning that are not part of the cube. Move
// the scan start point so that it only touches keys belonging to the cube.
truncated.setStartRow(rangeStart);
}
if (scanEnd.length == 0 || bytesCompare(rangeEnd, scanEnd) < 0) {
// The scan includes extra keys at the end that are not part of the cube. Move the
// scan end point so it only touches keys belonging to the cube.
truncated.setStopRow(rangeEnd);
}
return truncated;
}
}
/**
* @param scan the scan specification
* @throws Exception
*/
public static ScannerModel fromScan(Scan scan) throws Exception {
ScannerModel model = new ScannerModel();
model.setStartRow(scan.getStartRow());
model.setEndRow(scan.getStopRow());
Map<byte[], NavigableSet<byte[]>> families = scan.getFamilyMap();
if (families != null) {
for (Map.Entry<byte[], NavigableSet<byte[]>> entry : families.entrySet()) {
if (entry.getValue() != null) {
for (byte[] qualifier : entry.getValue()) {
model.addColumn(Bytes.add(entry.getKey(), COLUMN_DIVIDER, qualifier));
}
} else {
model.addColumn(entry.getKey());
}
}
}
model.setStartTime(scan.getTimeRange().getMin());
model.setEndTime(scan.getTimeRange().getMax());
int caching = scan.getCaching();
if (caching > 0) {
model.setBatch(caching);
}
int maxVersions = scan.getMaxVersions();
if (maxVersions > 0) {
model.setMaxVersions(maxVersions);
}
Filter filter = scan.getFilter();
if (filter != null) {
model.setFilter(stringifyFilter(filter));
}
return model;
}
protected boolean checkScanStopRow(final byte[] endKey) {
if (this.scan.getStopRow().length > 0) {
byte[] stopRow = scan.getStopRow();
int cmp = Bytes.compareTo(stopRow, 0, stopRow.length, endKey, 0, endKey.length);
if (cmp <= 0) {
return true;
}
}
return false;
}
@Override
public Builder adapt(Scan scan, ReadHooks readHooks) {
throwIfUnsupportedScan(scan);
// For gets, startRow == stopRow. There's no need to create a new ByteString for stopRow
RowFilter filter = buildFilter(scan, readHooks);
return ReadRowsRequest.newBuilder()
.setFilter(filter)
.setRowRange(
RowRange.newBuilder()
.setStartKey(ByteString.copyFrom(scan.getStartRow()))
.setEndKey(ByteString.copyFrom(scan.getStopRow())));
}
// Start/stop row must be swapped if scan is being done in reverse
public static void setupReverseScan(Scan scan) {
if (isReversed(scan)) {
byte[] startRow = scan.getStartRow();
byte[] stopRow = scan.getStopRow();
byte[] newStartRow = startRow;
byte[] newStopRow = stopRow;
if (startRow.length != 0) {
/*
* Must get previous key because this is going from an inclusive start key to an exclusive stop key, and
* we need the start key to be included. We get the previous key by decrementing the last byte by one.
* However, with variable length data types, we need to fill with the max byte value, otherwise, if the
* start key is 'ab', we lower it to 'aa' which would cause 'aab' to be included (which isn't correct).
* So we fill with a 0xFF byte to prevent this. A single 0xFF would be enough for our primitive types (as
* that byte wouldn't occur), but for an arbitrary VARBINARY key we can't know how many bytes to tack
* on. It's lame of HBase to force us to do this.
*/
newStartRow =
Arrays.copyOf(startRow, startRow.length + MAX_FILL_LENGTH_FOR_PREVIOUS_KEY.length);
if (ByteUtil.previousKey(newStartRow, startRow.length)) {
System.arraycopy(
MAX_FILL_LENGTH_FOR_PREVIOUS_KEY,
0,
newStartRow,
startRow.length,
MAX_FILL_LENGTH_FOR_PREVIOUS_KEY.length);
} else {
newStartRow = HConstants.EMPTY_START_ROW;
}
}
if (stopRow.length != 0) {
// Must add null byte because we need the start to be exclusive while it was inclusive
newStopRow = ByteUtil.concat(stopRow, QueryConstants.SEPARATOR_BYTE_ARRAY);
}
scan.setStartRow(newStopRow);
scan.setStopRow(newStartRow);
scan.setReversed(true);
}
}
public static boolean intersectScanRange(
Scan scan, byte[] startKey, byte[] stopKey, boolean useSkipScan) {
boolean mayHaveRows = false;
int offset = 0;
if (ScanUtil.isLocalIndex(scan)) {
offset = startKey.length != 0 ? startKey.length : stopKey.length;
}
byte[] existingStartKey = scan.getStartRow();
byte[] existingStopKey = scan.getStopRow();
if (existingStartKey.length > 0) {
if (startKey.length == 0 || Bytes.compareTo(existingStartKey, startKey) > 0) {
startKey = existingStartKey;
}
} else {
mayHaveRows = true;
}
if (existingStopKey.length > 0) {
if (stopKey.length == 0 || Bytes.compareTo(existingStopKey, stopKey) < 0) {
stopKey = existingStopKey;
}
} else {
mayHaveRows = true;
}
scan.setStartRow(startKey);
scan.setStopRow(stopKey);
if (offset > 0 && useSkipScan) {
byte[] temp = null;
if (startKey.length != 0) {
temp = new byte[startKey.length - offset];
System.arraycopy(startKey, offset, temp, 0, startKey.length - offset);
startKey = temp;
}
if (stopKey.length != 0) {
temp = new byte[stopKey.length - offset];
System.arraycopy(stopKey, offset, temp, 0, stopKey.length - offset);
stopKey = temp;
}
}
mayHaveRows = mayHaveRows || Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) < 0;
// If the scan is using skip scan filter, intersect and replace the filter.
if (mayHaveRows && useSkipScan) {
Filter filter = scan.getFilter();
if (filter instanceof SkipScanFilter) {
SkipScanFilter oldFilter = (SkipScanFilter) filter;
SkipScanFilter newFilter = oldFilter.intersect(startKey, stopKey);
if (newFilter == null) {
return false;
}
// Intersect found: replace skip scan with intersected one
scan.setFilter(newFilter);
} else if (filter instanceof FilterList) {
FilterList oldList = (FilterList) filter;
FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
for (Filter f : oldList.getFilters()) {
if (f instanceof SkipScanFilter) {
SkipScanFilter newFilter = ((SkipScanFilter) f).intersect(startKey, stopKey);
if (newFilter == null) {
return false;
}
newList.addFilter(newFilter);
} else {
newList.addFilter(f);
}
}
scan.setFilter(newList);
}
}
return mayHaveRows;
}
@Override
public List<String> copQueryAvailableNear(
String timestamp, final double latitude, final double longitude, final double radius) {
this.getStatLog(STAT_FILE_NAME);
this.getCSVLog(this.FILE_NAME_PREFIX, 0);
this.getCSVLog(this.FILE_NAME_PREFIX, 1);
this.getCSVLog(this.FILE_NAME_PREFIX, 2);
this.timePhase.clear();
try {
/** Step1** Call back class definition * */
class BixiCallBack implements Batch.Callback<RCopResult> {
RCopResult res = new RCopResult();
int count = 0;
QueryAbstraction query = null;
public BixiCallBack(QueryAbstraction query) {
this.query = query;
}
@Override
public void update(byte[] region, byte[] row, RCopResult result) {
long current = System.currentTimeMillis();
count++;
res.getRes().addAll(result.getRes()); // to verify the error when large data
res.setStart(result.getStart());
res.setEnd(result.getEnd());
res.setRows((res.getRows() + result.getRows()));
res.setCells(res.getCells() + result.getCells());
// System.out.println((count++) + ": come back region: "+ Bytes.toString(region) + ";
// result: "+ result.size());
/* String outStr="count=>"+count+";start=>"+result.getStart()+";end=>"+result.getEnd()+";process=>"+(result.getEnd()-result.getStart())+
";tranmission=>"+(System.currentTimeMillis()-result.getEnd())+
";region=>"+Bytes.toString(region)+
";row=>"+result.getRows()+";result=>"+result.getRes().size();
this.query.writeStat(outStr);
outStr = query.regions.get(Bytes.toString(region)).toString();
this.query.writeStat(outStr); */
// write them into csv file
String outStr = "";
outStr +=
"within,"
+ "cop,"
+ result.getParameter()
+ ","
+ result.getStart()
+ ","
+ result.getEnd()
+ ","
+ current
+ ","
+ result.getRows()
+ ","
+ result.getCells()
+ ","
+ ","
+ result.getKvLength()
+ result.getRes().size()
+ ","
+ this.query.regionAndRS.get(Bytes.toString(region))
+ ","
+ Bytes.toString(region);
this.query.writeCSVLog(outStr, 1);
}
}
BixiCallBack callBack = new BixiCallBack(this);
/** Step2** generate the scan ********** */
long s_time = System.currentTimeMillis();
this.timePhase.add(s_time);
// build up the Raster
XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
// match the query area in Raster to get the row range and column
// range
long match_s = System.currentTimeMillis();
XBox[] match_boxes = raster.match(latitude, longitude, radius);
long match_time = System.currentTimeMillis() - match_s;
String[] rowRange = new String[2];
rowRange[0] = match_boxes[0].getRow();
rowRange[1] = match_boxes[1].getRow() + "-*";
String[] c = raster.getColumns(match_boxes[0], match_boxes[1]);
// generate the scan
final Scan scan =
hbaseUtil.generateScan(
rowRange, null, new String[] {BixiConstant.LOCATION_FAMILY_NAME}, c, 1000000);
/** Step3** send out the query to trigger the corresponding function in Coprocessor*** */
this.timePhase.add(System.currentTimeMillis());
hbaseUtil
.getHTable()
.coprocessorExec(
BixiProtocol.class,
scan.getStartRow(),
scan.getStopRow(),
new Batch.Call<BixiProtocol, RCopResult>() {
public RCopResult call(BixiProtocol instance) throws IOException {
return instance.copQueryNeighbor4LS2(scan, latitude, longitude, radius);
};
},
callBack);
long e_time = System.currentTimeMillis();
this.timePhase.add(e_time);
long exe_time = e_time - s_time;
/* System.out.println("exe_time=>"+exe_time+";result=>"+callBack.res.getRes().size());
String outStr = "m=>cop;"+"radius=>"+radius+";exe_time=>"+exe_time+";result=>"+callBack.res.getRes().size();
this.writeStat(outStr);*/
// write to csv file
String outStr = "";
outStr +=
"within,"
+ "cop,"
+ callBack.res.getCells()
+ ","
+ callBack.res.getRes().size()
+ ","
+ exe_time
+ ","
+ callBack.res.getRows()
+ ","
+ match_time
+ ","
+ this.min_size_of_height
+ ","
+ radius;
this.writeCSVLog(outStr, 0);
String timeStr = "within,cop," + radius + ",";
for (int i = 0; i < this.timePhase.size(); i++) {
timeStr += this.timePhase.get(i) + ",";
}
this.writeCSVLog(timeStr, 2);
return callBack.res.getRes();
} catch (Exception e) {
e.printStackTrace();
} catch (Throwable ee) {
ee.printStackTrace();
} finally {
hbaseUtil.closeTableHandler();
this.closeStatLog();
this.closeCSVLog();
}
return null;
}
@Override
public String copQueryPoint(final double latitude, final double longitude) {
this.getStatLog(STAT_FILE_NAME);
long s_time = System.currentTimeMillis();
try {
/** Step1** Call back class definition * */
class BixiCallBack implements Batch.Callback<String> {
String res = null;
int count = 0;
QueryAbstraction query = null;
public BixiCallBack(QueryAbstraction query) {
this.query = query;
}
@Override
public void update(byte[] region, byte[] row, String result) {
count++;
String outStr =
"endTime=>"
+ System.currentTimeMillis()
+ ";region=>"
+ Bytes.toString(region)
+ ";count=>"
+ count
+ ";result=>"
+ result;
this.query.writeStat(outStr);
outStr = query.regions.get(Bytes.toString(region)).toString();
this.query.writeStat(outStr);
res = result;
}
}
BixiCallBack callBack = new BixiCallBack(this);
/** Step2** generate the scan ********** */
// build up the Raster
XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
XBox match_box = raster.locate(latitude, Math.abs(longitude));
System.out.println("match_box is : " + match_box.toString());
String[] rowRange = new String[2];
rowRange[0] = match_box.getRow();
rowRange[1] = match_box.getRow() + "0";
// generate the scan
final Scan scan =
hbaseUtil.generateScan(
rowRange,
null,
new String[] {BixiConstant.LOCATION_FAMILY_NAME},
new String[] {match_box.getColumn()},
100000);
/** Step3** send out the query to trigger the corresponding function in Coprocessor*** */
hbaseUtil
.getHTable()
.coprocessorExec(
BixiProtocol.class,
scan.getStartRow(),
scan.getStopRow(),
new Batch.Call<BixiProtocol, String>() {
public String call(BixiProtocol instance) throws IOException {
return instance.copQueryPoint4LS2(scan, latitude, longitude);
};
},
callBack);
long e_time = System.currentTimeMillis();
long exe_time = e_time - s_time;
// TODO store the time into database
System.out.println("exe_time=>" + exe_time + ";result=>" + callBack.res);
String outStr = "q=point;m=>cop;" + ";exe_time=>" + exe_time + ";result=>" + callBack.res;
this.writeStat(outStr);
return callBack.res;
} catch (Exception e) {
e.printStackTrace();
} catch (Throwable ee) {
ee.printStackTrace();
} finally {
hbaseUtil.closeTableHandler();
this.closeStatLog();
}
return null;
}
public Scan intersectScan(
Scan scan,
final byte[] originalStartKey,
final byte[] originalStopKey,
final int keyOffset,
boolean crossesRegionBoundary) {
byte[] startKey = originalStartKey;
byte[] stopKey = originalStopKey;
if (stopKey.length > 0 && Bytes.compareTo(startKey, stopKey) >= 0) {
return null;
}
boolean mayHaveRows = false;
// Keep the keys as they are if we have a point lookup, as we've already resolved the
// salt bytes in that case.
final int scanKeyOffset =
this.isSalted && !this.isPointLookup ? SaltingUtil.NUM_SALTING_BYTES : 0;
assert (scanKeyOffset == 0 || keyOffset == 0);
// Total offset for startKey/stopKey. Either 1 for salted tables or the prefix length
// of the current region for local indexes. We'll never have a case where a table is
// both salted and local.
final int totalKeyOffset = scanKeyOffset + keyOffset;
byte[] prefixBytes = ByteUtil.EMPTY_BYTE_ARRAY;
if (totalKeyOffset > 0) {
prefixBytes = ScanUtil.getPrefix(startKey, totalKeyOffset);
/*
* If our startKey to stopKey crosses a region boundary consider everything after the startKey as our scan
* is always done within a single region. This prevents us from having to prefix the key prior to knowing
* whether or not there may be an intersection. We can't calculate whether or not we've crossed a region
* boundary for local indexes, because we don't know the key offset of the next region, but only for the
* current one (which is the one passed in). If the next prefix happened to be a subset of the previous
* prefix, then this wouldn't detect that we crossed a region boundary.
*/
if (crossesRegionBoundary) {
stopKey = ByteUtil.EMPTY_BYTE_ARRAY;
}
}
int scanStartKeyOffset = scanKeyOffset;
byte[] scanStartKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStartRow();
// Compare ignoring key prefix and salt byte
if (scanStartKey.length > 0) {
if (startKey.length > 0
&& Bytes.compareTo(
scanStartKey,
scanKeyOffset,
scanStartKey.length - scanKeyOffset,
startKey,
totalKeyOffset,
startKey.length - totalKeyOffset)
< 0) {
scanStartKey = startKey;
scanStartKeyOffset = totalKeyOffset;
}
} else {
scanStartKey = startKey;
scanStartKeyOffset = totalKeyOffset;
mayHaveRows = true;
}
int scanStopKeyOffset = scanKeyOffset;
byte[] scanStopKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStopRow();
if (scanStopKey.length > 0) {
if (stopKey.length > 0
&& Bytes.compareTo(
scanStopKey,
scanKeyOffset,
scanStopKey.length - scanKeyOffset,
stopKey,
totalKeyOffset,
stopKey.length - totalKeyOffset)
> 0) {
scanStopKey = stopKey;
scanStopKeyOffset = totalKeyOffset;
}
} else {
scanStopKey = stopKey;
scanStopKeyOffset = totalKeyOffset;
mayHaveRows = true;
}
mayHaveRows =
mayHaveRows
|| Bytes.compareTo(
scanStartKey,
scanStartKeyOffset,
scanStartKey.length - scanStartKeyOffset,
scanStopKey,
scanStopKeyOffset,
scanStopKey.length - scanStopKeyOffset)
< 0;
if (!mayHaveRows) {
return null;
}
if (originalStopKey.length != 0 && scanStopKey.length == 0) {
scanStopKey = originalStopKey;
}
Filter newFilter = null;
// If the scan is using skip scan filter, intersect and replace the filter.
if (scan == null || this.useSkipScanFilter()) {
byte[] skipScanStartKey = scanStartKey;
byte[] skipScanStopKey = scanStopKey;
// If we have a keyOffset and we've used the startKey/stopKey that
// were passed in (which have the prefix) for the above range check,
// we need to remove the prefix before running our intersect method.
// TODO: we could use skipScanFilter.setOffset(keyOffset) if both
// the startKey and stopKey were used above *and* our intersect
// method honored the skipScanFilter.offset variable.
if (scanKeyOffset > 0) {
if (skipScanStartKey != originalStartKey) { // original already has correct salt byte
skipScanStartKey = replaceSaltByte(skipScanStartKey, prefixBytes);
}
if (skipScanStopKey != originalStopKey) {
skipScanStopKey = replaceSaltByte(skipScanStopKey, prefixBytes);
}
} else if (keyOffset > 0) {
if (skipScanStartKey == originalStartKey) {
skipScanStartKey = stripPrefix(skipScanStartKey, keyOffset);
}
if (skipScanStopKey == originalStopKey) {
skipScanStopKey = stripPrefix(skipScanStopKey, keyOffset);
}
}
if (scan == null) {
return filter.hasIntersect(skipScanStartKey, skipScanStopKey) ? HAS_INTERSECTION : null;
}
Filter filter = scan.getFilter();
SkipScanFilter newSkipScanFilter = null;
if (filter instanceof SkipScanFilter) {
SkipScanFilter oldSkipScanFilter = (SkipScanFilter) filter;
newFilter =
newSkipScanFilter = oldSkipScanFilter.intersect(skipScanStartKey, skipScanStopKey);
if (newFilter == null) {
return null;
}
} else if (filter instanceof FilterList) {
FilterList oldList = (FilterList) filter;
FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
newFilter = newList;
for (Filter f : oldList.getFilters()) {
if (f instanceof SkipScanFilter) {
newSkipScanFilter = ((SkipScanFilter) f).intersect(skipScanStartKey, skipScanStopKey);
if (newSkipScanFilter == null) {
return null;
}
newList.addFilter(newSkipScanFilter);
} else {
newList.addFilter(f);
}
}
}
// TODO: it seems that our SkipScanFilter or HBase runs into problems if we don't
// have an enclosing range when we do a point lookup.
if (isPointLookup) {
scanStartKey = ScanUtil.getMinKey(schema, newSkipScanFilter.getSlots(), slotSpan);
scanStopKey = ScanUtil.getMaxKey(schema, newSkipScanFilter.getSlots(), slotSpan);
}
}
if (newFilter == null) {
newFilter = scan.getFilter();
}
Scan newScan = ScanUtil.newScan(scan);
newScan.setFilter(newFilter);
// If we have an offset (salted table or local index), we need to make sure to
// prefix our scan start/stop row by the prefix of the startKey or stopKey that
// were passed in. Our scan either doesn't have the prefix or has a placeholder
// for it.
if (totalKeyOffset > 0) {
if (scanStartKey != originalStartKey) {
scanStartKey = prefixKey(scanStartKey, scanKeyOffset, prefixBytes, keyOffset);
}
if (scanStopKey != originalStopKey) {
scanStopKey = prefixKey(scanStopKey, scanKeyOffset, prefixBytes, keyOffset);
}
}
// Don't let the stopRow of the scan go beyond the originalStopKey
if (originalStopKey.length > 0 && Bytes.compareTo(scanStopKey, originalStopKey) > 0) {
scanStopKey = originalStopKey;
}
if (scanStopKey.length > 0 && Bytes.compareTo(scanStartKey, scanStopKey) >= 0) {
return null;
}
newScan.setAttribute(SCAN_ACTUAL_START_ROW, scanStartKey);
newScan.setStartRow(scanStartKey);
newScan.setStopRow(scanStopKey);
if (keyOffset > 0) {
newScan.setAttribute(STARTKEY_OFFSET, Bytes.toBytes(keyOffset));
}
return newScan;
}