public Scan intersectScan(
Scan scan,
final byte[] originalStartKey,
final byte[] originalStopKey,
final int keyOffset,
boolean crossesRegionBoundary) {
byte[] startKey = originalStartKey;
byte[] stopKey = originalStopKey;
if (stopKey.length > 0 && Bytes.compareTo(startKey, stopKey) >= 0) {
return null;
}
boolean mayHaveRows = false;
// Keep the keys as they are if we have a point lookup, as we've already resolved the
// salt bytes in that case.
final int scanKeyOffset =
this.isSalted && !this.isPointLookup ? SaltingUtil.NUM_SALTING_BYTES : 0;
assert (scanKeyOffset == 0 || keyOffset == 0);
// Total offset for startKey/stopKey. Either 1 for salted tables or the prefix length
// of the current region for local indexes. We'll never have a case where a table is
// both salted and local.
final int totalKeyOffset = scanKeyOffset + keyOffset;
byte[] prefixBytes = ByteUtil.EMPTY_BYTE_ARRAY;
if (totalKeyOffset > 0) {
prefixBytes = ScanUtil.getPrefix(startKey, totalKeyOffset);
/*
* If our startKey to stopKey crosses a region boundary consider everything after the startKey as our scan
* is always done within a single region. This prevents us from having to prefix the key prior to knowing
* whether or not there may be an intersection. We can't calculate whether or not we've crossed a region
* boundary for local indexes, because we don't know the key offset of the next region, but only for the
* current one (which is the one passed in). If the next prefix happened to be a subset of the previous
* prefix, then this wouldn't detect that we crossed a region boundary.
*/
if (crossesRegionBoundary) {
stopKey = ByteUtil.EMPTY_BYTE_ARRAY;
}
}
int scanStartKeyOffset = scanKeyOffset;
byte[] scanStartKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStartRow();
// Compare ignoring key prefix and salt byte
if (scanStartKey.length > 0) {
if (startKey.length > 0
&& Bytes.compareTo(
scanStartKey,
scanKeyOffset,
scanStartKey.length - scanKeyOffset,
startKey,
totalKeyOffset,
startKey.length - totalKeyOffset)
< 0) {
scanStartKey = startKey;
scanStartKeyOffset = totalKeyOffset;
}
} else {
scanStartKey = startKey;
scanStartKeyOffset = totalKeyOffset;
mayHaveRows = true;
}
int scanStopKeyOffset = scanKeyOffset;
byte[] scanStopKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStopRow();
if (scanStopKey.length > 0) {
if (stopKey.length > 0
&& Bytes.compareTo(
scanStopKey,
scanKeyOffset,
scanStopKey.length - scanKeyOffset,
stopKey,
totalKeyOffset,
stopKey.length - totalKeyOffset)
> 0) {
scanStopKey = stopKey;
scanStopKeyOffset = totalKeyOffset;
}
} else {
scanStopKey = stopKey;
scanStopKeyOffset = totalKeyOffset;
mayHaveRows = true;
}
mayHaveRows =
mayHaveRows
|| Bytes.compareTo(
scanStartKey,
scanStartKeyOffset,
scanStartKey.length - scanStartKeyOffset,
scanStopKey,
scanStopKeyOffset,
scanStopKey.length - scanStopKeyOffset)
< 0;
if (!mayHaveRows) {
return null;
}
if (originalStopKey.length != 0 && scanStopKey.length == 0) {
scanStopKey = originalStopKey;
}
Filter newFilter = null;
// If the scan is using skip scan filter, intersect and replace the filter.
if (scan == null || this.useSkipScanFilter()) {
byte[] skipScanStartKey = scanStartKey;
byte[] skipScanStopKey = scanStopKey;
// If we have a keyOffset and we've used the startKey/stopKey that
// were passed in (which have the prefix) for the above range check,
// we need to remove the prefix before running our intersect method.
// TODO: we could use skipScanFilter.setOffset(keyOffset) if both
// the startKey and stopKey were used above *and* our intersect
// method honored the skipScanFilter.offset variable.
if (scanKeyOffset > 0) {
if (skipScanStartKey != originalStartKey) { // original already has correct salt byte
skipScanStartKey = replaceSaltByte(skipScanStartKey, prefixBytes);
}
if (skipScanStopKey != originalStopKey) {
skipScanStopKey = replaceSaltByte(skipScanStopKey, prefixBytes);
}
} else if (keyOffset > 0) {
if (skipScanStartKey == originalStartKey) {
skipScanStartKey = stripPrefix(skipScanStartKey, keyOffset);
}
if (skipScanStopKey == originalStopKey) {
skipScanStopKey = stripPrefix(skipScanStopKey, keyOffset);
}
}
if (scan == null) {
return filter.hasIntersect(skipScanStartKey, skipScanStopKey) ? HAS_INTERSECTION : null;
}
Filter filter = scan.getFilter();
SkipScanFilter newSkipScanFilter = null;
if (filter instanceof SkipScanFilter) {
SkipScanFilter oldSkipScanFilter = (SkipScanFilter) filter;
newFilter =
newSkipScanFilter = oldSkipScanFilter.intersect(skipScanStartKey, skipScanStopKey);
if (newFilter == null) {
return null;
}
} else if (filter instanceof FilterList) {
FilterList oldList = (FilterList) filter;
FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
newFilter = newList;
for (Filter f : oldList.getFilters()) {
if (f instanceof SkipScanFilter) {
newSkipScanFilter = ((SkipScanFilter) f).intersect(skipScanStartKey, skipScanStopKey);
if (newSkipScanFilter == null) {
return null;
}
newList.addFilter(newSkipScanFilter);
} else {
newList.addFilter(f);
}
}
}
// TODO: it seems that our SkipScanFilter or HBase runs into problems if we don't
// have an enclosing range when we do a point lookup.
if (isPointLookup) {
scanStartKey = ScanUtil.getMinKey(schema, newSkipScanFilter.getSlots(), slotSpan);
scanStopKey = ScanUtil.getMaxKey(schema, newSkipScanFilter.getSlots(), slotSpan);
}
}
if (newFilter == null) {
newFilter = scan.getFilter();
}
Scan newScan = ScanUtil.newScan(scan);
newScan.setFilter(newFilter);
// If we have an offset (salted table or local index), we need to make sure to
// prefix our scan start/stop row by the prefix of the startKey or stopKey that
// were passed in. Our scan either doesn't have the prefix or has a placeholder
// for it.
if (totalKeyOffset > 0) {
if (scanStartKey != originalStartKey) {
scanStartKey = prefixKey(scanStartKey, scanKeyOffset, prefixBytes, keyOffset);
}
if (scanStopKey != originalStopKey) {
scanStopKey = prefixKey(scanStopKey, scanKeyOffset, prefixBytes, keyOffset);
}
}
// Don't let the stopRow of the scan go beyond the originalStopKey
if (originalStopKey.length > 0 && Bytes.compareTo(scanStopKey, originalStopKey) > 0) {
scanStopKey = originalStopKey;
}
if (scanStopKey.length > 0 && Bytes.compareTo(scanStartKey, scanStopKey) >= 0) {
return null;
}
newScan.setAttribute(SCAN_ACTUAL_START_ROW, scanStartKey);
newScan.setStartRow(scanStartKey);
newScan.setStopRow(scanStopKey);
if (keyOffset > 0) {
newScan.setAttribute(STARTKEY_OFFSET, Bytes.toBytes(keyOffset));
}
return newScan;
}
public static ScanRanges create(
RowKeySchema schema,
List<List<KeyRange>> ranges,
int[] slotSpan,
KeyRange minMaxRange,
Integer nBuckets,
boolean useSkipScan,
int rowTimestampColIndex) {
int offset = nBuckets == null ? 0 : SaltingUtil.NUM_SALTING_BYTES;
int nSlots = ranges.size();
if (nSlots == offset && minMaxRange == KeyRange.EVERYTHING_RANGE) {
return EVERYTHING;
} else if (minMaxRange == KeyRange.EMPTY_RANGE
|| (nSlots == 1 + offset
&& ranges.get(offset).size() == 1
&& ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) {
return NOTHING;
}
TimeRange rowTimestampRange = getRowTimestampColumnRange(ranges, schema, rowTimestampColIndex);
boolean isPointLookup = isPointLookup(schema, ranges, slotSpan, useSkipScan);
if (isPointLookup) {
// TODO: consider keeping original to use for serialization as it would be smaller?
List<byte[]> keys = ScanRanges.getPointKeys(ranges, slotSpan, schema, nBuckets);
List<KeyRange> keyRanges = Lists.newArrayListWithExpectedSize(keys.size());
KeyRange unsaltedMinMaxRange = minMaxRange;
if (nBuckets != null && minMaxRange != KeyRange.EVERYTHING_RANGE) {
unsaltedMinMaxRange =
KeyRange.getKeyRange(
stripPrefix(minMaxRange.getLowerRange(), offset),
minMaxRange.lowerUnbound(),
stripPrefix(minMaxRange.getUpperRange(), offset),
minMaxRange.upperUnbound());
}
// We have full keys here, so use field from our varbinary schema
BytesComparator comparator = ScanUtil.getComparator(SchemaUtil.VAR_BINARY_SCHEMA.getField(0));
for (byte[] key : keys) {
// Filter now based on unsalted minMaxRange and ignore the point key salt byte
if (unsaltedMinMaxRange.compareLowerToUpperBound(
key, offset, key.length - offset, true, comparator)
<= 0
&& unsaltedMinMaxRange.compareUpperToLowerBound(
key, offset, key.length - offset, true, comparator)
>= 0) {
keyRanges.add(KeyRange.getKeyRange(key));
}
}
ranges = Collections.singletonList(keyRanges);
useSkipScan = keyRanges.size() > 1;
// Treat as binary if descending because we've got a separator byte at the end
// which is not part of the value.
if (keys.size() > 1
|| SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, schema.getField(0))
== QueryConstants.DESC_SEPARATOR_BYTE) {
schema = SchemaUtil.VAR_BINARY_SCHEMA;
slotSpan = ScanUtil.SINGLE_COLUMN_SLOT_SPAN;
} else {
// Keep original schema and don't use skip scan as it's not necessary
// when there's a single key.
slotSpan = new int[] {schema.getMaxFields() - 1};
}
}
List<List<KeyRange>> sortedRanges = Lists.newArrayListWithExpectedSize(ranges.size());
for (int i = 0; i < ranges.size(); i++) {
List<KeyRange> sorted = Lists.newArrayList(ranges.get(i));
Collections.sort(sorted, KeyRange.COMPARATOR);
sortedRanges.add(ImmutableList.copyOf(sorted));
}
// Don't set minMaxRange for point lookup because it causes issues during intersect
// by going across region boundaries
KeyRange scanRange = KeyRange.EVERYTHING_RANGE;
// if (!isPointLookup && (nBuckets == null || !useSkipScanFilter)) {
// if (! ( isPointLookup || (nBuckets != null && useSkipScanFilter) ) ) {
// if (nBuckets == null || (nBuckets != null && (!isPointLookup || !useSkipScanFilter))) {
if (nBuckets == null || !isPointLookup || !useSkipScan) {
byte[] minKey = ScanUtil.getMinKey(schema, sortedRanges, slotSpan);
byte[] maxKey = ScanUtil.getMaxKey(schema, sortedRanges, slotSpan);
// If the maxKey has crossed the salt byte boundary, then we do not
// have anything to filter at the upper end of the range
if (ScanUtil.crossesPrefixBoundary(maxKey, ScanUtil.getPrefix(minKey, offset), offset)) {
maxKey = KeyRange.UNBOUND;
}
// We won't filter anything at the low end of the range if we just have the salt byte
if (minKey.length <= offset) {
minKey = KeyRange.UNBOUND;
}
scanRange = KeyRange.getKeyRange(minKey, maxKey);
}
if (minMaxRange != KeyRange.EVERYTHING_RANGE) {
minMaxRange =
ScanUtil.convertToInclusiveExclusiveRange(
minMaxRange, schema, new ImmutableBytesWritable());
scanRange = scanRange.intersect(minMaxRange);
}
if (scanRange == KeyRange.EMPTY_RANGE) {
return NOTHING;
}
return new ScanRanges(
schema,
slotSpan,
sortedRanges,
scanRange,
minMaxRange,
useSkipScan,
isPointLookup,
nBuckets,
rowTimestampRange);
}