private void appendScanRow(StringBuilder buf, Bound bound) {
ScanRanges scanRanges = context.getScanRanges();
KeyRange minMaxRange = context.getMinMaxRange();
Iterator<byte[]> minMaxIterator = Iterators.emptyIterator();
if (minMaxRange != null) {
RowKeySchema schema = tableRef.getTable().getRowKeySchema();
if (!minMaxRange.isUnbound(bound)) {
minMaxIterator = new RowKeyValueIterator(schema, minMaxRange.getRange(bound));
}
}
int nRanges = scanRanges.getRanges().size();
for (int i = 0, minPos = 0; minPos < nRanges || minMaxIterator.hasNext(); i++) {
List<KeyRange> ranges = minPos >= nRanges ? EVERYTHING : scanRanges.getRanges().get(minPos++);
KeyRange range = bound == Bound.LOWER ? ranges.get(0) : ranges.get(ranges.size() - 1);
byte[] b = range.getRange(bound);
Boolean isNull =
KeyRange.IS_NULL_RANGE == range
? Boolean.TRUE
: KeyRange.IS_NOT_NULL_RANGE == range ? Boolean.FALSE : null;
if (minMaxIterator.hasNext()) {
byte[] bMinMax = minMaxIterator.next();
int cmp = Bytes.compareTo(bMinMax, b) * (bound == Bound.LOWER ? 1 : -1);
if (cmp > 0) {
minPos = nRanges;
b = bMinMax;
isNull = null;
} else if (cmp < 0) {
minMaxIterator = Iterators.emptyIterator();
}
}
appendPKColumnValue(buf, b, isNull, i);
buf.append(',');
}
}
private static byte[] getKey(
RowKeySchema schema, List<List<KeyRange>> slots, int[] slotSpan, Bound bound) {
if (slots.isEmpty()) {
return KeyRange.UNBOUND;
}
int[] position = new int[slots.size()];
int maxLength = 0;
for (int i = 0; i < position.length; i++) {
position[i] = bound == Bound.LOWER ? 0 : slots.get(i).size() - 1;
KeyRange range = slots.get(i).get(position[i]);
Field field = schema.getField(i + slotSpan[i]);
int keyLength = range.getRange(bound).length;
if (!field.getDataType().isFixedWidth()) {
keyLength++;
if (range.isUnbound(bound)
&& !range.isInclusive(bound)
&& field.getSortOrder() == SortOrder.DESC) {
keyLength++;
}
}
maxLength += keyLength;
}
byte[] key = new byte[maxLength];
int length = setKey(schema, slots, slotSpan, position, bound, key, 0, 0, position.length);
if (length == 0) {
return KeyRange.UNBOUND;
}
if (length == maxLength) {
return key;
}
byte[] keyCopy = new byte[length];
System.arraycopy(key, 0, keyCopy, 0, length);
return keyCopy;
}
public static int getMaxKeyLength(RowKeySchema schema, List<List<KeyRange>> slots) {
int maxKeyLength = getTerminatorCount(schema) * 2;
for (List<KeyRange> slot : slots) {
int maxSlotLength = 0;
for (KeyRange range : slot) {
int maxRangeLength = Math.max(range.getLowerRange().length, range.getUpperRange().length);
if (maxSlotLength < maxRangeLength) {
maxSlotLength = maxRangeLength;
}
}
maxKeyLength += maxSlotLength;
}
return maxKeyLength;
}
public int getBoundSlotCount() {
int count = 0;
boolean hasUnbound = false;
int nRanges = ranges.size();
for (int i = 0; i < nRanges && !hasUnbound; i++) {
List<KeyRange> orRanges = ranges.get(i);
for (KeyRange range : orRanges) {
if (range == KeyRange.EVERYTHING_RANGE) {
return count;
}
if (range.isUnbound()) {
hasUnbound = true;
}
}
count++;
}
return count;
}
/**
* Finds the total number of row keys spanned by this ranges / slotSpan pair. This accounts for
* slots in the ranges that may span more than on row key.
*
* @param ranges the KeyRange slots paired with this slotSpan. corresponds to {@link
* ScanRanges#ranges}
* @param slotSpan the extra span per skip scan slot. corresponds to {@link ScanRanges#slotSpan}
* @return the total number of row keys spanned yb this ranges / slotSpan pair.
*/
private static int getBoundPkSpan(List<List<KeyRange>> ranges, int[] slotSpan) {
int count = 0;
boolean hasUnbound = false;
int nRanges = ranges.size();
for (int i = 0; i < nRanges && !hasUnbound; i++) {
List<KeyRange> orRanges = ranges.get(i);
for (KeyRange range : orRanges) {
if (range == KeyRange.EVERYTHING_RANGE) {
return count;
}
if (range.isUnbound()) {
hasUnbound = true;
}
}
count += slotSpan[i] + 1;
}
return count;
}
private static boolean isPointLookup(
RowKeySchema schema, List<List<KeyRange>> ranges, int[] slotSpan, boolean useSkipScan) {
if (!isFullyQualified(schema, ranges, slotSpan)) {
return false;
}
int lastIndex = ranges.size() - 1;
for (int i = lastIndex; i >= 0; i--) {
List<KeyRange> orRanges = ranges.get(i);
if (!useSkipScan && orRanges.size() > 1) {
return false;
}
for (KeyRange keyRange : orRanges) {
// Special case for single trailing IS NULL. We cannot consider this as a point key because
// we strip trailing nulls when we form the key.
if (!keyRange.isSingleKey() || (i == lastIndex && keyRange == KeyRange.IS_NULL_RANGE)) {
return false;
}
}
}
return true;
}
private boolean explainSkipScan(StringBuilder buf) {
ScanRanges scanRanges = context.getScanRanges();
if (scanRanges.isPointLookup()) {
int keyCount = scanRanges.getPointLookupCount();
buf.append("POINT LOOKUP ON " + keyCount + " KEY" + (keyCount > 1 ? "S " : " "));
} else if (scanRanges.useSkipScanFilter()) {
buf.append("SKIP SCAN ");
int count = 1;
boolean hasRanges = false;
for (List<KeyRange> ranges : scanRanges.getRanges()) {
count *= ranges.size();
for (KeyRange range : ranges) {
hasRanges |= !range.isSingleKey();
}
}
buf.append("ON ");
buf.append(count);
buf.append(hasRanges ? " RANGE" : " KEY");
buf.append(count > 1 ? "S " : " ");
} else {
buf.append("RANGE SCAN ");
}
return scanRanges.useSkipScanFilter();
}
/**
* Converts a partially qualified KeyRange into a KeyRange with a inclusive lower bound and an
* exclusive upper bound, widening as necessary.
*/
public static KeyRange convertToInclusiveExclusiveRange(
KeyRange partialRange, RowKeySchema schema, ImmutableBytesWritable ptr) {
// Ensure minMaxRange is lower inclusive and upper exclusive, as that's
// what we need to intersect against for the HBase scan.
byte[] lowerRange = partialRange.getLowerRange();
if (!partialRange.lowerUnbound()) {
if (!partialRange.isLowerInclusive()) {
lowerRange = ScanUtil.nextKey(lowerRange, schema, ptr);
}
}
byte[] upperRange = partialRange.getUpperRange();
if (!partialRange.upperUnbound()) {
if (partialRange.isUpperInclusive()) {
upperRange = ScanUtil.nextKey(upperRange, schema, ptr);
}
}
if (partialRange.getLowerRange() != lowerRange || partialRange.getUpperRange() != upperRange) {
partialRange = KeyRange.getKeyRange(lowerRange, upperRange);
}
return partialRange;
}
public static TimeRange getDescTimeRange(
KeyRange lowestKeyRange, KeyRange highestKeyRange, Field f) throws IOException {
boolean lowerUnbound = lowestKeyRange.lowerUnbound();
boolean lowerInclusive = lowestKeyRange.isLowerInclusive();
boolean upperUnbound = highestKeyRange.upperUnbound();
boolean upperInclusive = highestKeyRange.isUpperInclusive();
long low =
lowerUnbound
? -1
: f.getDataType()
.getCodec()
.decodeLong(lowestKeyRange.getLowerRange(), 0, SortOrder.DESC);
long high =
upperUnbound
? -1
: f.getDataType()
.getCodec()
.decodeLong(highestKeyRange.getUpperRange(), 0, SortOrder.DESC);
long newHigh;
long newLow;
if (!lowerUnbound && !upperUnbound) {
newHigh = lowerInclusive ? safelyIncrement(low) : low;
newLow = upperInclusive ? high : safelyIncrement(high);
return new TimeRange(newLow, newHigh);
} else if (!lowerUnbound && upperUnbound) {
newHigh = lowerInclusive ? safelyIncrement(low) : low;
newLow = 0;
return new TimeRange(newLow, newHigh);
} else if (lowerUnbound && !upperUnbound) {
newLow = upperInclusive ? high : safelyIncrement(high);
newHigh = HConstants.LATEST_TIMESTAMP;
return new TimeRange(newLow, newHigh);
} else {
newLow = 0;
newHigh = HConstants.LATEST_TIMESTAMP;
return new TimeRange(newLow, newHigh);
}
}
private static TimeRange getAscTimeRange(KeyRange lowestRange, KeyRange highestRange, Field f)
throws IOException {
long low;
long high;
if (lowestRange.lowerUnbound()) {
low = 0;
} else {
long lowerRange =
f.getDataType().getCodec().decodeLong(lowestRange.getLowerRange(), 0, SortOrder.ASC);
low = lowestRange.isLowerInclusive() ? lowerRange : safelyIncrement(lowerRange);
}
if (highestRange.upperUnbound()) {
high = HConstants.LATEST_TIMESTAMP;
} else {
long upperRange =
f.getDataType().getCodec().decodeLong(highestRange.getUpperRange(), 0, SortOrder.ASC);
if (highestRange.isUpperInclusive()) {
high = safelyIncrement(upperRange);
} else {
high = upperRange;
}
}
return new TimeRange(low, high);
}
public static int setKey(
RowKeySchema schema,
List<List<KeyRange>> slots,
int[] slotSpan,
int[] position,
Bound bound,
byte[] key,
int byteOffset,
int slotStartIndex,
int slotEndIndex,
int schemaStartIndex) {
int offset = byteOffset;
boolean lastInclusiveUpperSingleKey = false;
boolean anyInclusiveUpperRangeKey = false;
// The index used for slots should be incremented by 1,
// but the index for the field it represents in the schema
// should be incremented by 1 + value in the current slotSpan index
// slotSpan stores the number of columns beyond one that the range spans
Field field = null;
int i = slotStartIndex, fieldIndex = ScanUtil.getRowKeyPosition(slotSpan, slotStartIndex);
for (i = slotStartIndex; i < slotEndIndex; i++) {
// Build up the key by appending the bound of each key range
// from the current position of each slot.
KeyRange range = slots.get(i).get(position[i]);
// Use last slot in a multi-span column to determine if fixed width
field = schema.getField(fieldIndex + slotSpan[i]);
boolean isFixedWidth = field.getDataType().isFixedWidth();
fieldIndex += slotSpan[i] + 1;
/*
* If the current slot is unbound then stop if:
* 1) setting the upper bound. There's no value in
* continuing because nothing will be filtered.
* 2) setting the lower bound when the type is fixed length
* for the same reason. However, if the type is variable width
* continue building the key because null values will be filtered
* since our separator byte will be appended and incremented.
*/
if (range.isUnbound(bound) && (bound == Bound.UPPER || isFixedWidth)) {
break;
}
byte[] bytes = range.getRange(bound);
System.arraycopy(bytes, 0, key, offset, bytes.length);
offset += bytes.length;
/*
* We must add a terminator to a variable length key even for the last PK column if
* the lower key is non inclusive or the upper key is inclusive. Otherwise, we'd be
* incrementing the key value itself, and thus bumping it up too much.
*/
boolean inclusiveUpper = range.isInclusive(bound) && bound == Bound.UPPER;
boolean exclusiveLower = !range.isInclusive(bound) && bound == Bound.LOWER;
// If we are setting the upper bound of using inclusive single key, we remember
// to increment the key if we exit the loop after this iteration.
//
// We remember to increment the last slot if we are setting the upper bound with an
// inclusive range key.
//
// We cannot combine the two flags together in case for single-inclusive key followed
// by the range-exclusive key. In that case, we do not need to increment the end at the
// end. But if we combine the two flag, the single inclusive key in the middle of the
// key slots would cause the flag to become true.
lastInclusiveUpperSingleKey = range.isSingleKey() && inclusiveUpper;
anyInclusiveUpperRangeKey |= !range.isSingleKey() && inclusiveUpper;
// A null or empty byte array is always represented as a zero byte
byte sepByte =
SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), bytes.length == 0, field);
if (!isFixedWidth
&& (fieldIndex < schema.getMaxFields()
|| inclusiveUpper
|| exclusiveLower
|| sepByte == QueryConstants.DESC_SEPARATOR_BYTE)) {
key[offset++] = sepByte;
// Set lastInclusiveUpperSingleKey back to false if this is the last pk column
// as we don't want to increment the null byte in this case
lastInclusiveUpperSingleKey &= i < schema.getMaxFields() - 1;
}
// If we are setting the lower bound with an exclusive range key, we need to bump the
// slot up for each key part. For an upper bound, we bump up an inclusive key, but
// only after the last key part.
if (exclusiveLower) {
if (!ByteUtil.nextKey(key, offset)) {
// Special case for not being able to increment.
// In this case we return a negative byteOffset to
// remove this part from the key being formed. Since the
// key has overflowed, this means that we should not
// have an end key specified.
return -byteOffset;
}
// We're filtering on values being non null here, but we still need the 0xFF
// terminator, since DESC keys ignore the last byte as it's expected to be
// the terminator. Without this, we'd ignore the separator byte that was
// just added and incremented.
if (!isFixedWidth
&& bytes.length == 0
&& SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, field)
== QueryConstants.DESC_SEPARATOR_BYTE) {
key[offset++] = QueryConstants.DESC_SEPARATOR_BYTE;
}
}
}
if (lastInclusiveUpperSingleKey || anyInclusiveUpperRangeKey) {
if (!ByteUtil.nextKey(key, offset)) {
// Special case for not being able to increment.
// In this case we return a negative byteOffset to
// remove this part from the key being formed. Since the
// key has overflowed, this means that we should not
// have an end key specified.
return -byteOffset;
}
}
// Remove trailing separator bytes, since the columns may have been added
// after the table has data, in which case there won't be a separator
// byte.
if (bound == Bound.LOWER) {
while (--i >= schemaStartIndex
&& offset > byteOffset
&& !(field = schema.getField(--fieldIndex)).getDataType().isFixedWidth()
&& field.getSortOrder() == SortOrder.ASC
&& key[offset - 1] == QueryConstants.SEPARATOR_BYTE) {
offset--;
fieldIndex -= slotSpan[i];
}
}
return offset - byteOffset;
}
public void initializeScan(Scan scan) {
scan.setStartRow(scanRange.getLowerRange());
scan.setStopRow(scanRange.getUpperRange());
}
public static ScanRanges create(
RowKeySchema schema,
List<List<KeyRange>> ranges,
int[] slotSpan,
KeyRange minMaxRange,
Integer nBuckets,
boolean useSkipScan,
int rowTimestampColIndex) {
int offset = nBuckets == null ? 0 : SaltingUtil.NUM_SALTING_BYTES;
int nSlots = ranges.size();
if (nSlots == offset && minMaxRange == KeyRange.EVERYTHING_RANGE) {
return EVERYTHING;
} else if (minMaxRange == KeyRange.EMPTY_RANGE
|| (nSlots == 1 + offset
&& ranges.get(offset).size() == 1
&& ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) {
return NOTHING;
}
TimeRange rowTimestampRange = getRowTimestampColumnRange(ranges, schema, rowTimestampColIndex);
boolean isPointLookup = isPointLookup(schema, ranges, slotSpan, useSkipScan);
if (isPointLookup) {
// TODO: consider keeping original to use for serialization as it would be smaller?
List<byte[]> keys = ScanRanges.getPointKeys(ranges, slotSpan, schema, nBuckets);
List<KeyRange> keyRanges = Lists.newArrayListWithExpectedSize(keys.size());
KeyRange unsaltedMinMaxRange = minMaxRange;
if (nBuckets != null && minMaxRange != KeyRange.EVERYTHING_RANGE) {
unsaltedMinMaxRange =
KeyRange.getKeyRange(
stripPrefix(minMaxRange.getLowerRange(), offset),
minMaxRange.lowerUnbound(),
stripPrefix(minMaxRange.getUpperRange(), offset),
minMaxRange.upperUnbound());
}
// We have full keys here, so use field from our varbinary schema
BytesComparator comparator = ScanUtil.getComparator(SchemaUtil.VAR_BINARY_SCHEMA.getField(0));
for (byte[] key : keys) {
// Filter now based on unsalted minMaxRange and ignore the point key salt byte
if (unsaltedMinMaxRange.compareLowerToUpperBound(
key, offset, key.length - offset, true, comparator)
<= 0
&& unsaltedMinMaxRange.compareUpperToLowerBound(
key, offset, key.length - offset, true, comparator)
>= 0) {
keyRanges.add(KeyRange.getKeyRange(key));
}
}
ranges = Collections.singletonList(keyRanges);
useSkipScan = keyRanges.size() > 1;
// Treat as binary if descending because we've got a separator byte at the end
// which is not part of the value.
if (keys.size() > 1
|| SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, schema.getField(0))
== QueryConstants.DESC_SEPARATOR_BYTE) {
schema = SchemaUtil.VAR_BINARY_SCHEMA;
slotSpan = ScanUtil.SINGLE_COLUMN_SLOT_SPAN;
} else {
// Keep original schema and don't use skip scan as it's not necessary
// when there's a single key.
slotSpan = new int[] {schema.getMaxFields() - 1};
}
}
List<List<KeyRange>> sortedRanges = Lists.newArrayListWithExpectedSize(ranges.size());
for (int i = 0; i < ranges.size(); i++) {
List<KeyRange> sorted = Lists.newArrayList(ranges.get(i));
Collections.sort(sorted, KeyRange.COMPARATOR);
sortedRanges.add(ImmutableList.copyOf(sorted));
}
// Don't set minMaxRange for point lookup because it causes issues during intersect
// by going across region boundaries
KeyRange scanRange = KeyRange.EVERYTHING_RANGE;
// if (!isPointLookup && (nBuckets == null || !useSkipScanFilter)) {
// if (! ( isPointLookup || (nBuckets != null && useSkipScanFilter) ) ) {
// if (nBuckets == null || (nBuckets != null && (!isPointLookup || !useSkipScanFilter))) {
if (nBuckets == null || !isPointLookup || !useSkipScan) {
byte[] minKey = ScanUtil.getMinKey(schema, sortedRanges, slotSpan);
byte[] maxKey = ScanUtil.getMaxKey(schema, sortedRanges, slotSpan);
// If the maxKey has crossed the salt byte boundary, then we do not
// have anything to filter at the upper end of the range
if (ScanUtil.crossesPrefixBoundary(maxKey, ScanUtil.getPrefix(minKey, offset), offset)) {
maxKey = KeyRange.UNBOUND;
}
// We won't filter anything at the low end of the range if we just have the salt byte
if (minKey.length <= offset) {
minKey = KeyRange.UNBOUND;
}
scanRange = KeyRange.getKeyRange(minKey, maxKey);
}
if (minMaxRange != KeyRange.EVERYTHING_RANGE) {
minMaxRange =
ScanUtil.convertToInclusiveExclusiveRange(
minMaxRange, schema, new ImmutableBytesWritable());
scanRange = scanRange.intersect(minMaxRange);
}
if (scanRange == KeyRange.EMPTY_RANGE) {
return NOTHING;
}
return new ScanRanges(
schema,
slotSpan,
sortedRanges,
scanRange,
minMaxRange,
useSkipScan,
isPointLookup,
nBuckets,
rowTimestampRange);
}