Ejemplo n.º 1
0
 private void appendScanRow(StringBuilder buf, Bound bound) {
   ScanRanges scanRanges = context.getScanRanges();
   KeyRange minMaxRange = context.getMinMaxRange();
   Iterator<byte[]> minMaxIterator = Iterators.emptyIterator();
   if (minMaxRange != null) {
     RowKeySchema schema = tableRef.getTable().getRowKeySchema();
     if (!minMaxRange.isUnbound(bound)) {
       minMaxIterator = new RowKeyValueIterator(schema, minMaxRange.getRange(bound));
     }
   }
   int nRanges = scanRanges.getRanges().size();
   for (int i = 0, minPos = 0; minPos < nRanges || minMaxIterator.hasNext(); i++) {
     List<KeyRange> ranges = minPos >= nRanges ? EVERYTHING : scanRanges.getRanges().get(minPos++);
     KeyRange range = bound == Bound.LOWER ? ranges.get(0) : ranges.get(ranges.size() - 1);
     byte[] b = range.getRange(bound);
     Boolean isNull =
         KeyRange.IS_NULL_RANGE == range
             ? Boolean.TRUE
             : KeyRange.IS_NOT_NULL_RANGE == range ? Boolean.FALSE : null;
     if (minMaxIterator.hasNext()) {
       byte[] bMinMax = minMaxIterator.next();
       int cmp = Bytes.compareTo(bMinMax, b) * (bound == Bound.LOWER ? 1 : -1);
       if (cmp > 0) {
         minPos = nRanges;
         b = bMinMax;
         isNull = null;
       } else if (cmp < 0) {
         minMaxIterator = Iterators.emptyIterator();
       }
     }
     appendPKColumnValue(buf, b, isNull, i);
     buf.append(',');
   }
 }
Ejemplo n.º 2
0
 private static byte[] getKey(
     RowKeySchema schema, List<List<KeyRange>> slots, int[] slotSpan, Bound bound) {
   if (slots.isEmpty()) {
     return KeyRange.UNBOUND;
   }
   int[] position = new int[slots.size()];
   int maxLength = 0;
   for (int i = 0; i < position.length; i++) {
     position[i] = bound == Bound.LOWER ? 0 : slots.get(i).size() - 1;
     KeyRange range = slots.get(i).get(position[i]);
     Field field = schema.getField(i + slotSpan[i]);
     int keyLength = range.getRange(bound).length;
     if (!field.getDataType().isFixedWidth()) {
       keyLength++;
       if (range.isUnbound(bound)
           && !range.isInclusive(bound)
           && field.getSortOrder() == SortOrder.DESC) {
         keyLength++;
       }
     }
     maxLength += keyLength;
   }
   byte[] key = new byte[maxLength];
   int length = setKey(schema, slots, slotSpan, position, bound, key, 0, 0, position.length);
   if (length == 0) {
     return KeyRange.UNBOUND;
   }
   if (length == maxLength) {
     return key;
   }
   byte[] keyCopy = new byte[length];
   System.arraycopy(key, 0, keyCopy, 0, length);
   return keyCopy;
 }
Ejemplo n.º 3
0
 public static int getMaxKeyLength(RowKeySchema schema, List<List<KeyRange>> slots) {
   int maxKeyLength = getTerminatorCount(schema) * 2;
   for (List<KeyRange> slot : slots) {
     int maxSlotLength = 0;
     for (KeyRange range : slot) {
       int maxRangeLength = Math.max(range.getLowerRange().length, range.getUpperRange().length);
       if (maxSlotLength < maxRangeLength) {
         maxSlotLength = maxRangeLength;
       }
     }
     maxKeyLength += maxSlotLength;
   }
   return maxKeyLength;
 }
Ejemplo n.º 4
0
  public int getBoundSlotCount() {
    int count = 0;
    boolean hasUnbound = false;
    int nRanges = ranges.size();

    for (int i = 0; i < nRanges && !hasUnbound; i++) {
      List<KeyRange> orRanges = ranges.get(i);
      for (KeyRange range : orRanges) {
        if (range == KeyRange.EVERYTHING_RANGE) {
          return count;
        }
        if (range.isUnbound()) {
          hasUnbound = true;
        }
      }
      count++;
    }

    return count;
  }
Ejemplo n.º 5
0
  /**
   * Finds the total number of row keys spanned by this ranges / slotSpan pair. This accounts for
   * slots in the ranges that may span more than on row key.
   *
   * @param ranges the KeyRange slots paired with this slotSpan. corresponds to {@link
   *     ScanRanges#ranges}
   * @param slotSpan the extra span per skip scan slot. corresponds to {@link ScanRanges#slotSpan}
   * @return the total number of row keys spanned yb this ranges / slotSpan pair.
   */
  private static int getBoundPkSpan(List<List<KeyRange>> ranges, int[] slotSpan) {
    int count = 0;
    boolean hasUnbound = false;
    int nRanges = ranges.size();

    for (int i = 0; i < nRanges && !hasUnbound; i++) {
      List<KeyRange> orRanges = ranges.get(i);
      for (KeyRange range : orRanges) {
        if (range == KeyRange.EVERYTHING_RANGE) {
          return count;
        }
        if (range.isUnbound()) {
          hasUnbound = true;
        }
      }
      count += slotSpan[i] + 1;
    }

    return count;
  }
Ejemplo n.º 6
0
 private static boolean isPointLookup(
     RowKeySchema schema, List<List<KeyRange>> ranges, int[] slotSpan, boolean useSkipScan) {
   if (!isFullyQualified(schema, ranges, slotSpan)) {
     return false;
   }
   int lastIndex = ranges.size() - 1;
   for (int i = lastIndex; i >= 0; i--) {
     List<KeyRange> orRanges = ranges.get(i);
     if (!useSkipScan && orRanges.size() > 1) {
       return false;
     }
     for (KeyRange keyRange : orRanges) {
       // Special case for single trailing IS NULL. We cannot consider this as a point key because
       // we strip trailing nulls when we form the key.
       if (!keyRange.isSingleKey() || (i == lastIndex && keyRange == KeyRange.IS_NULL_RANGE)) {
         return false;
       }
     }
   }
   return true;
 }
Ejemplo n.º 7
0
 private boolean explainSkipScan(StringBuilder buf) {
   ScanRanges scanRanges = context.getScanRanges();
   if (scanRanges.isPointLookup()) {
     int keyCount = scanRanges.getPointLookupCount();
     buf.append("POINT LOOKUP ON " + keyCount + " KEY" + (keyCount > 1 ? "S " : " "));
   } else if (scanRanges.useSkipScanFilter()) {
     buf.append("SKIP SCAN ");
     int count = 1;
     boolean hasRanges = false;
     for (List<KeyRange> ranges : scanRanges.getRanges()) {
       count *= ranges.size();
       for (KeyRange range : ranges) {
         hasRanges |= !range.isSingleKey();
       }
     }
     buf.append("ON ");
     buf.append(count);
     buf.append(hasRanges ? " RANGE" : " KEY");
     buf.append(count > 1 ? "S " : " ");
   } else {
     buf.append("RANGE SCAN ");
   }
   return scanRanges.useSkipScanFilter();
 }
Ejemplo n.º 8
0
  /**
   * Converts a partially qualified KeyRange into a KeyRange with a inclusive lower bound and an
   * exclusive upper bound, widening as necessary.
   */
  public static KeyRange convertToInclusiveExclusiveRange(
      KeyRange partialRange, RowKeySchema schema, ImmutableBytesWritable ptr) {
    // Ensure minMaxRange is lower inclusive and upper exclusive, as that's
    // what we need to intersect against for the HBase scan.
    byte[] lowerRange = partialRange.getLowerRange();
    if (!partialRange.lowerUnbound()) {
      if (!partialRange.isLowerInclusive()) {
        lowerRange = ScanUtil.nextKey(lowerRange, schema, ptr);
      }
    }

    byte[] upperRange = partialRange.getUpperRange();
    if (!partialRange.upperUnbound()) {
      if (partialRange.isUpperInclusive()) {
        upperRange = ScanUtil.nextKey(upperRange, schema, ptr);
      }
    }
    if (partialRange.getLowerRange() != lowerRange || partialRange.getUpperRange() != upperRange) {
      partialRange = KeyRange.getKeyRange(lowerRange, upperRange);
    }
    return partialRange;
  }
Ejemplo n.º 9
0
  public static TimeRange getDescTimeRange(
      KeyRange lowestKeyRange, KeyRange highestKeyRange, Field f) throws IOException {
    boolean lowerUnbound = lowestKeyRange.lowerUnbound();
    boolean lowerInclusive = lowestKeyRange.isLowerInclusive();
    boolean upperUnbound = highestKeyRange.upperUnbound();
    boolean upperInclusive = highestKeyRange.isUpperInclusive();

    long low =
        lowerUnbound
            ? -1
            : f.getDataType()
                .getCodec()
                .decodeLong(lowestKeyRange.getLowerRange(), 0, SortOrder.DESC);
    long high =
        upperUnbound
            ? -1
            : f.getDataType()
                .getCodec()
                .decodeLong(highestKeyRange.getUpperRange(), 0, SortOrder.DESC);
    long newHigh;
    long newLow;
    if (!lowerUnbound && !upperUnbound) {
      newHigh = lowerInclusive ? safelyIncrement(low) : low;
      newLow = upperInclusive ? high : safelyIncrement(high);
      return new TimeRange(newLow, newHigh);
    } else if (!lowerUnbound && upperUnbound) {
      newHigh = lowerInclusive ? safelyIncrement(low) : low;
      newLow = 0;
      return new TimeRange(newLow, newHigh);
    } else if (lowerUnbound && !upperUnbound) {
      newLow = upperInclusive ? high : safelyIncrement(high);
      newHigh = HConstants.LATEST_TIMESTAMP;
      return new TimeRange(newLow, newHigh);
    } else {
      newLow = 0;
      newHigh = HConstants.LATEST_TIMESTAMP;
      return new TimeRange(newLow, newHigh);
    }
  }
Ejemplo n.º 10
0
 private static TimeRange getAscTimeRange(KeyRange lowestRange, KeyRange highestRange, Field f)
     throws IOException {
   long low;
   long high;
   if (lowestRange.lowerUnbound()) {
     low = 0;
   } else {
     long lowerRange =
         f.getDataType().getCodec().decodeLong(lowestRange.getLowerRange(), 0, SortOrder.ASC);
     low = lowestRange.isLowerInclusive() ? lowerRange : safelyIncrement(lowerRange);
   }
   if (highestRange.upperUnbound()) {
     high = HConstants.LATEST_TIMESTAMP;
   } else {
     long upperRange =
         f.getDataType().getCodec().decodeLong(highestRange.getUpperRange(), 0, SortOrder.ASC);
     if (highestRange.isUpperInclusive()) {
       high = safelyIncrement(upperRange);
     } else {
       high = upperRange;
     }
   }
   return new TimeRange(low, high);
 }
Ejemplo n.º 11
0
  public static int setKey(
      RowKeySchema schema,
      List<List<KeyRange>> slots,
      int[] slotSpan,
      int[] position,
      Bound bound,
      byte[] key,
      int byteOffset,
      int slotStartIndex,
      int slotEndIndex,
      int schemaStartIndex) {
    int offset = byteOffset;
    boolean lastInclusiveUpperSingleKey = false;
    boolean anyInclusiveUpperRangeKey = false;
    // The index used for slots should be incremented by 1,
    // but the index for the field it represents in the schema
    // should be incremented by 1 + value in the current slotSpan index
    // slotSpan stores the number of columns beyond one that the range spans
    Field field = null;
    int i = slotStartIndex, fieldIndex = ScanUtil.getRowKeyPosition(slotSpan, slotStartIndex);
    for (i = slotStartIndex; i < slotEndIndex; i++) {
      // Build up the key by appending the bound of each key range
      // from the current position of each slot.
      KeyRange range = slots.get(i).get(position[i]);
      // Use last slot in a multi-span column to determine if fixed width
      field = schema.getField(fieldIndex + slotSpan[i]);
      boolean isFixedWidth = field.getDataType().isFixedWidth();
      fieldIndex += slotSpan[i] + 1;
      /*
       * If the current slot is unbound then stop if:
       * 1) setting the upper bound. There's no value in
       *    continuing because nothing will be filtered.
       * 2) setting the lower bound when the type is fixed length
       *    for the same reason. However, if the type is variable width
       *    continue building the key because null values will be filtered
       *    since our separator byte will be appended and incremented.
       */
      if (range.isUnbound(bound) && (bound == Bound.UPPER || isFixedWidth)) {
        break;
      }
      byte[] bytes = range.getRange(bound);
      System.arraycopy(bytes, 0, key, offset, bytes.length);
      offset += bytes.length;
      /*
       * We must add a terminator to a variable length key even for the last PK column if
       * the lower key is non inclusive or the upper key is inclusive. Otherwise, we'd be
       * incrementing the key value itself, and thus bumping it up too much.
       */
      boolean inclusiveUpper = range.isInclusive(bound) && bound == Bound.UPPER;
      boolean exclusiveLower = !range.isInclusive(bound) && bound == Bound.LOWER;
      // If we are setting the upper bound of using inclusive single key, we remember
      // to increment the key if we exit the loop after this iteration.
      //
      // We remember to increment the last slot if we are setting the upper bound with an
      // inclusive range key.
      //
      // We cannot combine the two flags together in case for single-inclusive key followed
      // by the range-exclusive key. In that case, we do not need to increment the end at the
      // end. But if we combine the two flag, the single inclusive key in the middle of the
      // key slots would cause the flag to become true.
      lastInclusiveUpperSingleKey = range.isSingleKey() && inclusiveUpper;
      anyInclusiveUpperRangeKey |= !range.isSingleKey() && inclusiveUpper;
      // A null or empty byte array is always represented as a zero byte
      byte sepByte =
          SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), bytes.length == 0, field);

      if (!isFixedWidth
          && (fieldIndex < schema.getMaxFields()
              || inclusiveUpper
              || exclusiveLower
              || sepByte == QueryConstants.DESC_SEPARATOR_BYTE)) {
        key[offset++] = sepByte;
        // Set lastInclusiveUpperSingleKey back to false if this is the last pk column
        // as we don't want to increment the null byte in this case
        lastInclusiveUpperSingleKey &= i < schema.getMaxFields() - 1;
      }
      // If we are setting the lower bound with an exclusive range key, we need to bump the
      // slot up for each key part. For an upper bound, we bump up an inclusive key, but
      // only after the last key part.
      if (exclusiveLower) {
        if (!ByteUtil.nextKey(key, offset)) {
          // Special case for not being able to increment.
          // In this case we return a negative byteOffset to
          // remove this part from the key being formed. Since the
          // key has overflowed, this means that we should not
          // have an end key specified.
          return -byteOffset;
        }
        // We're filtering on values being non null here, but we still need the 0xFF
        // terminator, since DESC keys ignore the last byte as it's expected to be
        // the terminator. Without this, we'd ignore the separator byte that was
        // just added and incremented.
        if (!isFixedWidth
            && bytes.length == 0
            && SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, field)
                == QueryConstants.DESC_SEPARATOR_BYTE) {
          key[offset++] = QueryConstants.DESC_SEPARATOR_BYTE;
        }
      }
    }
    if (lastInclusiveUpperSingleKey || anyInclusiveUpperRangeKey) {
      if (!ByteUtil.nextKey(key, offset)) {
        // Special case for not being able to increment.
        // In this case we return a negative byteOffset to
        // remove this part from the key being formed. Since the
        // key has overflowed, this means that we should not
        // have an end key specified.
        return -byteOffset;
      }
    }
    // Remove trailing separator bytes, since the columns may have been added
    // after the table has data, in which case there won't be a separator
    // byte.
    if (bound == Bound.LOWER) {
      while (--i >= schemaStartIndex
          && offset > byteOffset
          && !(field = schema.getField(--fieldIndex)).getDataType().isFixedWidth()
          && field.getSortOrder() == SortOrder.ASC
          && key[offset - 1] == QueryConstants.SEPARATOR_BYTE) {
        offset--;
        fieldIndex -= slotSpan[i];
      }
    }
    return offset - byteOffset;
  }
Ejemplo n.º 12
0
 public void initializeScan(Scan scan) {
   scan.setStartRow(scanRange.getLowerRange());
   scan.setStopRow(scanRange.getUpperRange());
 }
Ejemplo n.º 13
0
  public static ScanRanges create(
      RowKeySchema schema,
      List<List<KeyRange>> ranges,
      int[] slotSpan,
      KeyRange minMaxRange,
      Integer nBuckets,
      boolean useSkipScan,
      int rowTimestampColIndex) {
    int offset = nBuckets == null ? 0 : SaltingUtil.NUM_SALTING_BYTES;
    int nSlots = ranges.size();
    if (nSlots == offset && minMaxRange == KeyRange.EVERYTHING_RANGE) {
      return EVERYTHING;
    } else if (minMaxRange == KeyRange.EMPTY_RANGE
        || (nSlots == 1 + offset
            && ranges.get(offset).size() == 1
            && ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) {
      return NOTHING;
    }
    TimeRange rowTimestampRange = getRowTimestampColumnRange(ranges, schema, rowTimestampColIndex);
    boolean isPointLookup = isPointLookup(schema, ranges, slotSpan, useSkipScan);
    if (isPointLookup) {
      // TODO: consider keeping original to use for serialization as it would be smaller?
      List<byte[]> keys = ScanRanges.getPointKeys(ranges, slotSpan, schema, nBuckets);
      List<KeyRange> keyRanges = Lists.newArrayListWithExpectedSize(keys.size());
      KeyRange unsaltedMinMaxRange = minMaxRange;
      if (nBuckets != null && minMaxRange != KeyRange.EVERYTHING_RANGE) {
        unsaltedMinMaxRange =
            KeyRange.getKeyRange(
                stripPrefix(minMaxRange.getLowerRange(), offset),
                minMaxRange.lowerUnbound(),
                stripPrefix(minMaxRange.getUpperRange(), offset),
                minMaxRange.upperUnbound());
      }
      // We have full keys here, so use field from our varbinary schema
      BytesComparator comparator = ScanUtil.getComparator(SchemaUtil.VAR_BINARY_SCHEMA.getField(0));
      for (byte[] key : keys) {
        // Filter now based on unsalted minMaxRange and ignore the point key salt byte
        if (unsaltedMinMaxRange.compareLowerToUpperBound(
                    key, offset, key.length - offset, true, comparator)
                <= 0
            && unsaltedMinMaxRange.compareUpperToLowerBound(
                    key, offset, key.length - offset, true, comparator)
                >= 0) {
          keyRanges.add(KeyRange.getKeyRange(key));
        }
      }
      ranges = Collections.singletonList(keyRanges);
      useSkipScan = keyRanges.size() > 1;
      // Treat as binary if descending because we've got a separator byte at the end
      // which is not part of the value.
      if (keys.size() > 1
          || SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, schema.getField(0))
              == QueryConstants.DESC_SEPARATOR_BYTE) {
        schema = SchemaUtil.VAR_BINARY_SCHEMA;
        slotSpan = ScanUtil.SINGLE_COLUMN_SLOT_SPAN;
      } else {
        // Keep original schema and don't use skip scan as it's not necessary
        // when there's a single key.
        slotSpan = new int[] {schema.getMaxFields() - 1};
      }
    }
    List<List<KeyRange>> sortedRanges = Lists.newArrayListWithExpectedSize(ranges.size());
    for (int i = 0; i < ranges.size(); i++) {
      List<KeyRange> sorted = Lists.newArrayList(ranges.get(i));
      Collections.sort(sorted, KeyRange.COMPARATOR);
      sortedRanges.add(ImmutableList.copyOf(sorted));
    }

    // Don't set minMaxRange for point lookup because it causes issues during intersect
    // by going across region boundaries
    KeyRange scanRange = KeyRange.EVERYTHING_RANGE;
    // if (!isPointLookup && (nBuckets == null || !useSkipScanFilter)) {
    // if (! ( isPointLookup || (nBuckets != null && useSkipScanFilter) ) ) {
    // if (nBuckets == null || (nBuckets != null && (!isPointLookup || !useSkipScanFilter))) {
    if (nBuckets == null || !isPointLookup || !useSkipScan) {
      byte[] minKey = ScanUtil.getMinKey(schema, sortedRanges, slotSpan);
      byte[] maxKey = ScanUtil.getMaxKey(schema, sortedRanges, slotSpan);
      // If the maxKey has crossed the salt byte boundary, then we do not
      // have anything to filter at the upper end of the range
      if (ScanUtil.crossesPrefixBoundary(maxKey, ScanUtil.getPrefix(minKey, offset), offset)) {
        maxKey = KeyRange.UNBOUND;
      }
      // We won't filter anything at the low end of the range if we just have the salt byte
      if (minKey.length <= offset) {
        minKey = KeyRange.UNBOUND;
      }
      scanRange = KeyRange.getKeyRange(minKey, maxKey);
    }
    if (minMaxRange != KeyRange.EVERYTHING_RANGE) {
      minMaxRange =
          ScanUtil.convertToInclusiveExclusiveRange(
              minMaxRange, schema, new ImmutableBytesWritable());
      scanRange = scanRange.intersect(minMaxRange);
    }

    if (scanRange == KeyRange.EMPTY_RANGE) {
      return NOTHING;
    }
    return new ScanRanges(
        schema,
        slotSpan,
        sortedRanges,
        scanRange,
        minMaxRange,
        useSkipScan,
        isPointLookup,
        nBuckets,
        rowTimestampRange);
  }