Example #1
0
 // For testing
 public static ScanRanges createSingleSpan(RowKeySchema schema, List<List<KeyRange>> ranges) {
   return create(
       schema,
       ranges,
       ScanUtil.getDefaultSlotSpans(ranges.size()),
       KeyRange.EVERYTHING_RANGE,
       null,
       true,
       -1);
 }
Example #2
0
  /**
   * Converts a partially qualified KeyRange into a KeyRange with a inclusive lower bound and an
   * exclusive upper bound, widening as necessary.
   */
  public static KeyRange convertToInclusiveExclusiveRange(
      KeyRange partialRange, RowKeySchema schema, ImmutableBytesWritable ptr) {
    // Ensure minMaxRange is lower inclusive and upper exclusive, as that's
    // what we need to intersect against for the HBase scan.
    byte[] lowerRange = partialRange.getLowerRange();
    if (!partialRange.lowerUnbound()) {
      if (!partialRange.isLowerInclusive()) {
        lowerRange = ScanUtil.nextKey(lowerRange, schema, ptr);
      }
    }

    byte[] upperRange = partialRange.getUpperRange();
    if (!partialRange.upperUnbound()) {
      if (partialRange.isUpperInclusive()) {
        upperRange = ScanUtil.nextKey(upperRange, schema, ptr);
      }
    }
    if (partialRange.getLowerRange() != lowerRange || partialRange.getUpperRange() != upperRange) {
      partialRange = KeyRange.getKeyRange(lowerRange, upperRange);
    }
    return partialRange;
  }
 private static Collection<?> foreach(
     KeyRange[][] ranges, int[] widths, KeyRange[] expectedSplits) {
   RowKeySchema schema = buildSchema(widths);
   List<List<KeyRange>> slots = Lists.transform(Lists.newArrayList(ranges), ARRAY_TO_LIST);
   SkipScanFilter filter = new SkipScanFilter(slots, schema);
   // Always set start and stop key to max to verify we are using the information in skipscan
   // filter over the scan's KMIN and KMAX.
   Scan scan = new Scan().setFilter(filter);
   ScanRanges scanRanges =
       ScanRanges.create(schema, slots, ScanUtil.getDefaultSlotSpans(ranges.length));
   List<Object> ret = Lists.newArrayList();
   ret.add(new Object[] {scan, scanRanges, Arrays.<KeyRange>asList(expectedSplits)});
   return ret;
 }
Example #4
0
 private static List<byte[]> getPointKeys(
     List<List<KeyRange>> ranges, int[] slotSpan, RowKeySchema schema, Integer bucketNum) {
   if (ranges == null || ranges.isEmpty()) {
     return Collections.emptyList();
   }
   boolean isSalted = bucketNum != null;
   int count = 1;
   int offset = isSalted ? 1 : 0;
   // Skip salt byte range in the first position if salted
   for (int i = offset; i < ranges.size(); i++) {
     count *= ranges.get(i).size();
   }
   List<byte[]> keys = Lists.newArrayListWithExpectedSize(count);
   int[] position = new int[ranges.size()];
   int maxKeyLength = SchemaUtil.getMaxKeyLength(schema, ranges);
   int length;
   byte[] key = new byte[maxKeyLength];
   do {
     length =
         ScanUtil.setKey(
             schema,
             ranges,
             slotSpan,
             position,
             Bound.LOWER,
             key,
             offset,
             offset,
             ranges.size(),
             offset);
     if (isSalted) {
       key[0] = SaltingUtil.getSaltingByte(key, offset, length, bucketNum);
     }
     keys.add(Arrays.copyOf(key, length + offset));
   } while (incrementKey(ranges, position));
   return keys;
 }
Example #5
0
 /**
  * Perform a binary lookup on the list of KeyRange for the tightest slot such that the slotBound
  * of the current slot is higher or equal than the slotBound of our range.
  *
  * @return the index of the slot whose slot bound equals or are the tightest one that is smaller
  *     than rangeBound of range, or slots.length if no bound can be found.
  */
 public static int searchClosestKeyRangeWithUpperHigherThanPtr(
     List<KeyRange> slots, ImmutableBytesWritable ptr, int lower, Field field) {
   int upper = slots.size() - 1;
   int mid;
   BytesComparator comparator =
       ScanUtil.getComparator(field.getDataType().isFixedWidth(), field.getSortOrder());
   while (lower <= upper) {
     mid = (lower + upper) / 2;
     int cmp = slots.get(mid).compareUpperToLowerBound(ptr, true, comparator);
     if (cmp < 0) {
       lower = mid + 1;
     } else if (cmp > 0) {
       upper = mid - 1;
     } else {
       return mid;
     }
   }
   mid = (lower + upper) / 2;
   if (mid == 0 && slots.get(mid).compareUpperToLowerBound(ptr, true, comparator) > 0) {
     return mid;
   } else {
     return ++mid;
   }
 }
Example #6
0
  public static int setKey(
      RowKeySchema schema,
      List<List<KeyRange>> slots,
      int[] slotSpan,
      int[] position,
      Bound bound,
      byte[] key,
      int byteOffset,
      int slotStartIndex,
      int slotEndIndex,
      int schemaStartIndex) {
    int offset = byteOffset;
    boolean lastInclusiveUpperSingleKey = false;
    boolean anyInclusiveUpperRangeKey = false;
    // The index used for slots should be incremented by 1,
    // but the index for the field it represents in the schema
    // should be incremented by 1 + value in the current slotSpan index
    // slotSpan stores the number of columns beyond one that the range spans
    Field field = null;
    int i = slotStartIndex, fieldIndex = ScanUtil.getRowKeyPosition(slotSpan, slotStartIndex);
    for (i = slotStartIndex; i < slotEndIndex; i++) {
      // Build up the key by appending the bound of each key range
      // from the current position of each slot.
      KeyRange range = slots.get(i).get(position[i]);
      // Use last slot in a multi-span column to determine if fixed width
      field = schema.getField(fieldIndex + slotSpan[i]);
      boolean isFixedWidth = field.getDataType().isFixedWidth();
      fieldIndex += slotSpan[i] + 1;
      /*
       * If the current slot is unbound then stop if:
       * 1) setting the upper bound. There's no value in
       *    continuing because nothing will be filtered.
       * 2) setting the lower bound when the type is fixed length
       *    for the same reason. However, if the type is variable width
       *    continue building the key because null values will be filtered
       *    since our separator byte will be appended and incremented.
       */
      if (range.isUnbound(bound) && (bound == Bound.UPPER || isFixedWidth)) {
        break;
      }
      byte[] bytes = range.getRange(bound);
      System.arraycopy(bytes, 0, key, offset, bytes.length);
      offset += bytes.length;
      /*
       * We must add a terminator to a variable length key even for the last PK column if
       * the lower key is non inclusive or the upper key is inclusive. Otherwise, we'd be
       * incrementing the key value itself, and thus bumping it up too much.
       */
      boolean inclusiveUpper = range.isInclusive(bound) && bound == Bound.UPPER;
      boolean exclusiveLower = !range.isInclusive(bound) && bound == Bound.LOWER;
      // If we are setting the upper bound of using inclusive single key, we remember
      // to increment the key if we exit the loop after this iteration.
      //
      // We remember to increment the last slot if we are setting the upper bound with an
      // inclusive range key.
      //
      // We cannot combine the two flags together in case for single-inclusive key followed
      // by the range-exclusive key. In that case, we do not need to increment the end at the
      // end. But if we combine the two flag, the single inclusive key in the middle of the
      // key slots would cause the flag to become true.
      lastInclusiveUpperSingleKey = range.isSingleKey() && inclusiveUpper;
      anyInclusiveUpperRangeKey |= !range.isSingleKey() && inclusiveUpper;
      // A null or empty byte array is always represented as a zero byte
      byte sepByte =
          SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), bytes.length == 0, field);

      if (!isFixedWidth
          && (fieldIndex < schema.getMaxFields()
              || inclusiveUpper
              || exclusiveLower
              || sepByte == QueryConstants.DESC_SEPARATOR_BYTE)) {
        key[offset++] = sepByte;
        // Set lastInclusiveUpperSingleKey back to false if this is the last pk column
        // as we don't want to increment the null byte in this case
        lastInclusiveUpperSingleKey &= i < schema.getMaxFields() - 1;
      }
      // If we are setting the lower bound with an exclusive range key, we need to bump the
      // slot up for each key part. For an upper bound, we bump up an inclusive key, but
      // only after the last key part.
      if (exclusiveLower) {
        if (!ByteUtil.nextKey(key, offset)) {
          // Special case for not being able to increment.
          // In this case we return a negative byteOffset to
          // remove this part from the key being formed. Since the
          // key has overflowed, this means that we should not
          // have an end key specified.
          return -byteOffset;
        }
        // We're filtering on values being non null here, but we still need the 0xFF
        // terminator, since DESC keys ignore the last byte as it's expected to be
        // the terminator. Without this, we'd ignore the separator byte that was
        // just added and incremented.
        if (!isFixedWidth
            && bytes.length == 0
            && SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, field)
                == QueryConstants.DESC_SEPARATOR_BYTE) {
          key[offset++] = QueryConstants.DESC_SEPARATOR_BYTE;
        }
      }
    }
    if (lastInclusiveUpperSingleKey || anyInclusiveUpperRangeKey) {
      if (!ByteUtil.nextKey(key, offset)) {
        // Special case for not being able to increment.
        // In this case we return a negative byteOffset to
        // remove this part from the key being formed. Since the
        // key has overflowed, this means that we should not
        // have an end key specified.
        return -byteOffset;
      }
    }
    // Remove trailing separator bytes, since the columns may have been added
    // after the table has data, in which case there won't be a separator
    // byte.
    if (bound == Bound.LOWER) {
      while (--i >= schemaStartIndex
          && offset > byteOffset
          && !(field = schema.getField(--fieldIndex)).getDataType().isFixedWidth()
          && field.getSortOrder() == SortOrder.ASC
          && key[offset - 1] == QueryConstants.SEPARATOR_BYTE) {
        offset--;
        fieldIndex -= slotSpan[i];
      }
    }
    return offset - byteOffset;
  }
Example #7
0
  public static boolean intersectScanRange(
      Scan scan, byte[] startKey, byte[] stopKey, boolean useSkipScan) {
    boolean mayHaveRows = false;
    int offset = 0;
    if (ScanUtil.isLocalIndex(scan)) {
      offset = startKey.length != 0 ? startKey.length : stopKey.length;
    }
    byte[] existingStartKey = scan.getStartRow();
    byte[] existingStopKey = scan.getStopRow();
    if (existingStartKey.length > 0) {
      if (startKey.length == 0 || Bytes.compareTo(existingStartKey, startKey) > 0) {
        startKey = existingStartKey;
      }
    } else {
      mayHaveRows = true;
    }
    if (existingStopKey.length > 0) {
      if (stopKey.length == 0 || Bytes.compareTo(existingStopKey, stopKey) < 0) {
        stopKey = existingStopKey;
      }
    } else {
      mayHaveRows = true;
    }
    scan.setStartRow(startKey);
    scan.setStopRow(stopKey);
    if (offset > 0 && useSkipScan) {
      byte[] temp = null;
      if (startKey.length != 0) {
        temp = new byte[startKey.length - offset];
        System.arraycopy(startKey, offset, temp, 0, startKey.length - offset);
        startKey = temp;
      }
      if (stopKey.length != 0) {
        temp = new byte[stopKey.length - offset];
        System.arraycopy(stopKey, offset, temp, 0, stopKey.length - offset);
        stopKey = temp;
      }
    }
    mayHaveRows = mayHaveRows || Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) < 0;

    // If the scan is using skip scan filter, intersect and replace the filter.
    if (mayHaveRows && useSkipScan) {
      Filter filter = scan.getFilter();
      if (filter instanceof SkipScanFilter) {
        SkipScanFilter oldFilter = (SkipScanFilter) filter;
        SkipScanFilter newFilter = oldFilter.intersect(startKey, stopKey);
        if (newFilter == null) {
          return false;
        }
        // Intersect found: replace skip scan with intersected one
        scan.setFilter(newFilter);
      } else if (filter instanceof FilterList) {
        FilterList oldList = (FilterList) filter;
        FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        for (Filter f : oldList.getFilters()) {
          if (f instanceof SkipScanFilter) {
            SkipScanFilter newFilter = ((SkipScanFilter) f).intersect(startKey, stopKey);
            if (newFilter == null) {
              return false;
            }
            newList.addFilter(newFilter);
          } else {
            newList.addFilter(f);
          }
        }
        scan.setFilter(newList);
      }
    }
    return mayHaveRows;
  }
Example #8
0
  public ServerCache addServerCache(
      ScanRanges keyRanges,
      final ImmutableBytesWritable cachePtr,
      final byte[] txState,
      final ServerCacheFactory cacheFactory,
      final TableRef cacheUsingTableRef)
      throws SQLException {
    ConnectionQueryServices services = connection.getQueryServices();
    MemoryChunk chunk = services.getMemoryManager().allocate(cachePtr.getLength());
    List<Closeable> closeables = new ArrayList<Closeable>();
    closeables.add(chunk);
    ServerCache hashCacheSpec = null;
    SQLException firstException = null;
    final byte[] cacheId = generateId();
    /** Execute EndPoint in parallel on each server to send compressed hash cache */
    // TODO: generalize and package as a per region server EndPoint caller
    // (ideally this would be functionality provided by the coprocessor framework)
    boolean success = false;
    ExecutorService executor = services.getExecutor();
    List<Future<Boolean>> futures = Collections.emptyList();
    try {
      final PTable cacheUsingTable = cacheUsingTableRef.getTable();
      List<HRegionLocation> locations =
          services.getAllTableRegions(cacheUsingTable.getPhysicalName().getBytes());
      int nRegions = locations.size();
      // Size these based on worst case
      futures = new ArrayList<Future<Boolean>>(nRegions);
      Set<HRegionLocation> servers = new HashSet<HRegionLocation>(nRegions);
      for (HRegionLocation entry : locations) {
        // Keep track of servers we've sent to and only send once
        byte[] regionStartKey = entry.getRegionInfo().getStartKey();
        byte[] regionEndKey = entry.getRegionInfo().getEndKey();
        if (!servers.contains(entry)
            && keyRanges.intersects(
                regionStartKey,
                regionEndKey,
                cacheUsingTable.getIndexType() == IndexType.LOCAL
                    ? ScanUtil.getRowKeyOffset(regionStartKey, regionEndKey)
                    : 0,
                true)) {
          // Call RPC once per server
          servers.add(entry);
          if (LOG.isDebugEnabled()) {
            LOG.debug(
                addCustomAnnotations("Adding cache entry to be sent for " + entry, connection));
          }
          final byte[] key = entry.getRegionInfo().getStartKey();
          final HTableInterface htable =
              services.getTable(cacheUsingTableRef.getTable().getPhysicalName().getBytes());
          closeables.add(htable);
          futures.add(
              executor.submit(
                  new JobCallable<Boolean>() {

                    @Override
                    public Boolean call() throws Exception {
                      final Map<byte[], AddServerCacheResponse> results;
                      try {
                        results =
                            htable.coprocessorService(
                                ServerCachingService.class,
                                key,
                                key,
                                new Batch.Call<ServerCachingService, AddServerCacheResponse>() {
                                  @Override
                                  public AddServerCacheResponse call(ServerCachingService instance)
                                      throws IOException {
                                    ServerRpcController controller = new ServerRpcController();
                                    BlockingRpcCallback<AddServerCacheResponse> rpcCallback =
                                        new BlockingRpcCallback<AddServerCacheResponse>();
                                    AddServerCacheRequest.Builder builder =
                                        AddServerCacheRequest.newBuilder();
                                    if (connection.getTenantId() != null) {
                                      try {
                                        byte[] tenantIdBytes =
                                            ScanUtil.getTenantIdBytes(
                                                cacheUsingTable.getRowKeySchema(),
                                                cacheUsingTable.getBucketNum() != null,
                                                connection.getTenantId(),
                                                cacheUsingTable.isMultiTenant());
                                        builder.setTenantId(ByteStringer.wrap(tenantIdBytes));
                                      } catch (SQLException e) {
                                        new IOException(e);
                                      }
                                    }
                                    builder.setCacheId(ByteStringer.wrap(cacheId));
                                    builder.setCachePtr(
                                        org.apache.phoenix.protobuf.ProtobufUtil.toProto(cachePtr));
                                    ServerCacheFactoryProtos.ServerCacheFactory.Builder
                                        svrCacheFactoryBuider =
                                            ServerCacheFactoryProtos.ServerCacheFactory
                                                .newBuilder();
                                    svrCacheFactoryBuider.setClassName(
                                        cacheFactory.getClass().getName());
                                    builder.setCacheFactory(svrCacheFactoryBuider.build());
                                    builder.setTxState(HBaseZeroCopyByteString.wrap(txState));
                                    instance.addServerCache(
                                        controller, builder.build(), rpcCallback);
                                    if (controller.getFailedOn() != null) {
                                      throw controller.getFailedOn();
                                    }
                                    return rpcCallback.get();
                                  }
                                });
                      } catch (Throwable t) {
                        throw new Exception(t);
                      }
                      if (results != null && results.size() == 1) {
                        return results.values().iterator().next().getReturn();
                      }
                      return false;
                    }

                    /**
                     * Defines the grouping for round robin behavior. All threads spawned to process
                     * this scan will be grouped together and time sliced with other simultaneously
                     * executing parallel scans.
                     */
                    @Override
                    public Object getJobId() {
                      return ServerCacheClient.this;
                    }

                    @Override
                    public TaskExecutionMetricsHolder getTaskExecutionMetric() {
                      return NO_OP_INSTANCE;
                    }
                  }));
        } else {
          if (LOG.isDebugEnabled()) {
            LOG.debug(
                addCustomAnnotations(
                    "NOT adding cache entry to be sent for "
                        + entry
                        + " since one already exists for that entry",
                    connection));
          }
        }
      }

      hashCacheSpec = new ServerCache(cacheId, servers, cachePtr.getLength());
      // Execute in parallel
      int timeoutMs =
          services
              .getProps()
              .getInt(
                  QueryServices.THREAD_TIMEOUT_MS_ATTRIB,
                  QueryServicesOptions.DEFAULT_THREAD_TIMEOUT_MS);
      for (Future<Boolean> future : futures) {
        future.get(timeoutMs, TimeUnit.MILLISECONDS);
      }

      cacheUsingTableRefMap.put(Bytes.mapKey(cacheId), cacheUsingTableRef);
      success = true;
    } catch (SQLException e) {
      firstException = e;
    } catch (Exception e) {
      firstException = new SQLException(e);
    } finally {
      try {
        if (!success) {
          SQLCloseables.closeAllQuietly(Collections.singletonList(hashCacheSpec));
          for (Future<Boolean> future : futures) {
            future.cancel(true);
          }
        }
      } finally {
        try {
          Closeables.closeAll(closeables);
        } catch (IOException e) {
          if (firstException == null) {
            firstException = new SQLException(e);
          }
        } finally {
          if (firstException != null) {
            throw firstException;
          }
        }
      }
    }
    if (LOG.isDebugEnabled()) {
      LOG.debug(
          addCustomAnnotations("Cache " + cacheId + " successfully added to servers.", connection));
    }
    return hashCacheSpec;
  }
  @Override
  protected RegionScanner doPostScannerOpen(
      final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s)
      throws IOException {
    byte[] isUngroupedAgg = scan.getAttribute(BaseScannerRegionObserver.UNGROUPED_AGG);
    if (isUngroupedAgg == null) {
      return s;
    }

    final ScanProjector p = ScanProjector.deserializeProjectorFromScan(scan);
    final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan);
    RegionScanner theScanner = s;
    if (p != null || j != null) {
      theScanner =
          new HashJoinRegionScanner(s, p, j, ScanUtil.getTenantId(scan), c.getEnvironment());
    }
    final RegionScanner innerScanner = theScanner;

    byte[] indexUUID = scan.getAttribute(PhoenixIndexCodec.INDEX_UUID);
    PTable projectedTable = null;
    List<Expression> selectExpressions = null;
    byte[] upsertSelectTable = scan.getAttribute(BaseScannerRegionObserver.UPSERT_SELECT_TABLE);
    boolean isUpsert = false;
    boolean isDelete = false;
    byte[] deleteCQ = null;
    byte[] deleteCF = null;
    byte[][] values = null;
    byte[] emptyCF = null;
    ImmutableBytesWritable ptr = null;
    if (upsertSelectTable != null) {
      isUpsert = true;
      projectedTable = deserializeTable(upsertSelectTable);
      selectExpressions =
          deserializeExpressions(scan.getAttribute(BaseScannerRegionObserver.UPSERT_SELECT_EXPRS));
      values = new byte[projectedTable.getPKColumns().size()][];
      ptr = new ImmutableBytesWritable();
    } else {
      byte[] isDeleteAgg = scan.getAttribute(BaseScannerRegionObserver.DELETE_AGG);
      isDelete = isDeleteAgg != null && Bytes.compareTo(PDataType.TRUE_BYTES, isDeleteAgg) == 0;
      if (!isDelete) {
        deleteCF = scan.getAttribute(BaseScannerRegionObserver.DELETE_CF);
        deleteCQ = scan.getAttribute(BaseScannerRegionObserver.DELETE_CQ);
      }
      emptyCF = scan.getAttribute(BaseScannerRegionObserver.EMPTY_CF);
    }

    int batchSize = 0;
    long ts = scan.getTimeRange().getMax();
    HRegion region = c.getEnvironment().getRegion();
    List<Mutation> mutations = Collections.emptyList();
    if (isDelete || isUpsert || (deleteCQ != null && deleteCF != null) || emptyCF != null) {
      // TODO: size better
      mutations = Lists.newArrayListWithExpectedSize(1024);
      batchSize =
          c.getEnvironment()
              .getConfiguration()
              .getInt(MUTATE_BATCH_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
    }
    Aggregators aggregators =
        ServerAggregators.deserialize(
            scan.getAttribute(BaseScannerRegionObserver.AGGREGATORS),
            c.getEnvironment().getConfiguration());
    Aggregator[] rowAggregators = aggregators.getAggregators();
    boolean hasMore;
    boolean hasAny = false;
    MultiKeyValueTuple result = new MultiKeyValueTuple();
    if (logger.isInfoEnabled()) {
      logger.info("Starting ungrouped coprocessor scan " + scan);
    }
    long rowCount = 0;
    region.startRegionOperation();
    try {
      do {
        List<Cell> results = new ArrayList<Cell>();
        // Results are potentially returned even when the return value of s.next is false
        // since this is an indication of whether or not there are more values after the
        // ones returned
        hasMore = innerScanner.nextRaw(results);
        if (!results.isEmpty()) {
          rowCount++;
          result.setKeyValues(results);
          try {
            if (isDelete) {
              // FIXME: the version of the Delete constructor without the lock args was introduced
              // in 0.94.4, thus if we try to use it here we can no longer use the 0.94.2 version
              // of the client.
              Cell firstKV = results.get(0);
              Delete delete =
                  new Delete(
                      firstKV.getRowArray(), firstKV.getRowOffset(), firstKV.getRowLength(), ts);
              mutations.add(delete);
            } else if (isUpsert) {
              Arrays.fill(values, null);
              int i = 0;
              List<PColumn> projectedColumns = projectedTable.getColumns();
              for (; i < projectedTable.getPKColumns().size(); i++) {
                Expression expression = selectExpressions.get(i);
                if (expression.evaluate(result, ptr)) {
                  values[i] = ptr.copyBytes();
                  // If SortOrder from expression in SELECT doesn't match the
                  // column being projected into then invert the bits.
                  if (expression.getSortOrder() != projectedColumns.get(i).getSortOrder()) {
                    SortOrder.invert(values[i], 0, values[i], 0, values[i].length);
                  }
                }
              }
              projectedTable.newKey(ptr, values);
              PRow row = projectedTable.newRow(kvBuilder, ts, ptr);
              for (; i < projectedColumns.size(); i++) {
                Expression expression = selectExpressions.get(i);
                if (expression.evaluate(result, ptr)) {
                  PColumn column = projectedColumns.get(i);
                  Object value = expression.getDataType().toObject(ptr, column.getSortOrder());
                  // We are guaranteed that the two column will have the same type.
                  if (!column
                      .getDataType()
                      .isSizeCompatible(
                          ptr,
                          value,
                          column.getDataType(),
                          expression.getMaxLength(),
                          expression.getScale(),
                          column.getMaxLength(),
                          column.getScale())) {
                    throw new ValueTypeIncompatibleException(
                        column.getDataType(), column.getMaxLength(), column.getScale());
                  }
                  column
                      .getDataType()
                      .coerceBytes(
                          ptr,
                          value,
                          expression.getDataType(),
                          expression.getMaxLength(),
                          expression.getScale(),
                          expression.getSortOrder(),
                          column.getMaxLength(),
                          column.getScale(),
                          column.getSortOrder());
                  byte[] bytes = ByteUtil.copyKeyBytesIfNecessary(ptr);
                  row.setValue(column, bytes);
                }
              }
              for (Mutation mutation : row.toRowMutations()) {
                mutations.add(mutation);
              }
            } else if (deleteCF != null && deleteCQ != null) {
              // No need to search for delete column, since we project only it
              // if no empty key value is being set
              if (emptyCF == null || result.getValue(deleteCF, deleteCQ) != null) {
                Delete delete =
                    new Delete(
                        results.get(0).getRowArray(),
                        results.get(0).getRowOffset(),
                        results.get(0).getRowLength());
                delete.deleteColumns(deleteCF, deleteCQ, ts);
                mutations.add(delete);
              }
            }
            if (emptyCF != null) {
              /*
               * If we've specified an emptyCF, then we need to insert an empty
               * key value "retroactively" for any key value that is visible at
               * the timestamp that the DDL was issued. Key values that are not
               * visible at this timestamp will not ever be projected up to
               * scans past this timestamp, so don't need to be considered.
               * We insert one empty key value per row per timestamp.
               */
              Set<Long> timeStamps = Sets.newHashSetWithExpectedSize(results.size());
              for (Cell kv : results) {
                long kvts = kv.getTimestamp();
                if (!timeStamps.contains(kvts)) {
                  Put put = new Put(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
                  put.add(
                      emptyCF, QueryConstants.EMPTY_COLUMN_BYTES, kvts, ByteUtil.EMPTY_BYTE_ARRAY);
                  mutations.add(put);
                }
              }
            }
            // Commit in batches based on UPSERT_BATCH_SIZE_ATTRIB in config
            if (!mutations.isEmpty() && batchSize > 0 && mutations.size() % batchSize == 0) {
              commitBatch(region, mutations, indexUUID);
              mutations.clear();
            }
          } catch (ConstraintViolationException e) {
            // Log and ignore in count
            logger.error(
                "Failed to create row in "
                    + region.getRegionNameAsString()
                    + " with values "
                    + SchemaUtil.toString(values),
                e);
            continue;
          }
          aggregators.aggregate(rowAggregators, result);
          hasAny = true;
        }
      } while (hasMore);
    } finally {
      innerScanner.close();
      region.closeRegionOperation();
    }

    if (logger.isInfoEnabled()) {
      logger.info("Finished scanning " + rowCount + " rows for ungrouped coprocessor scan " + scan);
    }

    if (!mutations.isEmpty()) {
      commitBatch(region, mutations, indexUUID);
    }

    final boolean hadAny = hasAny;
    KeyValue keyValue = null;
    if (hadAny) {
      byte[] value = aggregators.toBytes(rowAggregators);
      keyValue =
          KeyValueUtil.newKeyValue(
              UNGROUPED_AGG_ROW_KEY,
              SINGLE_COLUMN_FAMILY,
              SINGLE_COLUMN,
              AGG_TIMESTAMP,
              value,
              0,
              value.length);
    }
    final KeyValue aggKeyValue = keyValue;

    RegionScanner scanner =
        new BaseRegionScanner() {
          private boolean done = !hadAny;

          @Override
          public HRegionInfo getRegionInfo() {
            return innerScanner.getRegionInfo();
          }

          @Override
          public boolean isFilterDone() {
            return done;
          }

          @Override
          public void close() throws IOException {
            innerScanner.close();
          }

          @Override
          public boolean next(List<Cell> results) throws IOException {
            if (done) return false;
            done = true;
            results.add(aggKeyValue);
            return false;
          }

          @Override
          public long getMaxResultSize() {
            return scan.getMaxResultSize();
          }
        };
    return scanner;
  }
Example #10
0
 public int getBoundPkColumnCount() {
   return this.useSkipScanFilter
       ? ScanUtil.getRowKeyPosition(slotSpan, ranges.size())
       : getBoundPkSpan(ranges, slotSpan);
 }
Example #11
0
  public Scan intersectScan(
      Scan scan,
      final byte[] originalStartKey,
      final byte[] originalStopKey,
      final int keyOffset,
      boolean crossesRegionBoundary) {
    byte[] startKey = originalStartKey;
    byte[] stopKey = originalStopKey;
    if (stopKey.length > 0 && Bytes.compareTo(startKey, stopKey) >= 0) {
      return null;
    }
    boolean mayHaveRows = false;
    // Keep the keys as they are if we have a point lookup, as we've already resolved the
    // salt bytes in that case.
    final int scanKeyOffset =
        this.isSalted && !this.isPointLookup ? SaltingUtil.NUM_SALTING_BYTES : 0;
    assert (scanKeyOffset == 0 || keyOffset == 0);
    // Total offset for startKey/stopKey. Either 1 for salted tables or the prefix length
    // of the current region for local indexes. We'll never have a case where a table is
    // both salted and local.
    final int totalKeyOffset = scanKeyOffset + keyOffset;
    byte[] prefixBytes = ByteUtil.EMPTY_BYTE_ARRAY;
    if (totalKeyOffset > 0) {
      prefixBytes = ScanUtil.getPrefix(startKey, totalKeyOffset);
      /*
       * If our startKey to stopKey crosses a region boundary consider everything after the startKey as our scan
       * is always done within a single region. This prevents us from having to prefix the key prior to knowing
       * whether or not there may be an intersection. We can't calculate whether or not we've crossed a region
       * boundary for local indexes, because we don't know the key offset of the next region, but only for the
       * current one (which is the one passed in). If the next prefix happened to be a subset of the previous
       * prefix, then this wouldn't detect that we crossed a region boundary.
       */
      if (crossesRegionBoundary) {
        stopKey = ByteUtil.EMPTY_BYTE_ARRAY;
      }
    }
    int scanStartKeyOffset = scanKeyOffset;
    byte[] scanStartKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStartRow();
    // Compare ignoring key prefix and salt byte
    if (scanStartKey.length > 0) {
      if (startKey.length > 0
          && Bytes.compareTo(
                  scanStartKey,
                  scanKeyOffset,
                  scanStartKey.length - scanKeyOffset,
                  startKey,
                  totalKeyOffset,
                  startKey.length - totalKeyOffset)
              < 0) {
        scanStartKey = startKey;
        scanStartKeyOffset = totalKeyOffset;
      }
    } else {
      scanStartKey = startKey;
      scanStartKeyOffset = totalKeyOffset;
      mayHaveRows = true;
    }
    int scanStopKeyOffset = scanKeyOffset;
    byte[] scanStopKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStopRow();
    if (scanStopKey.length > 0) {
      if (stopKey.length > 0
          && Bytes.compareTo(
                  scanStopKey,
                  scanKeyOffset,
                  scanStopKey.length - scanKeyOffset,
                  stopKey,
                  totalKeyOffset,
                  stopKey.length - totalKeyOffset)
              > 0) {
        scanStopKey = stopKey;
        scanStopKeyOffset = totalKeyOffset;
      }
    } else {
      scanStopKey = stopKey;
      scanStopKeyOffset = totalKeyOffset;
      mayHaveRows = true;
    }
    mayHaveRows =
        mayHaveRows
            || Bytes.compareTo(
                    scanStartKey,
                    scanStartKeyOffset,
                    scanStartKey.length - scanStartKeyOffset,
                    scanStopKey,
                    scanStopKeyOffset,
                    scanStopKey.length - scanStopKeyOffset)
                < 0;

    if (!mayHaveRows) {
      return null;
    }
    if (originalStopKey.length != 0 && scanStopKey.length == 0) {
      scanStopKey = originalStopKey;
    }
    Filter newFilter = null;
    // If the scan is using skip scan filter, intersect and replace the filter.
    if (scan == null || this.useSkipScanFilter()) {
      byte[] skipScanStartKey = scanStartKey;
      byte[] skipScanStopKey = scanStopKey;
      // If we have a keyOffset and we've used the startKey/stopKey that
      // were passed in (which have the prefix) for the above range check,
      // we need to remove the prefix before running our intersect method.
      // TODO: we could use skipScanFilter.setOffset(keyOffset) if both
      // the startKey and stopKey were used above *and* our intersect
      // method honored the skipScanFilter.offset variable.
      if (scanKeyOffset > 0) {
        if (skipScanStartKey != originalStartKey) { // original already has correct salt byte
          skipScanStartKey = replaceSaltByte(skipScanStartKey, prefixBytes);
        }
        if (skipScanStopKey != originalStopKey) {
          skipScanStopKey = replaceSaltByte(skipScanStopKey, prefixBytes);
        }
      } else if (keyOffset > 0) {
        if (skipScanStartKey == originalStartKey) {
          skipScanStartKey = stripPrefix(skipScanStartKey, keyOffset);
        }
        if (skipScanStopKey == originalStopKey) {
          skipScanStopKey = stripPrefix(skipScanStopKey, keyOffset);
        }
      }
      if (scan == null) {
        return filter.hasIntersect(skipScanStartKey, skipScanStopKey) ? HAS_INTERSECTION : null;
      }
      Filter filter = scan.getFilter();
      SkipScanFilter newSkipScanFilter = null;
      if (filter instanceof SkipScanFilter) {
        SkipScanFilter oldSkipScanFilter = (SkipScanFilter) filter;
        newFilter =
            newSkipScanFilter = oldSkipScanFilter.intersect(skipScanStartKey, skipScanStopKey);
        if (newFilter == null) {
          return null;
        }
      } else if (filter instanceof FilterList) {
        FilterList oldList = (FilterList) filter;
        FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        newFilter = newList;
        for (Filter f : oldList.getFilters()) {
          if (f instanceof SkipScanFilter) {
            newSkipScanFilter = ((SkipScanFilter) f).intersect(skipScanStartKey, skipScanStopKey);
            if (newSkipScanFilter == null) {
              return null;
            }
            newList.addFilter(newSkipScanFilter);
          } else {
            newList.addFilter(f);
          }
        }
      }
      // TODO: it seems that our SkipScanFilter or HBase runs into problems if we don't
      // have an enclosing range when we do a point lookup.
      if (isPointLookup) {
        scanStartKey = ScanUtil.getMinKey(schema, newSkipScanFilter.getSlots(), slotSpan);
        scanStopKey = ScanUtil.getMaxKey(schema, newSkipScanFilter.getSlots(), slotSpan);
      }
    }
    if (newFilter == null) {
      newFilter = scan.getFilter();
    }
    Scan newScan = ScanUtil.newScan(scan);
    newScan.setFilter(newFilter);
    // If we have an offset (salted table or local index), we need to make sure to
    // prefix our scan start/stop row by the prefix of the startKey or stopKey that
    // were passed in. Our scan either doesn't have the prefix or has a placeholder
    // for it.
    if (totalKeyOffset > 0) {
      if (scanStartKey != originalStartKey) {
        scanStartKey = prefixKey(scanStartKey, scanKeyOffset, prefixBytes, keyOffset);
      }
      if (scanStopKey != originalStopKey) {
        scanStopKey = prefixKey(scanStopKey, scanKeyOffset, prefixBytes, keyOffset);
      }
    }
    // Don't let the stopRow of the scan go beyond the originalStopKey
    if (originalStopKey.length > 0 && Bytes.compareTo(scanStopKey, originalStopKey) > 0) {
      scanStopKey = originalStopKey;
    }
    if (scanStopKey.length > 0 && Bytes.compareTo(scanStartKey, scanStopKey) >= 0) {
      return null;
    }
    newScan.setAttribute(SCAN_ACTUAL_START_ROW, scanStartKey);
    newScan.setStartRow(scanStartKey);
    newScan.setStopRow(scanStopKey);
    if (keyOffset > 0) {
      newScan.setAttribute(STARTKEY_OFFSET, Bytes.toBytes(keyOffset));
    }

    return newScan;
  }
Example #12
0
  public static ScanRanges create(
      RowKeySchema schema,
      List<List<KeyRange>> ranges,
      int[] slotSpan,
      KeyRange minMaxRange,
      Integer nBuckets,
      boolean useSkipScan,
      int rowTimestampColIndex) {
    int offset = nBuckets == null ? 0 : SaltingUtil.NUM_SALTING_BYTES;
    int nSlots = ranges.size();
    if (nSlots == offset && minMaxRange == KeyRange.EVERYTHING_RANGE) {
      return EVERYTHING;
    } else if (minMaxRange == KeyRange.EMPTY_RANGE
        || (nSlots == 1 + offset
            && ranges.get(offset).size() == 1
            && ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) {
      return NOTHING;
    }
    TimeRange rowTimestampRange = getRowTimestampColumnRange(ranges, schema, rowTimestampColIndex);
    boolean isPointLookup = isPointLookup(schema, ranges, slotSpan, useSkipScan);
    if (isPointLookup) {
      // TODO: consider keeping original to use for serialization as it would be smaller?
      List<byte[]> keys = ScanRanges.getPointKeys(ranges, slotSpan, schema, nBuckets);
      List<KeyRange> keyRanges = Lists.newArrayListWithExpectedSize(keys.size());
      KeyRange unsaltedMinMaxRange = minMaxRange;
      if (nBuckets != null && minMaxRange != KeyRange.EVERYTHING_RANGE) {
        unsaltedMinMaxRange =
            KeyRange.getKeyRange(
                stripPrefix(minMaxRange.getLowerRange(), offset),
                minMaxRange.lowerUnbound(),
                stripPrefix(minMaxRange.getUpperRange(), offset),
                minMaxRange.upperUnbound());
      }
      // We have full keys here, so use field from our varbinary schema
      BytesComparator comparator = ScanUtil.getComparator(SchemaUtil.VAR_BINARY_SCHEMA.getField(0));
      for (byte[] key : keys) {
        // Filter now based on unsalted minMaxRange and ignore the point key salt byte
        if (unsaltedMinMaxRange.compareLowerToUpperBound(
                    key, offset, key.length - offset, true, comparator)
                <= 0
            && unsaltedMinMaxRange.compareUpperToLowerBound(
                    key, offset, key.length - offset, true, comparator)
                >= 0) {
          keyRanges.add(KeyRange.getKeyRange(key));
        }
      }
      ranges = Collections.singletonList(keyRanges);
      useSkipScan = keyRanges.size() > 1;
      // Treat as binary if descending because we've got a separator byte at the end
      // which is not part of the value.
      if (keys.size() > 1
          || SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, schema.getField(0))
              == QueryConstants.DESC_SEPARATOR_BYTE) {
        schema = SchemaUtil.VAR_BINARY_SCHEMA;
        slotSpan = ScanUtil.SINGLE_COLUMN_SLOT_SPAN;
      } else {
        // Keep original schema and don't use skip scan as it's not necessary
        // when there's a single key.
        slotSpan = new int[] {schema.getMaxFields() - 1};
      }
    }
    List<List<KeyRange>> sortedRanges = Lists.newArrayListWithExpectedSize(ranges.size());
    for (int i = 0; i < ranges.size(); i++) {
      List<KeyRange> sorted = Lists.newArrayList(ranges.get(i));
      Collections.sort(sorted, KeyRange.COMPARATOR);
      sortedRanges.add(ImmutableList.copyOf(sorted));
    }

    // Don't set minMaxRange for point lookup because it causes issues during intersect
    // by going across region boundaries
    KeyRange scanRange = KeyRange.EVERYTHING_RANGE;
    // if (!isPointLookup && (nBuckets == null || !useSkipScanFilter)) {
    // if (! ( isPointLookup || (nBuckets != null && useSkipScanFilter) ) ) {
    // if (nBuckets == null || (nBuckets != null && (!isPointLookup || !useSkipScanFilter))) {
    if (nBuckets == null || !isPointLookup || !useSkipScan) {
      byte[] minKey = ScanUtil.getMinKey(schema, sortedRanges, slotSpan);
      byte[] maxKey = ScanUtil.getMaxKey(schema, sortedRanges, slotSpan);
      // If the maxKey has crossed the salt byte boundary, then we do not
      // have anything to filter at the upper end of the range
      if (ScanUtil.crossesPrefixBoundary(maxKey, ScanUtil.getPrefix(minKey, offset), offset)) {
        maxKey = KeyRange.UNBOUND;
      }
      // We won't filter anything at the low end of the range if we just have the salt byte
      if (minKey.length <= offset) {
        minKey = KeyRange.UNBOUND;
      }
      scanRange = KeyRange.getKeyRange(minKey, maxKey);
    }
    if (minMaxRange != KeyRange.EVERYTHING_RANGE) {
      minMaxRange =
          ScanUtil.convertToInclusiveExclusiveRange(
              minMaxRange, schema, new ImmutableBytesWritable());
      scanRange = scanRange.intersect(minMaxRange);
    }

    if (scanRange == KeyRange.EMPTY_RANGE) {
      return NOTHING;
    }
    return new ScanRanges(
        schema,
        slotSpan,
        sortedRanges,
        scanRange,
        minMaxRange,
        useSkipScan,
        isPointLookup,
        nBuckets,
        rowTimestampRange);
  }