예제 #1
0
  private Iterator<Pair<byte[], List<Mutation>>> addRowMutations(
      final TableRef tableRef,
      final Map<ImmutableBytesPtr, Map<PColumn, byte[]>> values,
      long timestamp,
      boolean includeMutableIndexes) {
    final List<Mutation> mutations = Lists.newArrayListWithExpectedSize(values.size());
    Iterator<Map.Entry<ImmutableBytesPtr, Map<PColumn, byte[]>>> iterator =
        values.entrySet().iterator();
    while (iterator.hasNext()) {
      Map.Entry<ImmutableBytesPtr, Map<PColumn, byte[]>> rowEntry = iterator.next();
      ImmutableBytesPtr key = rowEntry.getKey();
      PRow row = tableRef.getTable().newRow(connection.getKeyValueBuilder(), timestamp, key);
      if (rowEntry.getValue() == PRow.DELETE_MARKER) { // means delete
        row.delete();
      } else {
        for (Map.Entry<PColumn, byte[]> valueEntry : rowEntry.getValue().entrySet()) {
          row.setValue(valueEntry.getKey(), valueEntry.getValue());
        }
      }
      mutations.addAll(row.toRowMutations());
    }
    final Iterator<PTable>
        indexes = // Only maintain tables with immutable rows through this client-side mechanism
        (tableRef.getTable().isImmutableRows() || includeMutableIndexes)
                ? IndexMaintainer.nonDisabledIndexIterator(
                    tableRef.getTable().getIndexes().iterator())
                : Iterators.<PTable>emptyIterator();
    return new Iterator<Pair<byte[], List<Mutation>>>() {
      boolean isFirst = true;

      @Override
      public boolean hasNext() {
        return isFirst || indexes.hasNext();
      }

      @Override
      public Pair<byte[], List<Mutation>> next() {
        if (isFirst) {
          isFirst = false;
          return new Pair<byte[], List<Mutation>>(
              tableRef.getTable().getPhysicalName().getBytes(), mutations);
        }
        PTable index = indexes.next();
        List<Mutation> indexMutations;
        try {
          indexMutations =
              IndexUtil.generateIndexData(
                  tableRef.getTable(), index, mutations, tempPtr, connection.getKeyValueBuilder());
        } catch (SQLException e) {
          throw new IllegalDataException(e);
        }
        return new Pair<byte[], List<Mutation>>(index.getPhysicalName().getBytes(), indexMutations);
      }

      @Override
      public void remove() {
        throw new UnsupportedOperationException();
      }
    };
  }
  @Override
  protected RegionScanner doPostScannerOpen(
      final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s)
      throws IOException {
    byte[] isUngroupedAgg = scan.getAttribute(BaseScannerRegionObserver.UNGROUPED_AGG);
    if (isUngroupedAgg == null) {
      return s;
    }

    final ScanProjector p = ScanProjector.deserializeProjectorFromScan(scan);
    final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan);
    RegionScanner theScanner = s;
    if (p != null || j != null) {
      theScanner =
          new HashJoinRegionScanner(s, p, j, ScanUtil.getTenantId(scan), c.getEnvironment());
    }
    final RegionScanner innerScanner = theScanner;

    byte[] indexUUID = scan.getAttribute(PhoenixIndexCodec.INDEX_UUID);
    PTable projectedTable = null;
    List<Expression> selectExpressions = null;
    byte[] upsertSelectTable = scan.getAttribute(BaseScannerRegionObserver.UPSERT_SELECT_TABLE);
    boolean isUpsert = false;
    boolean isDelete = false;
    byte[] deleteCQ = null;
    byte[] deleteCF = null;
    byte[][] values = null;
    byte[] emptyCF = null;
    ImmutableBytesWritable ptr = null;
    if (upsertSelectTable != null) {
      isUpsert = true;
      projectedTable = deserializeTable(upsertSelectTable);
      selectExpressions =
          deserializeExpressions(scan.getAttribute(BaseScannerRegionObserver.UPSERT_SELECT_EXPRS));
      values = new byte[projectedTable.getPKColumns().size()][];
      ptr = new ImmutableBytesWritable();
    } else {
      byte[] isDeleteAgg = scan.getAttribute(BaseScannerRegionObserver.DELETE_AGG);
      isDelete = isDeleteAgg != null && Bytes.compareTo(PDataType.TRUE_BYTES, isDeleteAgg) == 0;
      if (!isDelete) {
        deleteCF = scan.getAttribute(BaseScannerRegionObserver.DELETE_CF);
        deleteCQ = scan.getAttribute(BaseScannerRegionObserver.DELETE_CQ);
      }
      emptyCF = scan.getAttribute(BaseScannerRegionObserver.EMPTY_CF);
    }

    int batchSize = 0;
    long ts = scan.getTimeRange().getMax();
    HRegion region = c.getEnvironment().getRegion();
    List<Mutation> mutations = Collections.emptyList();
    if (isDelete || isUpsert || (deleteCQ != null && deleteCF != null) || emptyCF != null) {
      // TODO: size better
      mutations = Lists.newArrayListWithExpectedSize(1024);
      batchSize =
          c.getEnvironment()
              .getConfiguration()
              .getInt(MUTATE_BATCH_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
    }
    Aggregators aggregators =
        ServerAggregators.deserialize(
            scan.getAttribute(BaseScannerRegionObserver.AGGREGATORS),
            c.getEnvironment().getConfiguration());
    Aggregator[] rowAggregators = aggregators.getAggregators();
    boolean hasMore;
    boolean hasAny = false;
    MultiKeyValueTuple result = new MultiKeyValueTuple();
    if (logger.isInfoEnabled()) {
      logger.info("Starting ungrouped coprocessor scan " + scan);
    }
    long rowCount = 0;
    region.startRegionOperation();
    try {
      do {
        List<Cell> results = new ArrayList<Cell>();
        // Results are potentially returned even when the return value of s.next is false
        // since this is an indication of whether or not there are more values after the
        // ones returned
        hasMore = innerScanner.nextRaw(results);
        if (!results.isEmpty()) {
          rowCount++;
          result.setKeyValues(results);
          try {
            if (isDelete) {
              // FIXME: the version of the Delete constructor without the lock args was introduced
              // in 0.94.4, thus if we try to use it here we can no longer use the 0.94.2 version
              // of the client.
              Cell firstKV = results.get(0);
              Delete delete =
                  new Delete(
                      firstKV.getRowArray(), firstKV.getRowOffset(), firstKV.getRowLength(), ts);
              mutations.add(delete);
            } else if (isUpsert) {
              Arrays.fill(values, null);
              int i = 0;
              List<PColumn> projectedColumns = projectedTable.getColumns();
              for (; i < projectedTable.getPKColumns().size(); i++) {
                Expression expression = selectExpressions.get(i);
                if (expression.evaluate(result, ptr)) {
                  values[i] = ptr.copyBytes();
                  // If SortOrder from expression in SELECT doesn't match the
                  // column being projected into then invert the bits.
                  if (expression.getSortOrder() != projectedColumns.get(i).getSortOrder()) {
                    SortOrder.invert(values[i], 0, values[i], 0, values[i].length);
                  }
                }
              }
              projectedTable.newKey(ptr, values);
              PRow row = projectedTable.newRow(kvBuilder, ts, ptr);
              for (; i < projectedColumns.size(); i++) {
                Expression expression = selectExpressions.get(i);
                if (expression.evaluate(result, ptr)) {
                  PColumn column = projectedColumns.get(i);
                  Object value = expression.getDataType().toObject(ptr, column.getSortOrder());
                  // We are guaranteed that the two column will have the same type.
                  if (!column
                      .getDataType()
                      .isSizeCompatible(
                          ptr,
                          value,
                          column.getDataType(),
                          expression.getMaxLength(),
                          expression.getScale(),
                          column.getMaxLength(),
                          column.getScale())) {
                    throw new ValueTypeIncompatibleException(
                        column.getDataType(), column.getMaxLength(), column.getScale());
                  }
                  column
                      .getDataType()
                      .coerceBytes(
                          ptr,
                          value,
                          expression.getDataType(),
                          expression.getMaxLength(),
                          expression.getScale(),
                          expression.getSortOrder(),
                          column.getMaxLength(),
                          column.getScale(),
                          column.getSortOrder());
                  byte[] bytes = ByteUtil.copyKeyBytesIfNecessary(ptr);
                  row.setValue(column, bytes);
                }
              }
              for (Mutation mutation : row.toRowMutations()) {
                mutations.add(mutation);
              }
            } else if (deleteCF != null && deleteCQ != null) {
              // No need to search for delete column, since we project only it
              // if no empty key value is being set
              if (emptyCF == null || result.getValue(deleteCF, deleteCQ) != null) {
                Delete delete =
                    new Delete(
                        results.get(0).getRowArray(),
                        results.get(0).getRowOffset(),
                        results.get(0).getRowLength());
                delete.deleteColumns(deleteCF, deleteCQ, ts);
                mutations.add(delete);
              }
            }
            if (emptyCF != null) {
              /*
               * If we've specified an emptyCF, then we need to insert an empty
               * key value "retroactively" for any key value that is visible at
               * the timestamp that the DDL was issued. Key values that are not
               * visible at this timestamp will not ever be projected up to
               * scans past this timestamp, so don't need to be considered.
               * We insert one empty key value per row per timestamp.
               */
              Set<Long> timeStamps = Sets.newHashSetWithExpectedSize(results.size());
              for (Cell kv : results) {
                long kvts = kv.getTimestamp();
                if (!timeStamps.contains(kvts)) {
                  Put put = new Put(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
                  put.add(
                      emptyCF, QueryConstants.EMPTY_COLUMN_BYTES, kvts, ByteUtil.EMPTY_BYTE_ARRAY);
                  mutations.add(put);
                }
              }
            }
            // Commit in batches based on UPSERT_BATCH_SIZE_ATTRIB in config
            if (!mutations.isEmpty() && batchSize > 0 && mutations.size() % batchSize == 0) {
              commitBatch(region, mutations, indexUUID);
              mutations.clear();
            }
          } catch (ConstraintViolationException e) {
            // Log and ignore in count
            logger.error(
                "Failed to create row in "
                    + region.getRegionNameAsString()
                    + " with values "
                    + SchemaUtil.toString(values),
                e);
            continue;
          }
          aggregators.aggregate(rowAggregators, result);
          hasAny = true;
        }
      } while (hasMore);
    } finally {
      innerScanner.close();
      region.closeRegionOperation();
    }

    if (logger.isInfoEnabled()) {
      logger.info("Finished scanning " + rowCount + " rows for ungrouped coprocessor scan " + scan);
    }

    if (!mutations.isEmpty()) {
      commitBatch(region, mutations, indexUUID);
    }

    final boolean hadAny = hasAny;
    KeyValue keyValue = null;
    if (hadAny) {
      byte[] value = aggregators.toBytes(rowAggregators);
      keyValue =
          KeyValueUtil.newKeyValue(
              UNGROUPED_AGG_ROW_KEY,
              SINGLE_COLUMN_FAMILY,
              SINGLE_COLUMN,
              AGG_TIMESTAMP,
              value,
              0,
              value.length);
    }
    final KeyValue aggKeyValue = keyValue;

    RegionScanner scanner =
        new BaseRegionScanner() {
          private boolean done = !hadAny;

          @Override
          public HRegionInfo getRegionInfo() {
            return innerScanner.getRegionInfo();
          }

          @Override
          public boolean isFilterDone() {
            return done;
          }

          @Override
          public void close() throws IOException {
            innerScanner.close();
          }

          @Override
          public boolean next(List<Cell> results) throws IOException {
            if (done) return false;
            done = true;
            results.add(aggKeyValue);
            return false;
          }

          @Override
          public long getMaxResultSize() {
            return scan.getMaxResultSize();
          }
        };
    return scanner;
  }