/** * Checks if this permission grants access to perform the given action on the given table and key * value. * * @param table the table on which the operation is being performed * @param kv the KeyValue on which the operation is being requested * @param action the action requested * @return <code>true</code> if the action is allowed over the given scope by this permission, * otherwise <code>false</code> */ public boolean implies(TableName table, KeyValue kv, Action action) { if (!this.table.equals(table)) { return false; } if (family != null && (Bytes.compareTo( family, 0, family.length, kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()) != 0)) { return false; } if (qualifier != null && (Bytes.compareTo( qualifier, 0, qualifier.length, kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()) != 0)) { return false; } // check actions return super.implies(action); }
/** * Check if the specified KeyValue buffer has been deleted by a previously seen delete. * * @param kv * @param ds * @return True is the specified KeyValue is deleted, false if not */ public boolean isDeleted(final KeyValue kv, final NavigableSet<KeyValue> ds) { if (deletes == null || deletes.isEmpty()) return false; for (KeyValue d : ds) { long kvts = kv.getTimestamp(); long dts = d.getTimestamp(); if (d.isDeleteFamily()) { if (kvts <= dts) return true; continue; } // Check column int ret = Bytes.compareTo( kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(), d.getBuffer(), d.getQualifierOffset(), d.getQualifierLength()); if (ret <= -1) { // This delete is for an earlier column. continue; } else if (ret >= 1) { // Beyond this kv. break; } // Check Timestamp if (kvts > dts) return false; // Check Type switch (KeyValue.Type.codeToType(d.getType())) { case Delete: return kvts == dts; case DeleteColumn: return true; default: continue; } } return false; }
/** * Binary search for latest column value without allocating memory in the process * * @param r * @param searchTerm * @return */ public static KeyValue getColumnLatest(List<KeyValue> kvs, KeyValue searchTerm) { if (kvs.size() == 0) { return null; } // pos === ( -(insertion point) - 1) int pos = Collections.binarySearch(kvs, searchTerm, KeyValue.COMPARATOR); // never will exact match if (pos < 0) { pos = (pos + 1) * -1; // pos is now insertion point } if (pos == kvs.size()) { return null; // doesn't exist } KeyValue kv = kvs.get(pos); if (Bytes.compareTo( kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), searchTerm.getBuffer(), searchTerm.getFamilyOffset(), searchTerm.getFamilyLength()) != 0) { return null; } if (Bytes.compareTo( kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(), searchTerm.getBuffer(), searchTerm.getQualifierOffset(), searchTerm.getQualifierLength()) != 0) { return null; } return kv; }
@Override public void prePut( final ObserverContext<RegionCoprocessorEnvironment> e, final Put put, final WALEdit edit, final Durability durability) throws IOException { byte[] attribute = put.getAttribute("visibility"); byte[] cf = null; List<Cell> updatedCells = new ArrayList<Cell>(); if (attribute != null) { for (List<? extends Cell> edits : put.getFamilyCellMap().values()) { for (Cell cell : edits) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (cf == null) { cf = kv.getFamily(); } Tag tag = new Tag(TAG_TYPE, attribute); List<Tag> tagList = new ArrayList<Tag>(); tagList.add(tag); KeyValue newKV = new KeyValue( kv.getRow(), 0, kv.getRowLength(), kv.getFamily(), 0, kv.getFamilyLength(), kv.getQualifier(), 0, kv.getQualifierLength(), kv.getTimestamp(), KeyValue.Type.codeToType(kv.getType()), kv.getValue(), 0, kv.getValueLength(), tagList); ((List<Cell>) updatedCells).add(newKV); } } put.getFamilyCellMap().remove(cf); // Update the family map put.getFamilyCellMap().put(cf, updatedCells); } }
/** * Pretend we have done a seek but don't do it yet, if possible. The hope is that we find * requested columns in more recent files and won't have to seek in older files. Creates a fake * key/value with the given row/column and the highest (most recent) possible timestamp we might * get from this file. When users of such "lazy scanner" need to know the next KV precisely (e.g. * when this scanner is at the top of the heap), they run {@link #enforceSeek()}. * * <p>Note that this function does guarantee that the current KV of this scanner will be advanced * to at least the given KV. Because of this, it does have to do a real seek in cases when the * seek timestamp is older than the highest timestamp of the file, e.g. when we are trying to seek * to the next row/column and use OLDEST_TIMESTAMP in the seek key. */ @Override public boolean requestSeek(KeyValue kv, boolean forward, boolean useBloom) throws IOException { if (kv.getFamilyLength() == 0) { useBloom = false; } boolean haveToSeek = true; if (useBloom) { // check ROWCOL Bloom filter first. if (reader.getBloomFilterType() == StoreFile.BloomType.ROWCOL) { haveToSeek = reader.passesGeneralBloomFilter( kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()); } else if (this.matcher != null && !matcher.hasNullColumnInQuery() && kv.isDeleteFamily()) { // if there is no such delete family kv in the store file, // then no need to seek. haveToSeek = reader.passesDeleteFamilyBloomFilter( kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()); } } delayedReseek = forward; delayedSeekKV = kv; if (haveToSeek) { // This row/column might be in this store file (or we did not use the // Bloom filter), so we still need to seek. realSeekDone = false; long maxTimestampInFile = reader.getMaxTimestamp(); long seekTimestamp = kv.getTimestamp(); if (seekTimestamp > maxTimestampInFile) { // Create a fake key that is not greater than the real next key. // (Lower timestamps correspond to higher KVs.) // To understand this better, consider that we are asked to seek to // a higher timestamp than the max timestamp in this file. We know that // the next point when we have to consider this file again is when we // pass the max timestamp of this file (with the same row/column). cur = kv.createFirstOnRowColTS(maxTimestampInFile); } else { // This will be the case e.g. when we need to seek to the next // row/column, and we don't know exactly what they are, so we set the // seek key's timestamp to OLDEST_TIMESTAMP to skip the rest of this // row/column. enforceSeek(); } return cur != null; } // Multi-column Bloom filter optimization. // Create a fake key/value, so that this scanner only bubbles up to the top // of the KeyValueHeap in StoreScanner after we scanned this row/column in // all other store files. The query matcher will then just skip this fake // key/value and the store scanner will progress to the next column. This // is obviously not a "real real" seek, but unlike the fake KV earlier in // this method, we want this to be propagated to ScanQueryMatcher. cur = kv.createLastOnRowCol(); realSeekDone = true; return true; }
/** * Get an estimate of the number of rows and bytes per row in regions between startRowKey and * endRowKey. The more store files there are the more this will be off. Also, this does not take * into account any rows that are in the memstore. * * <p>The values computed here should be cached so that in high qps workloads the nn is not * overwhelmed. Could be done in load(); Synchronized to make sure that only one thread at a time * is using the htable. * * @param startRowKey First row key in the range * @param endRowKey Last row key in the range * @return The estimated number of rows in the regions between the row keys (first) and the * estimated row size in bytes (second). */ public synchronized Pair<Long, Long> getEstimatedRowStats(byte[] startRowKey, byte[] endRowKey) { Preconditions.checkNotNull(startRowKey); Preconditions.checkNotNull(endRowKey); long rowSize = 0; long rowCount = 0; long hdfsSize = 0; boolean isCompressed = false; try { // Check to see if things are compressed. // If they are we'll estimate a compression factor. if (columnFamilies_ == null) { columnFamilies_ = hTable_.getTableDescriptor().getColumnFamilies(); } Preconditions.checkNotNull(columnFamilies_); for (HColumnDescriptor desc : columnFamilies_) { isCompressed |= desc.getCompression() != Compression.Algorithm.NONE; } // For every region in the range. List<HRegionLocation> locations = getRegionsInRange(hTable_, startRowKey, endRowKey); for (HRegionLocation location : locations) { long currentHdfsSize = 0; long currentRowSize = 0; long currentRowCount = 0; HRegionInfo info = location.getRegionInfo(); // Get the size on hdfs currentHdfsSize += getHdfsSize(info); Scan s = new Scan(info.getStartKey()); // Get a small sample of rows s.setBatch(ROW_COUNT_ESTIMATE_BATCH_SIZE); // Try and get every version so the row's size can be used to estimate. s.setMaxVersions(Short.MAX_VALUE); // Don't cache the blocks as we don't think these are // necessarily important blocks. s.setCacheBlocks(false); // Try and get deletes too so their size can be counted. s.setRaw(true); ResultScanner rs = hTable_.getScanner(s); try { // And get the the ROW_COUNT_ESTIMATE_BATCH_SIZE fetched rows // for a representative sample for (int i = 0; i < ROW_COUNT_ESTIMATE_BATCH_SIZE; i++) { Result r = rs.next(); if (r == null) break; currentRowCount += 1; for (KeyValue kv : r.list()) { // some extra row size added to make up for shared overhead currentRowSize += kv.getRowLength() // row key + 4 // row key length field + kv.getFamilyLength() // Column family bytes + 4 // family length field + kv.getQualifierLength() // qualifier bytes + 4 // qualifier length field + kv.getValueLength() // length of the value + 4 // value length field + 10; // extra overhead for hfile index, checksums, metadata, etc } } // add these values to the cumulative totals in one shot just // in case there was an error in between getting the hdfs // size and the row/column sizes. hdfsSize += currentHdfsSize; rowCount += currentRowCount; rowSize += currentRowSize; } finally { rs.close(); } } } catch (IOException ioe) { // Print the stack trace, but we'll ignore it // as this is just an estimate. // TODO: Put this into the per query log. LOG.error("Error computing HBase row count estimate", ioe); } // If there are no rows then no need to estimate. if (rowCount == 0) return new Pair<Long, Long>(0L, 0L); // if something went wrong then set a signal value. if (rowSize <= 0 || hdfsSize <= 0) return new Pair<Long, Long>(-1L, -1L); // estimate the number of rows. double bytesPerRow = rowSize / (double) rowCount; long estimatedRowCount = (long) ((isCompressed ? 2 : 1) * (hdfsSize / bytesPerRow)); return new Pair<Long, Long>(estimatedRowCount, (long) bytesPerRow); }