/**
* Checks if this permission grants access to perform the given action on the given table and key
* value.
*
* @param table the table on which the operation is being performed
* @param kv the KeyValue on which the operation is being requested
* @param action the action requested
* @return <code>true</code> if the action is allowed over the given scope by this permission,
* otherwise <code>false</code>
*/
public boolean implies(TableName table, KeyValue kv, Action action) {
if (!this.table.equals(table)) {
return false;
}
if (family != null
&& (Bytes.compareTo(
family,
0,
family.length,
kv.getFamilyArray(),
kv.getFamilyOffset(),
kv.getFamilyLength())
!= 0)) {
return false;
}
if (qualifier != null
&& (Bytes.compareTo(
qualifier,
0,
qualifier.length,
kv.getQualifierArray(),
kv.getQualifierOffset(),
kv.getQualifierLength())
!= 0)) {
return false;
}
// check actions
return super.implies(action);
}
/**
* Check if the specified KeyValue buffer has been deleted by a previously seen delete.
*
* @param kv
* @param ds
* @return True is the specified KeyValue is deleted, false if not
*/
public boolean isDeleted(final KeyValue kv, final NavigableSet<KeyValue> ds) {
if (deletes == null || deletes.isEmpty()) return false;
for (KeyValue d : ds) {
long kvts = kv.getTimestamp();
long dts = d.getTimestamp();
if (d.isDeleteFamily()) {
if (kvts <= dts) return true;
continue;
}
// Check column
int ret =
Bytes.compareTo(
kv.getBuffer(),
kv.getQualifierOffset(),
kv.getQualifierLength(),
d.getBuffer(),
d.getQualifierOffset(),
d.getQualifierLength());
if (ret <= -1) {
// This delete is for an earlier column.
continue;
} else if (ret >= 1) {
// Beyond this kv.
break;
}
// Check Timestamp
if (kvts > dts) return false;
// Check Type
switch (KeyValue.Type.codeToType(d.getType())) {
case Delete:
return kvts == dts;
case DeleteColumn:
return true;
default:
continue;
}
}
return false;
}
/**
* Binary search for latest column value without allocating memory in the process
*
* @param r
* @param searchTerm
* @return
*/
public static KeyValue getColumnLatest(List<KeyValue> kvs, KeyValue searchTerm) {
if (kvs.size() == 0) {
return null;
}
// pos === ( -(insertion point) - 1)
int pos = Collections.binarySearch(kvs, searchTerm, KeyValue.COMPARATOR);
// never will exact match
if (pos < 0) {
pos = (pos + 1) * -1;
// pos is now insertion point
}
if (pos == kvs.size()) {
return null; // doesn't exist
}
KeyValue kv = kvs.get(pos);
if (Bytes.compareTo(
kv.getBuffer(),
kv.getFamilyOffset(),
kv.getFamilyLength(),
searchTerm.getBuffer(),
searchTerm.getFamilyOffset(),
searchTerm.getFamilyLength())
!= 0) {
return null;
}
if (Bytes.compareTo(
kv.getBuffer(),
kv.getQualifierOffset(),
kv.getQualifierLength(),
searchTerm.getBuffer(),
searchTerm.getQualifierOffset(),
searchTerm.getQualifierLength())
!= 0) {
return null;
}
return kv;
}
@Override
public void prePut(
final ObserverContext<RegionCoprocessorEnvironment> e,
final Put put,
final WALEdit edit,
final Durability durability)
throws IOException {
byte[] attribute = put.getAttribute("visibility");
byte[] cf = null;
List<Cell> updatedCells = new ArrayList<Cell>();
if (attribute != null) {
for (List<? extends Cell> edits : put.getFamilyCellMap().values()) {
for (Cell cell : edits) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
if (cf == null) {
cf = kv.getFamily();
}
Tag tag = new Tag(TAG_TYPE, attribute);
List<Tag> tagList = new ArrayList<Tag>();
tagList.add(tag);
KeyValue newKV =
new KeyValue(
kv.getRow(),
0,
kv.getRowLength(),
kv.getFamily(),
0,
kv.getFamilyLength(),
kv.getQualifier(),
0,
kv.getQualifierLength(),
kv.getTimestamp(),
KeyValue.Type.codeToType(kv.getType()),
kv.getValue(),
0,
kv.getValueLength(),
tagList);
((List<Cell>) updatedCells).add(newKV);
}
}
put.getFamilyCellMap().remove(cf);
// Update the family map
put.getFamilyCellMap().put(cf, updatedCells);
}
}
/**
* Pretend we have done a seek but don't do it yet, if possible. The hope is that we find
* requested columns in more recent files and won't have to seek in older files. Creates a fake
* key/value with the given row/column and the highest (most recent) possible timestamp we might
* get from this file. When users of such "lazy scanner" need to know the next KV precisely (e.g.
* when this scanner is at the top of the heap), they run {@link #enforceSeek()}.
*
* <p>Note that this function does guarantee that the current KV of this scanner will be advanced
* to at least the given KV. Because of this, it does have to do a real seek in cases when the
* seek timestamp is older than the highest timestamp of the file, e.g. when we are trying to seek
* to the next row/column and use OLDEST_TIMESTAMP in the seek key.
*/
@Override
public boolean requestSeek(KeyValue kv, boolean forward, boolean useBloom) throws IOException {
if (kv.getFamilyLength() == 0) {
useBloom = false;
}
boolean haveToSeek = true;
if (useBloom) {
// check ROWCOL Bloom filter first.
if (reader.getBloomFilterType() == StoreFile.BloomType.ROWCOL) {
haveToSeek =
reader.passesGeneralBloomFilter(
kv.getBuffer(),
kv.getRowOffset(),
kv.getRowLength(),
kv.getBuffer(),
kv.getQualifierOffset(),
kv.getQualifierLength());
} else if (this.matcher != null && !matcher.hasNullColumnInQuery() && kv.isDeleteFamily()) {
// if there is no such delete family kv in the store file,
// then no need to seek.
haveToSeek =
reader.passesDeleteFamilyBloomFilter(
kv.getBuffer(), kv.getRowOffset(), kv.getRowLength());
}
}
delayedReseek = forward;
delayedSeekKV = kv;
if (haveToSeek) {
// This row/column might be in this store file (or we did not use the
// Bloom filter), so we still need to seek.
realSeekDone = false;
long maxTimestampInFile = reader.getMaxTimestamp();
long seekTimestamp = kv.getTimestamp();
if (seekTimestamp > maxTimestampInFile) {
// Create a fake key that is not greater than the real next key.
// (Lower timestamps correspond to higher KVs.)
// To understand this better, consider that we are asked to seek to
// a higher timestamp than the max timestamp in this file. We know that
// the next point when we have to consider this file again is when we
// pass the max timestamp of this file (with the same row/column).
cur = kv.createFirstOnRowColTS(maxTimestampInFile);
} else {
// This will be the case e.g. when we need to seek to the next
// row/column, and we don't know exactly what they are, so we set the
// seek key's timestamp to OLDEST_TIMESTAMP to skip the rest of this
// row/column.
enforceSeek();
}
return cur != null;
}
// Multi-column Bloom filter optimization.
// Create a fake key/value, so that this scanner only bubbles up to the top
// of the KeyValueHeap in StoreScanner after we scanned this row/column in
// all other store files. The query matcher will then just skip this fake
// key/value and the store scanner will progress to the next column. This
// is obviously not a "real real" seek, but unlike the fake KV earlier in
// this method, we want this to be propagated to ScanQueryMatcher.
cur = kv.createLastOnRowCol();
realSeekDone = true;
return true;
}
/**
* Get an estimate of the number of rows and bytes per row in regions between startRowKey and
* endRowKey. The more store files there are the more this will be off. Also, this does not take
* into account any rows that are in the memstore.
*
* <p>The values computed here should be cached so that in high qps workloads the nn is not
* overwhelmed. Could be done in load(); Synchronized to make sure that only one thread at a time
* is using the htable.
*
* @param startRowKey First row key in the range
* @param endRowKey Last row key in the range
* @return The estimated number of rows in the regions between the row keys (first) and the
* estimated row size in bytes (second).
*/
public synchronized Pair<Long, Long> getEstimatedRowStats(byte[] startRowKey, byte[] endRowKey) {
Preconditions.checkNotNull(startRowKey);
Preconditions.checkNotNull(endRowKey);
long rowSize = 0;
long rowCount = 0;
long hdfsSize = 0;
boolean isCompressed = false;
try {
// Check to see if things are compressed.
// If they are we'll estimate a compression factor.
if (columnFamilies_ == null) {
columnFamilies_ = hTable_.getTableDescriptor().getColumnFamilies();
}
Preconditions.checkNotNull(columnFamilies_);
for (HColumnDescriptor desc : columnFamilies_) {
isCompressed |= desc.getCompression() != Compression.Algorithm.NONE;
}
// For every region in the range.
List<HRegionLocation> locations = getRegionsInRange(hTable_, startRowKey, endRowKey);
for (HRegionLocation location : locations) {
long currentHdfsSize = 0;
long currentRowSize = 0;
long currentRowCount = 0;
HRegionInfo info = location.getRegionInfo();
// Get the size on hdfs
currentHdfsSize += getHdfsSize(info);
Scan s = new Scan(info.getStartKey());
// Get a small sample of rows
s.setBatch(ROW_COUNT_ESTIMATE_BATCH_SIZE);
// Try and get every version so the row's size can be used to estimate.
s.setMaxVersions(Short.MAX_VALUE);
// Don't cache the blocks as we don't think these are
// necessarily important blocks.
s.setCacheBlocks(false);
// Try and get deletes too so their size can be counted.
s.setRaw(true);
ResultScanner rs = hTable_.getScanner(s);
try {
// And get the the ROW_COUNT_ESTIMATE_BATCH_SIZE fetched rows
// for a representative sample
for (int i = 0; i < ROW_COUNT_ESTIMATE_BATCH_SIZE; i++) {
Result r = rs.next();
if (r == null) break;
currentRowCount += 1;
for (KeyValue kv : r.list()) {
// some extra row size added to make up for shared overhead
currentRowSize +=
kv.getRowLength() // row key
+ 4 // row key length field
+ kv.getFamilyLength() // Column family bytes
+ 4 // family length field
+ kv.getQualifierLength() // qualifier bytes
+ 4 // qualifier length field
+ kv.getValueLength() // length of the value
+ 4 // value length field
+ 10; // extra overhead for hfile index, checksums, metadata, etc
}
}
// add these values to the cumulative totals in one shot just
// in case there was an error in between getting the hdfs
// size and the row/column sizes.
hdfsSize += currentHdfsSize;
rowCount += currentRowCount;
rowSize += currentRowSize;
} finally {
rs.close();
}
}
} catch (IOException ioe) {
// Print the stack trace, but we'll ignore it
// as this is just an estimate.
// TODO: Put this into the per query log.
LOG.error("Error computing HBase row count estimate", ioe);
}
// If there are no rows then no need to estimate.
if (rowCount == 0) return new Pair<Long, Long>(0L, 0L);
// if something went wrong then set a signal value.
if (rowSize <= 0 || hdfsSize <= 0) return new Pair<Long, Long>(-1L, -1L);
// estimate the number of rows.
double bytesPerRow = rowSize / (double) rowCount;
long estimatedRowCount = (long) ((isCompressed ? 2 : 1) * (hdfsSize / bytesPerRow));
return new Pair<Long, Long>(estimatedRowCount, (long) bytesPerRow);
}
