@SuppressWarnings("deprecation")
@Override
protected void writeToBuffer(MappedByteBuffer buffer, Tuple e) {
KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
buffer.putInt(kv.getLength() + Bytes.SIZEOF_INT);
buffer.putInt(kv.getLength());
buffer.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
}
/**
* Performs the compaction.
*
* @param scanner Where to read from.
* @param writer Where to write to.
* @param smallestReadPoint Smallest read point.
* @param cleanSeqId When true, remove seqId(used to be mvcc) value which is <= smallestReadPoint
* @return Whether compaction ended; false if it was interrupted for some reason.
*/
protected boolean performCompaction(
InternalScanner scanner, CellSink writer, long smallestReadPoint, boolean cleanSeqId)
throws IOException {
int bytesWritten = 0;
// Since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
List<Cell> kvs = new ArrayList<Cell>();
int closeCheckInterval = HStore.getCloseCheckInterval();
long lastMillis;
if (LOG.isDebugEnabled()) {
lastMillis = System.currentTimeMillis();
} else {
lastMillis = 0;
}
boolean hasMore;
do {
hasMore = scanner.next(kvs, compactionKVMax);
// output to writer:
for (Cell c : kvs) {
KeyValue kv = KeyValueUtil.ensureKeyValue(c);
if (cleanSeqId && kv.getSequenceId() <= smallestReadPoint) {
kv.setSequenceId(0);
}
writer.append(kv);
++progress.currentCompactedKVs;
progress.totalCompactedSize += kv.getLength();
// check periodically to see if a system stop is requested
if (closeCheckInterval > 0) {
bytesWritten += kv.getLength();
if (bytesWritten > closeCheckInterval) {
// Log the progress of long running compactions every minute if
// logging at DEBUG level
if (LOG.isDebugEnabled()) {
long now = System.currentTimeMillis();
if ((now - lastMillis) >= 60 * 1000) {
LOG.debug(
"Compaction progress: "
+ progress
+ String.format(
", rate=%.2f kB/sec",
(bytesWritten / 1024.0) / ((now - lastMillis) / 1000.0)));
lastMillis = now;
}
}
bytesWritten = 0;
if (!store.areWritesEnabled()) {
progress.cancel();
return false;
}
}
}
}
kvs.clear();
} while (hasMore);
progress.complete();
return true;
}
/** {@inheritDoc} */
@Override
public void write(HFileKeyValue entry, NullWritable unused) throws IOException {
final KeyValue kv = entry.getKeyValue();
kv.updateLatestStamp(mLatestTimestampBytes);
final long recordLength = kv.getLength();
if (mCurrentHFileSize + recordLength >= mMaxFileSizeBytes) {
// We can't fit this record in the current HFile without exceeding the max file size.
if (Arrays.equals(mCurrentRow, kv.getRow())) {
// But we're still adding data for a single row, so we can't close this HFile yet.
LOG.debug("Reached max HFile size, but waiting to finish this row before closing.");
} else {
// Close it and open a new one.
closeWriter(mWriter);
mWriter = openNewWriter();
}
}
mWriter.append(kv);
mTimeRangeTracker.includeTimestamp(kv);
mCurrentHFileSize += recordLength;
// Remember the row so we know when we are transitioning.
mCurrentRow = kv.getRow();
}
private void performMerge(List<StoreFileScanner> scanners, HStore store, StoreFile.Writer writer)
throws IOException {
InternalScanner scanner = null;
try {
Scan scan = new Scan();
// Include deletes
scanner =
new StoreScanner(
store,
store.scanInfo,
scan,
scanners,
ScanType.MAJOR_COMPACT,
Long.MIN_VALUE,
Long.MIN_VALUE);
ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
while (scanner.next(kvs) || kvs.size() != 0) {
numKV.addAndGet(kvs.size());
for (KeyValue kv : kvs) {
totalBytes.addAndGet(kv.getLength());
writer.append(kv);
}
kvs.clear();
}
} finally {
if (scanner != null) scanner.close();
}
}
/**
* Convert list of KeyValues to byte buffer.
*
* @param keyValues list of KeyValues to be converted.
* @return buffer with content from key values
*/
public static ByteBuffer convertKvToByteBuffer(
List<KeyValue> keyValues, boolean includesMemstoreTS) {
int totalSize = 0;
for (KeyValue kv : keyValues) {
totalSize += kv.getLength();
if (includesMemstoreTS) {
totalSize += WritableUtils.getVIntSize(kv.getMvccVersion());
}
}
ByteBuffer result = ByteBuffer.allocate(totalSize);
for (KeyValue kv : keyValues) {
result.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
if (includesMemstoreTS) {
ByteBufferUtils.writeVLong(result, kv.getMvccVersion());
}
}
return result;
}
@Override
public Boolean call() throws Exception {
Thread.currentThread().setName("reader " + readerId);
Random rand = new Random();
StoreFileScanner scanner = reader.getStoreFileScanner(true, pread);
while (System.currentTimeMillis() < endTime) {
byte[] row = createRandomRow(rand, firstRow, lastRow);
KeyValue kvToSeek = new KeyValue(row, family, createRandomQualifier(rand));
if (rand.nextDouble() < 0.0001) {
LOG.info("kvToSeek=" + kvToSeek);
}
boolean seekResult;
try {
seekResult = scanner.seek(kvToSeek);
} catch (IOException ex) {
throw new IOException("Seek failed for key " + kvToSeek + ", pread=" + pread, ex);
}
numSeeks.incrementAndGet();
if (!seekResult) {
error("Seek returned false for row " + Bytes.toStringBinary(row));
return false;
}
for (int i = 0; i < rand.nextInt(10) + 1; ++i) {
KeyValue kv = scanner.next();
numKV.incrementAndGet();
if (i == 0 && kv == null) {
error(
"scanner.next() returned null at the first iteration for "
+ "row "
+ Bytes.toStringBinary(row));
return false;
}
if (kv == null) break;
String keyHashStr = MD5Hash.getMD5AsHex(kv.getKey());
keysRead.add(keyHashStr);
totalBytes.addAndGet(kv.getLength());
}
}
return true;
}
@Override
protected int sizeOf(Tuple e) {
KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
return Bytes.SIZEOF_INT * 2 + kv.getLength();
}
/**
* Do a minor/major compaction on an explicit set of storefiles from a Store.
*
* @param request the requested compaction that contains all necessary information to complete the
* compaction (i.e. the store, the files, etc.)
* @return Product of compaction or null if all cells expired or deleted and nothing made it
* through the compaction.
* @throws IOException
*/
StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOException {
// Calculate maximum key count after compaction (for blooms)
// Also calculate earliest put timestamp if major compaction
int maxKeyCount = 0;
long earliestPutTs = HConstants.LATEST_TIMESTAMP;
long maxMVCCReadpoint = 0;
// pull out the interesting things from the CR for ease later
final Store store = request.getStore();
final boolean majorCompaction = request.isMajor();
final List<StoreFile> filesToCompact = request.getFiles();
for (StoreFile file : filesToCompact) {
StoreFile.Reader r = file.getReader();
if (r == null) {
LOG.warn("Null reader for " + file.getPath());
continue;
}
// NOTE: getFilterEntries could cause under-sized blooms if the user
// switches bloom type (e.g. from ROW to ROWCOL)
long keyCount =
(r.getBloomFilterType() == store.getFamily().getBloomFilterType())
? r.getFilterEntries()
: r.getEntries();
maxKeyCount += keyCount;
// Calculate the maximum MVCC readpoint used in any of the involved files
Map<byte[], byte[]> fileInfo = r.loadFileInfo();
byte[] tmp = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
if (tmp != null) {
maxMVCCReadpoint = Math.max(maxMVCCReadpoint, Bytes.toLong(tmp));
}
// For major compactions calculate the earliest put timestamp
// of all involved storefiles. This is used to remove
// family delete marker during the compaction.
if (majorCompaction) {
tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
if (tmp == null) {
// There's a file with no information, must be an old one
// assume we have very old puts
earliestPutTs = HConstants.OLDEST_TIMESTAMP;
} else {
earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Compacting "
+ file
+ ", keycount="
+ keyCount
+ ", bloomtype="
+ r.getBloomFilterType().toString()
+ ", size="
+ StringUtils.humanReadableInt(r.length())
+ ", encoding="
+ r.getHFileReader().getEncodingOnDisk()
+ (majorCompaction ? ", earliestPutTs=" + earliestPutTs : ""));
}
}
// keep track of compaction progress
this.progress = new CompactionProgress(maxKeyCount);
// Get some configs
int compactionKVMax = getConf().getInt("hbase.hstore.compaction.kv.max", 10);
Compression.Algorithm compression = store.getFamily().getCompression();
// Avoid overriding compression setting for major compactions if the user
// has not specified it separately
Compression.Algorithm compactionCompression =
(store.getFamily().getCompactionCompression() != Compression.Algorithm.NONE)
? store.getFamily().getCompactionCompression()
: compression;
// For each file, obtain a scanner:
List<StoreFileScanner> scanners =
StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, false, true);
// Make the instantiation lazy in case compaction produces no product; i.e.
// where all source cells are expired or deleted.
StoreFile.Writer writer = null;
// Find the smallest read point across all the Scanners.
long smallestReadPoint = store.getHRegion().getSmallestReadPoint();
MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
try {
InternalScanner scanner = null;
try {
if (store.getHRegion().getCoprocessorHost() != null) {
scanner =
store
.getHRegion()
.getCoprocessorHost()
.preCompactScannerOpen(
store,
scanners,
majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
earliestPutTs,
request);
}
if (scanner == null) {
Scan scan = new Scan();
scan.setMaxVersions(store.getFamily().getMaxVersions());
/* Include deletes, unless we are doing a major compaction */
scanner =
new StoreScanner(
store,
store.getScanInfo(),
scan,
scanners,
majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
smallestReadPoint,
earliestPutTs);
}
if (store.getHRegion().getCoprocessorHost() != null) {
InternalScanner cpScanner =
store.getHRegion().getCoprocessorHost().preCompact(store, scanner, request);
// NULL scanner returned from coprocessor hooks means skip normal processing
if (cpScanner == null) {
return null;
}
scanner = cpScanner;
}
int bytesWritten = 0;
// since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
List<KeyValue> kvs = new ArrayList<KeyValue>();
// Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
boolean hasMore;
do {
hasMore = scanner.next(kvs, compactionKVMax);
if (writer == null && !kvs.isEmpty()) {
writer =
store.createWriterInTmp(
maxKeyCount,
compactionCompression,
true,
maxMVCCReadpoint >= smallestReadPoint);
}
if (writer != null) {
// output to writer:
for (KeyValue kv : kvs) {
if (kv.getMemstoreTS() <= smallestReadPoint) {
kv.setMemstoreTS(0);
}
writer.append(kv);
// update progress per key
++progress.currentCompactedKVs;
// check periodically to see if a system stop is requested
if (Store.closeCheckInterval > 0) {
bytesWritten += kv.getLength();
if (bytesWritten > Store.closeCheckInterval) {
bytesWritten = 0;
isInterrupted(store, writer);
}
}
}
}
kvs.clear();
} while (hasMore);
} finally {
if (scanner != null) {
scanner.close();
}
}
} finally {
if (writer != null) {
writer.appendMetadata(maxId, majorCompaction);
writer.close();
}
}
return writer;
}
