@Override
public Collection<StoreFile> getUnneededFiles(long maxTs, List<StoreFile> filesCompacting) {
Collection<StoreFile> expiredStoreFiles = null;
ImmutableList<StoreFile> files = storefiles;
// 1) We can never get rid of the last file which has the maximum seqid.
// 2) Files that are not the latest can't become one due to (1), so the rest are fair game.
for (int i = 0; i < files.size() - 1; ++i) {
StoreFile sf = files.get(i);
long fileTs = sf.getReader().getMaxTimestamp();
if (fileTs < maxTs && !filesCompacting.contains(sf)) {
LOG.info(
"Found an expired store file: "
+ sf.getPath()
+ " whose maxTimeStamp is "
+ fileTs
+ ", which is below "
+ maxTs);
if (expiredStoreFiles == null) {
expiredStoreFiles = new ArrayList<StoreFile>();
}
expiredStoreFiles.add(sf);
}
}
return expiredStoreFiles;
}
/**
* @return the total key count in the files being merged
* @throws IOException
*/
private long prepareForMerge() throws IOException {
LOG.info("Merging " + inputFileNames);
LOG.info("Using block size: " + blockSize);
inputStoreFiles = new ArrayList<StoreFile>();
long maxKeyCount = 0;
for (String fileName : inputFileNames) {
Path filePath = new Path(fileName);
// Open without caching.
StoreFile sf = openStoreFile(filePath, false);
sf.createReader();
inputStoreFiles.add(sf);
StoreFile.Reader r = sf.getReader();
if (r != null) {
long keyCount = r.getFilterEntries();
maxKeyCount += keyCount;
LOG.info(
"Compacting: "
+ sf
+ "; keyCount = "
+ keyCount
+ "; Bloom Type = "
+ r.getBloomFilterType().toString()
+ "; Size = "
+ StringUtils.humanReadableInt(r.length()));
}
}
return maxKeyCount;
}
/**
* Moves multiple store files to the relative region's family store directory.
*
* @param storeFiles list of store files divided by family
* @throws IOException
*/
void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
for (Map.Entry<byte[], List<StoreFile>> es : storeFiles.entrySet()) {
String familyName = Bytes.toString(es.getKey());
for (StoreFile sf : es.getValue()) {
commitStoreFile(familyName, sf.getPath());
}
}
}
private static CompactionRequest createDummyRequest() throws Exception {
// "Files" are totally unused, it's Scanner class below that gives compactor fake KVs.
// But compaction depends on everything under the sun, so stub everything with dummies.
StoreFile sf = mock(StoreFile.class);
StoreFile.Reader r = mock(StoreFile.Reader.class);
when(r.length()).thenReturn(1L);
when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong()))
.thenReturn(mock(StoreFileScanner.class));
when(sf.getReader()).thenReturn(r);
when(sf.createReader()).thenReturn(r);
return new CompactionRequest(Arrays.asList(sf));
}
/**
* Return an array of scanners corresponding to the given set of store files, And set the
* ScanQueryMatcher for each store file scanner for further optimization
*/
public static List<StoreFileScanner> getScannersForStoreFiles(
Collection<StoreFile> files,
boolean cacheBlocks,
boolean isCompaction,
ScanQueryMatcher matcher)
throws IOException {
List<StoreFileScanner> scanners = new ArrayList<StoreFileScanner>(files.size());
for (StoreFile file : files) {
StoreFile.Reader r = file.createReader();
StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, isCompaction);
scanner.setScanQueryMatcher(matcher);
scanners.add(scanner);
}
return scanners;
}
private Pair<Path, Path> splitStoreFile(final byte[] family, final StoreFile sf)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Splitting started for store file: " + sf.getPath() + " for region: " + this.parent);
}
HRegionFileSystem fs = this.parent.getRegionFileSystem();
String familyName = Bytes.toString(family);
Path path_a =
fs.splitStoreFile(
this.hri_a, familyName, sf, this.splitrow, false, this.parent.getSplitPolicy());
Path path_b =
fs.splitStoreFile(
this.hri_b, familyName, sf, this.splitrow, true, this.parent.getSplitPolicy());
if (LOG.isDebugEnabled()) {
LOG.debug(
"Splitting complete for store file: " + sf.getPath() + " for region: " + this.parent);
}
return new Pair<Path, Path>(path_a, path_b);
}
/**
* Write out a merge reference under the given merges directory. Package local so it doesnt leak
* out of regionserver.
*
* @param mergedRegion {@link HRegionInfo} of the merged region
* @param familyName Column Family Name
* @param f File to create reference.
* @param mergedDir
* @return Path to created reference.
* @throws IOException
*/
Path mergeStoreFile(
final HRegionInfo mergedRegion,
final String familyName,
final StoreFile f,
final Path mergedDir)
throws IOException {
Path referenceDir = new Path(new Path(mergedDir, mergedRegion.getEncodedName()), familyName);
// A whole reference to the store file.
Reference r = Reference.createTopReference(regionInfo.getStartKey());
// Add the referred-to regions name as a dot separated suffix.
// See REF_NAME_REGEX regex above. The referred-to regions name is
// up in the path of the passed in <code>f</code> -- parentdir is family,
// then the directory above is the region name.
String mergingRegionName = regionInfo.getEncodedName();
// Write reference with same file id only with the other region name as
// suffix and into the new region location (under same family).
Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName);
return r.write(fs, p);
}
@Override
public void postFlush(
ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile)
throws IOException {
// Register HFiles for incremental backup
SpliceLogUtils.info(LOG, "Flushing region %s.%s", tableName, regionName);
try {
if (namespace.compareTo("splice") != 0) return;
BackupUtils.captureIncrementalChanges(
conf,
region,
path,
fs,
rootDir,
backupDir,
tableName,
resultFile.getPath().getName(),
preparing);
} catch (Exception ex) {
throw new IOException(ex);
}
}
@Override
public void setUp() throws Exception {
// setup config values necessary for store
this.conf = TEST_UTIL.getConfiguration();
this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
this.conf.setInt("hbase.hstore.compaction.min", minFiles);
this.conf.setInt("hbase.hstore.compaction.max", maxFiles);
this.conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, minSize);
this.conf.setLong("hbase.hstore.compaction.max.size", maxSize);
this.conf.setFloat("hbase.hstore.compaction.ratio", 1.0F);
// Setting up a Store
Path basedir = new Path(DIR);
String logName = "logs";
Path logdir = new Path(DIR, logName);
Path oldLogDir = new Path(basedir, HConstants.HREGION_OLDLOGDIR_NAME);
HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family"));
FileSystem fs = FileSystem.get(conf);
fs.delete(logdir, true);
HTableDescriptor htd = new HTableDescriptor(Bytes.toBytes("table"));
htd.addFamily(hcd);
HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
region = HRegion.createHRegion(info, basedir, conf, htd);
HRegion.closeHRegion(region);
Path tableDir = new Path(basedir, Bytes.toString(htd.getName()));
region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
store = new HStore(basedir, region, hcd, fs, conf);
manager = store.compactionPolicy;
TEST_FILE = StoreFile.getRandomFilename(fs, store.getHomedir());
fs.create(TEST_FILE);
}
public boolean runRandomReadWorkload() throws IOException {
if (inputFileNames.size() != 1) {
throw new IOException("Need exactly one input file for random reads: " + inputFileNames);
}
Path inputPath = new Path(inputFileNames.get(0));
// Make sure we are using caching.
StoreFile storeFile = openStoreFile(inputPath, true);
StoreFile.Reader reader = storeFile.createReader();
LOG.info("First key: " + Bytes.toStringBinary(reader.getFirstKey()));
LOG.info("Last key: " + Bytes.toStringBinary(reader.getLastKey()));
KeyValue firstKV = KeyValue.createKeyValueFromKey(reader.getFirstKey());
firstRow = firstKV.getRow();
KeyValue lastKV = KeyValue.createKeyValueFromKey(reader.getLastKey());
lastRow = lastKV.getRow();
byte[] family = firstKV.getFamily();
if (!Bytes.equals(family, lastKV.getFamily())) {
LOG.error(
"First and last key have different families: "
+ Bytes.toStringBinary(family)
+ " and "
+ Bytes.toStringBinary(lastKV.getFamily()));
return false;
}
if (Bytes.equals(firstRow, lastRow)) {
LOG.error(
"First and last row are the same, cannot run read workload: "
+ "firstRow="
+ Bytes.toStringBinary(firstRow)
+ ", "
+ "lastRow="
+ Bytes.toStringBinary(lastRow));
return false;
}
ExecutorService exec = Executors.newFixedThreadPool(numReadThreads + 1);
int numCompleted = 0;
int numFailed = 0;
try {
ExecutorCompletionService<Boolean> ecs = new ExecutorCompletionService<Boolean>(exec);
endTime = System.currentTimeMillis() + 1000 * durationSec;
boolean pread = true;
for (int i = 0; i < numReadThreads; ++i) ecs.submit(new RandomReader(i, reader, pread));
ecs.submit(new StatisticsPrinter());
Future<Boolean> result;
while (true) {
try {
result = ecs.poll(endTime + 1000 - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
if (result == null) break;
try {
if (result.get()) {
++numCompleted;
} else {
++numFailed;
}
} catch (ExecutionException e) {
LOG.error("Worker thread failure", e.getCause());
++numFailed;
}
} catch (InterruptedException ex) {
LOG.error("Interrupted after " + numCompleted + " workers completed");
Thread.currentThread().interrupt();
continue;
}
}
} finally {
storeFile.closeReader(true);
exec.shutdown();
BlockCache c = cacheConf.getBlockCache();
if (c != null) {
c.shutdown();
}
}
LOG.info("Worker threads completed: " + numCompleted);
LOG.info("Worker threads failed: " + numFailed);
return true;
}
// Mark the files as compactedAway once the storefiles and compactedfiles list is finalised
// Let a background thread close the actual reader on these compacted files and also
// ensure to evict the blocks from block cache so that they are no longer in
// cache
private void markCompactedAway(Collection<StoreFile> compactedFiles) {
for (StoreFile file : compactedFiles) {
file.markCompactedAway();
}
}
/**
* Do a minor/major compaction on an explicit set of storefiles from a Store.
*
* @param request the requested compaction that contains all necessary information to complete the
* compaction (i.e. the store, the files, etc.)
* @return Product of compaction or null if all cells expired or deleted and nothing made it
* through the compaction.
* @throws IOException
*/
StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOException {
// Calculate maximum key count after compaction (for blooms)
// Also calculate earliest put timestamp if major compaction
int maxKeyCount = 0;
long earliestPutTs = HConstants.LATEST_TIMESTAMP;
long maxMVCCReadpoint = 0;
// pull out the interesting things from the CR for ease later
final Store store = request.getStore();
final boolean majorCompaction = request.isMajor();
final List<StoreFile> filesToCompact = request.getFiles();
for (StoreFile file : filesToCompact) {
StoreFile.Reader r = file.getReader();
if (r == null) {
LOG.warn("Null reader for " + file.getPath());
continue;
}
// NOTE: getFilterEntries could cause under-sized blooms if the user
// switches bloom type (e.g. from ROW to ROWCOL)
long keyCount =
(r.getBloomFilterType() == store.getFamily().getBloomFilterType())
? r.getFilterEntries()
: r.getEntries();
maxKeyCount += keyCount;
// Calculate the maximum MVCC readpoint used in any of the involved files
Map<byte[], byte[]> fileInfo = r.loadFileInfo();
byte[] tmp = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
if (tmp != null) {
maxMVCCReadpoint = Math.max(maxMVCCReadpoint, Bytes.toLong(tmp));
}
// For major compactions calculate the earliest put timestamp
// of all involved storefiles. This is used to remove
// family delete marker during the compaction.
if (majorCompaction) {
tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
if (tmp == null) {
// There's a file with no information, must be an old one
// assume we have very old puts
earliestPutTs = HConstants.OLDEST_TIMESTAMP;
} else {
earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Compacting "
+ file
+ ", keycount="
+ keyCount
+ ", bloomtype="
+ r.getBloomFilterType().toString()
+ ", size="
+ StringUtils.humanReadableInt(r.length())
+ ", encoding="
+ r.getHFileReader().getEncodingOnDisk()
+ (majorCompaction ? ", earliestPutTs=" + earliestPutTs : ""));
}
}
// keep track of compaction progress
this.progress = new CompactionProgress(maxKeyCount);
// Get some configs
int compactionKVMax = getConf().getInt("hbase.hstore.compaction.kv.max", 10);
Compression.Algorithm compression = store.getFamily().getCompression();
// Avoid overriding compression setting for major compactions if the user
// has not specified it separately
Compression.Algorithm compactionCompression =
(store.getFamily().getCompactionCompression() != Compression.Algorithm.NONE)
? store.getFamily().getCompactionCompression()
: compression;
// For each file, obtain a scanner:
List<StoreFileScanner> scanners =
StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, false, true);
// Make the instantiation lazy in case compaction produces no product; i.e.
// where all source cells are expired or deleted.
StoreFile.Writer writer = null;
// Find the smallest read point across all the Scanners.
long smallestReadPoint = store.getHRegion().getSmallestReadPoint();
MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
try {
InternalScanner scanner = null;
try {
if (store.getHRegion().getCoprocessorHost() != null) {
scanner =
store
.getHRegion()
.getCoprocessorHost()
.preCompactScannerOpen(
store,
scanners,
majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
earliestPutTs,
request);
}
if (scanner == null) {
Scan scan = new Scan();
scan.setMaxVersions(store.getFamily().getMaxVersions());
/* Include deletes, unless we are doing a major compaction */
scanner =
new StoreScanner(
store,
store.getScanInfo(),
scan,
scanners,
majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
smallestReadPoint,
earliestPutTs);
}
if (store.getHRegion().getCoprocessorHost() != null) {
InternalScanner cpScanner =
store.getHRegion().getCoprocessorHost().preCompact(store, scanner, request);
// NULL scanner returned from coprocessor hooks means skip normal processing
if (cpScanner == null) {
return null;
}
scanner = cpScanner;
}
int bytesWritten = 0;
// since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
List<KeyValue> kvs = new ArrayList<KeyValue>();
// Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
boolean hasMore;
do {
hasMore = scanner.next(kvs, compactionKVMax);
if (writer == null && !kvs.isEmpty()) {
writer =
store.createWriterInTmp(
maxKeyCount,
compactionCompression,
true,
maxMVCCReadpoint >= smallestReadPoint);
}
if (writer != null) {
// output to writer:
for (KeyValue kv : kvs) {
if (kv.getMemstoreTS() <= smallestReadPoint) {
kv.setMemstoreTS(0);
}
writer.append(kv);
// update progress per key
++progress.currentCompactedKVs;
// check periodically to see if a system stop is requested
if (Store.closeCheckInterval > 0) {
bytesWritten += kv.getLength();
if (bytesWritten > Store.closeCheckInterval) {
bytesWritten = 0;
isInterrupted(store, writer);
}
}
}
}
kvs.clear();
} while (hasMore);
} finally {
if (scanner != null) {
scanner.close();
}
}
} finally {
if (writer != null) {
writer.appendMetadata(maxId, majorCompaction);
writer.close();
}
}
return writer;
}
/**
* Write out a split reference. Package local so it doesnt leak out of regionserver.
*
* @param hri {@link HRegionInfo} of the destination
* @param familyName Column Family Name
* @param f File to split.
* @param splitRow Split Row
* @param top True if we are referring to the top half of the hfile.
* @return Path to created reference.
* @throws IOException
*/
Path splitStoreFile(
final HRegionInfo hri,
final String familyName,
final StoreFile f,
final byte[] splitRow,
final boolean top)
throws IOException {
// Check whether the split row lies in the range of the store file
// If it is outside the range, return directly.
if (!isIndexTable()) {
if (top) {
// check if larger than last key.
KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
byte[] lastKey = f.createReader().getLastKey();
// If lastKey is null means storefile is empty.
if (lastKey == null) return null;
if (f.getReader()
.getComparator()
.compareFlatKey(
splitKey.getBuffer(),
splitKey.getKeyOffset(),
splitKey.getKeyLength(),
lastKey,
0,
lastKey.length)
> 0) {
return null;
}
} else {
// check if smaller than first key
KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
byte[] firstKey = f.createReader().getFirstKey();
// If firstKey is null means storefile is empty.
if (firstKey == null) return null;
if (f.getReader()
.getComparator()
.compareFlatKey(
splitKey.getBuffer(),
splitKey.getKeyOffset(),
splitKey.getKeyLength(),
firstKey,
0,
firstKey.length)
< 0) {
return null;
}
}
f.getReader().close(true);
}
Path splitDir = new Path(getSplitsDir(hri), familyName);
// A reference to the bottom half of the hsf store file.
Reference r =
top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow);
// Add the referred-to regions name as a dot separated suffix.
// See REF_NAME_REGEX regex above. The referred-to regions name is
// up in the path of the passed in <code>f</code> -- parentdir is family,
// then the directory above is the region name.
String parentRegionName = regionInfo.getEncodedName();
// Write reference with same file id only with the other region name as
// suffix and into the new region location (under same family).
Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
return r.write(fs, p);
}