/**
* Creates a 'manifest' for the specified region, by reading directly from the HRegion object.
* This is used by the "online snapshot" when the table is enabled.
*/
public void addRegion(final HRegion region) throws IOException {
// 0. Get the ManifestBuilder/RegionVisitor
RegionVisitor visitor = createRegionVisitor(desc);
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing '" + region + "' region-info for snapshot.");
Object regionData = visitor.regionOpen(region.getRegionInfo());
monitor.rethrowException();
// 2. iterate through all the stores in the region
LOG.debug("Creating references for hfiles");
for (Store store : region.getStores()) {
// 2.1. build the snapshot reference for the store
Object familyData = visitor.familyOpen(regionData, store.getFamily().getName());
monitor.rethrowException();
List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles());
if (LOG.isDebugEnabled()) {
LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
}
// 2.2. iterate through all the store's files and create "references".
for (int i = 0, sz = storeFiles.size(); i < sz; i++) {
StoreFile storeFile = storeFiles.get(i);
monitor.rethrowException();
// create "reference" to this store file.
LOG.debug("Adding reference for file (" + (i + 1) + "/" + sz + "): " + storeFile.getPath());
visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
}
visitor.familyClose(regionData, familyData);
}
visitor.regionClose(regionData);
}
/**
* Recalculate the size of the compaction based on current files.
*
* @param files files that should be included in the compaction
*/
private void recalculateSize() {
long sz = 0;
for (StoreFile sf : this.filesToCompact) {
StoreFileReader r = sf.getReader();
sz += r == null ? 0 : r.length();
}
this.totalSize = sz;
}
/**
* Create a new {@link Reference} as copy of the source one.
*
* <p>
*
* <blockquote>
*
* <pre>
* The source table looks like:
* 1234/abc (original file)
* 5678/abc.1234 (reference file)
*
* After the clone operation looks like:
* wxyz/table=1234-abc
* stuv/table=1234-abc.wxyz
*
* NOTE that the region name in the clone changes (md5 of regioninfo)
* and the reference should reflect that change.
* </pre>
*
* </blockquote>
*
* @param familyDir destination directory for the store file
* @param regionInfo destination region info for the table
* @param hfileName reference file name
*/
private void restoreReferenceFile(
final Path familyDir, final HRegionInfo regionInfo, final String hfileName)
throws IOException {
// Extract the referred information (hfile name and parent region)
String tableName = snapshotDesc.getTable();
Path refPath =
StoreFile.getReferredToFile(
new Path(
new Path(new Path(tableName, regionInfo.getEncodedName()), familyDir.getName()),
hfileName));
String snapshotRegionName = refPath.getParent().getParent().getName();
String fileName = refPath.getName();
// The new reference should have the cloned region name as parent, if it is a clone.
String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
// The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
String refLink = fileName;
if (!HFileLink.isHFileLink(fileName)) {
refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
}
Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
// Create the new reference
Path linkPath =
new Path(
familyDir,
HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
InputStream in = new HFileLink(conf, linkPath).open(fs);
OutputStream out = fs.create(outPath);
IOUtils.copyBytes(in, out, conf);
}
/**
* Create a new {@link HFileLink} to reference the store file.
*
* <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
*
* <ul>
* <li>hfile: abc -> table=region-abc
* <li>reference: abc.1234 -> table=region-abc.1234
* <li>hfilelink: table=region-hfile -> table=region-hfile
* </ul>
*
* @param familyDir destination directory for the store file
* @param regionInfo destination region info for the table
* @param hfileName store file name (can be a Reference, HFileLink or simple HFile)
*/
private void restoreStoreFile(
final Path familyDir, final HRegionInfo regionInfo, final String hfileName)
throws IOException {
if (HFileLink.isHFileLink(hfileName)) {
HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
} else if (StoreFile.isReference(hfileName)) {
restoreReferenceFile(familyDir, regionInfo, hfileName);
} else {
HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
}
}
@Override
public void postFlush(
ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile)
throws IOException {
// Register HFiles for incremental backup
SpliceLogUtils.info(LOG, "Flushing region %s.%s", tableName, regionName);
try {
if (namespace.compareTo("splice") != 0) return;
BackupUtils.captureIncrementalChanges(
conf,
region,
path,
fs,
rootDir,
backupDir,
tableName,
resultFile.getPath().getName(),
preparing);
} catch (Exception ex) {
throw new IOException(ex);
}
}
/**
* Extracts some details about the files to compact that are commonly needed by compactors.
*
* @param filesToCompact Files.
* @param allFiles Whether all files are included for compaction
* @return The result.
*/
protected FileDetails getFileDetails(Collection<StoreFile> filesToCompact, boolean allFiles)
throws IOException {
FileDetails fd = new FileDetails();
long oldestHFileTimeStampToKeepMVCC =
System.currentTimeMillis() - (1000L * 60 * 60 * 24 * this.keepSeqIdPeriod);
for (StoreFile file : filesToCompact) {
if (allFiles && (file.getModificationTimeStamp() < oldestHFileTimeStampToKeepMVCC)) {
// when isAllFiles is true, all files are compacted so we can calculate the smallest
// MVCC value to keep
if (fd.minSeqIdToKeep < file.getMaxMemstoreTS()) {
fd.minSeqIdToKeep = file.getMaxMemstoreTS();
}
}
long seqNum = file.getMaxSequenceId();
fd.maxSeqId = Math.max(fd.maxSeqId, seqNum);
StoreFile.Reader r = file.getReader();
if (r == null) {
LOG.warn("Null reader for " + file.getPath());
continue;
}
// NOTE: getFilterEntries could cause under-sized blooms if the user
// switches bloom type (e.g. from ROW to ROWCOL)
long keyCount =
(r.getBloomFilterType() == store.getFamily().getBloomFilterType())
? r.getFilterEntries()
: r.getEntries();
fd.maxKeyCount += keyCount;
// calculate the latest MVCC readpoint in any of the involved store files
Map<byte[], byte[]> fileInfo = r.loadFileInfo();
byte tmp[] = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
if (tmp != null) {
fd.maxMVCCReadpoint = Math.max(fd.maxMVCCReadpoint, Bytes.toLong(tmp));
}
tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
if (tmp != null) {
fd.maxTagsLength = Math.max(fd.maxTagsLength, Bytes.toInt(tmp));
}
// If required, calculate the earliest put timestamp of all involved storefiles.
// This is used to remove family delete marker during compaction.
long earliestPutTs = 0;
if (allFiles) {
tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
if (tmp == null) {
// There's a file with no information, must be an old one
// assume we have very old puts
fd.earliestPutTs = earliestPutTs = HConstants.OLDEST_TIMESTAMP;
} else {
earliestPutTs = Bytes.toLong(tmp);
fd.earliestPutTs = Math.min(fd.earliestPutTs, earliestPutTs);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Compacting "
+ file
+ ", keycount="
+ keyCount
+ ", bloomtype="
+ r.getBloomFilterType().toString()
+ ", size="
+ StringUtils.humanReadableInt(r.length())
+ ", encoding="
+ r.getHFileReader().getDataBlockEncoding()
+ ", seqNum="
+ seqNum
+ (allFiles ? ", earliestPutTs=" + earliestPutTs : ""));
}
}
return fd;
}