/**
* Creates a 'manifest' for the specified region, by reading directly from the HRegion object.
* This is used by the "online snapshot" when the table is enabled.
*/
public void addRegion(final HRegion region) throws IOException {
// 0. Get the ManifestBuilder/RegionVisitor
RegionVisitor visitor = createRegionVisitor(desc);
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing '" + region + "' region-info for snapshot.");
Object regionData = visitor.regionOpen(region.getRegionInfo());
monitor.rethrowException();
// 2. iterate through all the stores in the region
LOG.debug("Creating references for hfiles");
for (Store store : region.getStores()) {
// 2.1. build the snapshot reference for the store
Object familyData = visitor.familyOpen(regionData, store.getFamily().getName());
monitor.rethrowException();
List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles());
if (LOG.isDebugEnabled()) {
LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
}
// 2.2. iterate through all the store's files and create "references".
for (int i = 0, sz = storeFiles.size(); i < sz; i++) {
StoreFile storeFile = storeFiles.get(i);
monitor.rethrowException();
// create "reference" to this store file.
LOG.debug("Adding reference for file (" + (i + 1) + "/" + sz + "): " + storeFile.getPath());
visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
}
visitor.familyClose(regionData, familyData);
}
visitor.regionClose(regionData);
}
private void addReferenceFiles(
RegionVisitor visitor,
Object regionData,
Object familyData,
Collection<StoreFileInfo> storeFiles,
boolean isMob)
throws IOException {
final String fileType = isMob ? "mob file" : "hfile";
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Adding snapshot references for %s %ss", storeFiles, fileType));
}
int i = 0;
int sz = storeFiles.size();
for (StoreFileInfo storeFile : storeFiles) {
monitor.rethrowException();
LOG.debug(
String.format(
"Adding reference for %s (%d/%d): %s", fileType, ++i, sz, storeFile.getPath()));
// create "reference" to this store file.
visitor.storeFile(regionData, familyData, storeFile);
}
}
public void addMobRegion(HRegionInfo regionInfo, HColumnDescriptor[] hcds) throws IOException {
// 0. Get the ManifestBuilder/RegionVisitor
RegionVisitor visitor = createRegionVisitor(desc);
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot.");
Object regionData = visitor.regionOpen(regionInfo);
monitor.rethrowException();
// 2. iterate through all the stores in the region
LOG.debug("Creating references for mob files");
Path mobRegionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable());
for (HColumnDescriptor hcd : hcds) {
// 2.1. build the snapshot reference for the store if it's a mob store
if (!hcd.isMobEnabled()) {
continue;
}
Object familyData = visitor.familyOpen(regionData, hcd.getName());
monitor.rethrowException();
Path storePath = MobUtils.getMobFamilyPath(mobRegionPath, hcd.getNameAsString());
List<StoreFileInfo> storeFiles = getStoreFiles(storePath);
if (storeFiles == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("No mob files under family: " + hcd.getNameAsString());
}
continue;
}
addReferenceFiles(visitor, regionData, familyData, storeFiles, true);
visitor.familyClose(regionData, familyData);
}
visitor.regionClose(regionData);
}
/**
* Creates a 'manifest' for the specified region, by reading directly from the disk. This is used
* by the "offline snapshot" when the table is disabled.
*/
public void addRegion(final Path tableDir, final HRegionInfo regionInfo) throws IOException {
// 0. Get the ManifestBuilder/RegionVisitor
RegionVisitor visitor = createRegionVisitor(desc);
boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
try {
// Open the RegionFS
HRegionFileSystem regionFs =
HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true);
monitor.rethrowException();
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing region-info for snapshot.");
Object regionData = visitor.regionOpen(regionInfo);
monitor.rethrowException();
// 2. iterate through all the stores in the region
LOG.debug("Creating references for hfiles");
// This ensures that we have an atomic view of the directory as long as we have < ls limit
// (batch size of the files in a directory) on the namenode. Otherwise, we get back the files
// in batches and may miss files being added/deleted. This could be more robust (iteratively
// checking to see if we have all the files until we are sure), but the limit is currently
// 1000 files/batch, far more than the number of store files under a single column family.
Collection<String> familyNames = regionFs.getFamilies();
if (familyNames != null) {
for (String familyName : familyNames) {
Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
monitor.rethrowException();
Collection<StoreFileInfo> storeFiles = null;
if (isMobRegion) {
Path regionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable());
Path storePath = MobUtils.getMobFamilyPath(regionPath, familyName);
storeFiles = getStoreFiles(storePath);
} else {
storeFiles = regionFs.getStoreFiles(familyName);
}
if (storeFiles == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("No files under family: " + familyName);
}
continue;
}
// 2.1. build the snapshot reference for the store
// iterate through all the store's files and create "references".
addReferenceFiles(visitor, regionData, familyData, storeFiles, false);
visitor.familyClose(regionData, familyData);
}
}
visitor.regionClose(regionData);
} catch (IOException e) {
// the mob directory might not be created yet, so do nothing when it is a mob region
if (!isMobRegion) {
throw e;
}
}
}
