/** * Creates a 'manifest' for the specified region, by reading directly from the HRegion object. * This is used by the "online snapshot" when the table is enabled. */ public void addRegion(final HRegion region) throws IOException { // 0. Get the ManifestBuilder/RegionVisitor RegionVisitor visitor = createRegionVisitor(desc); // 1. dump region meta info into the snapshot directory LOG.debug("Storing '" + region + "' region-info for snapshot."); Object regionData = visitor.regionOpen(region.getRegionInfo()); monitor.rethrowException(); // 2. iterate through all the stores in the region LOG.debug("Creating references for hfiles"); for (Store store : region.getStores()) { // 2.1. build the snapshot reference for the store Object familyData = visitor.familyOpen(regionData, store.getFamily().getName()); monitor.rethrowException(); List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles()); if (LOG.isDebugEnabled()) { LOG.debug("Adding snapshot references for " + storeFiles + " hfiles"); } // 2.2. iterate through all the store's files and create "references". for (int i = 0, sz = storeFiles.size(); i < sz; i++) { StoreFile storeFile = storeFiles.get(i); monitor.rethrowException(); // create "reference" to this store file. LOG.debug("Adding reference for file (" + (i + 1) + "/" + sz + "): " + storeFile.getPath()); visitor.storeFile(regionData, familyData, storeFile.getFileInfo()); } visitor.familyClose(regionData, familyData); } visitor.regionClose(regionData); }
public void addMobRegion(HRegionInfo regionInfo, HColumnDescriptor[] hcds) throws IOException { // 0. Get the ManifestBuilder/RegionVisitor RegionVisitor visitor = createRegionVisitor(desc); // 1. dump region meta info into the snapshot directory LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot."); Object regionData = visitor.regionOpen(regionInfo); monitor.rethrowException(); // 2. iterate through all the stores in the region LOG.debug("Creating references for mob files"); Path mobRegionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable()); for (HColumnDescriptor hcd : hcds) { // 2.1. build the snapshot reference for the store if it's a mob store if (!hcd.isMobEnabled()) { continue; } Object familyData = visitor.familyOpen(regionData, hcd.getName()); monitor.rethrowException(); Path storePath = MobUtils.getMobFamilyPath(mobRegionPath, hcd.getNameAsString()); List<StoreFileInfo> storeFiles = getStoreFiles(storePath); if (storeFiles == null) { if (LOG.isDebugEnabled()) { LOG.debug("No mob files under family: " + hcd.getNameAsString()); } continue; } addReferenceFiles(visitor, regionData, familyData, storeFiles, true); visitor.familyClose(regionData, familyData); } visitor.regionClose(regionData); }
/** * Creates a 'manifest' for the specified region, by reading directly from the disk. This is used * by the "offline snapshot" when the table is disabled. */ public void addRegion(final Path tableDir, final HRegionInfo regionInfo) throws IOException { // 0. Get the ManifestBuilder/RegionVisitor RegionVisitor visitor = createRegionVisitor(desc); boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo); try { // Open the RegionFS HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true); monitor.rethrowException(); // 1. dump region meta info into the snapshot directory LOG.debug("Storing region-info for snapshot."); Object regionData = visitor.regionOpen(regionInfo); monitor.rethrowException(); // 2. iterate through all the stores in the region LOG.debug("Creating references for hfiles"); // This ensures that we have an atomic view of the directory as long as we have < ls limit // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files // in batches and may miss files being added/deleted. This could be more robust (iteratively // checking to see if we have all the files until we are sure), but the limit is currently // 1000 files/batch, far more than the number of store files under a single column family. Collection<String> familyNames = regionFs.getFamilies(); if (familyNames != null) { for (String familyName : familyNames) { Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName)); monitor.rethrowException(); Collection<StoreFileInfo> storeFiles = null; if (isMobRegion) { Path regionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable()); Path storePath = MobUtils.getMobFamilyPath(regionPath, familyName); storeFiles = getStoreFiles(storePath); } else { storeFiles = regionFs.getStoreFiles(familyName); } if (storeFiles == null) { if (LOG.isDebugEnabled()) { LOG.debug("No files under family: " + familyName); } continue; } // 2.1. build the snapshot reference for the store // iterate through all the store's files and create "references". addReferenceFiles(visitor, regionData, familyData, storeFiles, false); visitor.familyClose(regionData, familyData); } } visitor.regionClose(regionData); } catch (IOException e) { // the mob directory might not be created yet, so do nothing when it is a mob region if (!isMobRegion) { throw e; } } }