Beispiel #1
0
  /**
   * Creates a 'manifest' for the specified region, by reading directly from the HRegion object.
   * This is used by the "online snapshot" when the table is enabled.
   */
  public void addRegion(final HRegion region) throws IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    RegionVisitor visitor = createRegionVisitor(desc);

    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing '" + region + "' region-info for snapshot.");
    Object regionData = visitor.regionOpen(region.getRegionInfo());
    monitor.rethrowException();

    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");

    for (Store store : region.getStores()) {
      // 2.1. build the snapshot reference for the store
      Object familyData = visitor.familyOpen(regionData, store.getFamily().getName());
      monitor.rethrowException();

      List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles());
      if (LOG.isDebugEnabled()) {
        LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
      }

      // 2.2. iterate through all the store's files and create "references".
      for (int i = 0, sz = storeFiles.size(); i < sz; i++) {
        StoreFile storeFile = storeFiles.get(i);
        monitor.rethrowException();

        // create "reference" to this store file.
        LOG.debug("Adding reference for file (" + (i + 1) + "/" + sz + "): " + storeFile.getPath());
        visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
      }
      visitor.familyClose(regionData, familyData);
    }
    visitor.regionClose(regionData);
  }
Beispiel #2
0
  private void addReferenceFiles(
      RegionVisitor visitor,
      Object regionData,
      Object familyData,
      Collection<StoreFileInfo> storeFiles,
      boolean isMob)
      throws IOException {
    final String fileType = isMob ? "mob file" : "hfile";

    if (LOG.isDebugEnabled()) {
      LOG.debug(String.format("Adding snapshot references for %s %ss", storeFiles, fileType));
    }

    int i = 0;
    int sz = storeFiles.size();
    for (StoreFileInfo storeFile : storeFiles) {
      monitor.rethrowException();

      LOG.debug(
          String.format(
              "Adding reference for %s (%d/%d): %s", fileType, ++i, sz, storeFile.getPath()));

      // create "reference" to this store file.
      visitor.storeFile(regionData, familyData, storeFile);
    }
  }
Beispiel #3
0
  public void addMobRegion(HRegionInfo regionInfo, HColumnDescriptor[] hcds) throws IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    RegionVisitor visitor = createRegionVisitor(desc);

    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot.");
    Object regionData = visitor.regionOpen(regionInfo);
    monitor.rethrowException();

    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for mob files");

    Path mobRegionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable());
    for (HColumnDescriptor hcd : hcds) {
      // 2.1. build the snapshot reference for the store if it's a mob store
      if (!hcd.isMobEnabled()) {
        continue;
      }
      Object familyData = visitor.familyOpen(regionData, hcd.getName());
      monitor.rethrowException();

      Path storePath = MobUtils.getMobFamilyPath(mobRegionPath, hcd.getNameAsString());
      List<StoreFileInfo> storeFiles = getStoreFiles(storePath);
      if (storeFiles == null) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("No mob files under family: " + hcd.getNameAsString());
        }
        continue;
      }

      addReferenceFiles(visitor, regionData, familyData, storeFiles, true);

      visitor.familyClose(regionData, familyData);
    }
    visitor.regionClose(regionData);
  }
Beispiel #4
0
  /**
   * Creates a 'manifest' for the specified region, by reading directly from the disk. This is used
   * by the "offline snapshot" when the table is disabled.
   */
  public void addRegion(final Path tableDir, final HRegionInfo regionInfo) throws IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    RegionVisitor visitor = createRegionVisitor(desc);

    boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
    try {
      // Open the RegionFS
      HRegionFileSystem regionFs =
          HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true);
      monitor.rethrowException();

      // 1. dump region meta info into the snapshot directory
      LOG.debug("Storing region-info for snapshot.");
      Object regionData = visitor.regionOpen(regionInfo);
      monitor.rethrowException();

      // 2. iterate through all the stores in the region
      LOG.debug("Creating references for hfiles");

      // This ensures that we have an atomic view of the directory as long as we have < ls limit
      // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files
      // in batches and may miss files being added/deleted. This could be more robust (iteratively
      // checking to see if we have all the files until we are sure), but the limit is currently
      // 1000 files/batch, far more than the number of store files under a single column family.
      Collection<String> familyNames = regionFs.getFamilies();
      if (familyNames != null) {
        for (String familyName : familyNames) {
          Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
          monitor.rethrowException();

          Collection<StoreFileInfo> storeFiles = null;
          if (isMobRegion) {
            Path regionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable());
            Path storePath = MobUtils.getMobFamilyPath(regionPath, familyName);
            storeFiles = getStoreFiles(storePath);
          } else {
            storeFiles = regionFs.getStoreFiles(familyName);
          }

          if (storeFiles == null) {
            if (LOG.isDebugEnabled()) {
              LOG.debug("No files under family: " + familyName);
            }
            continue;
          }

          // 2.1. build the snapshot reference for the store
          // iterate through all the store's files and create "references".
          addReferenceFiles(visitor, regionData, familyData, storeFiles, false);

          visitor.familyClose(regionData, familyData);
        }
      }
      visitor.regionClose(regionData);
    } catch (IOException e) {
      // the mob directory might not be created yet, so do nothing when it is a mob region
      if (!isMobRegion) {
        throw e;
      }
    }
  }