Пример #1
0
  /**
   * Creates a 'manifest' for the specified region, by reading directly from the HRegion object.
   * This is used by the "online snapshot" when the table is enabled.
   */
  public void addRegion(final HRegion region) throws IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    RegionVisitor visitor = createRegionVisitor(desc);

    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing '" + region + "' region-info for snapshot.");
    Object regionData = visitor.regionOpen(region.getRegionInfo());
    monitor.rethrowException();

    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");

    for (Store store : region.getStores()) {
      // 2.1. build the snapshot reference for the store
      Object familyData = visitor.familyOpen(regionData, store.getFamily().getName());
      monitor.rethrowException();

      List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles());
      if (LOG.isDebugEnabled()) {
        LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
      }

      // 2.2. iterate through all the store's files and create "references".
      for (int i = 0, sz = storeFiles.size(); i < sz; i++) {
        StoreFile storeFile = storeFiles.get(i);
        monitor.rethrowException();

        // create "reference" to this store file.
        LOG.debug("Adding reference for file (" + (i + 1) + "/" + sz + "): " + storeFile.getPath());
        visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
      }
      visitor.familyClose(regionData, familyData);
    }
    visitor.regionClose(regionData);
  }
Пример #2
0
 /**
  * Recalculate the size of the compaction based on current files.
  *
  * @param files files that should be included in the compaction
  */
 private void recalculateSize() {
   long sz = 0;
   for (StoreFile sf : this.filesToCompact) {
     StoreFileReader r = sf.getReader();
     sz += r == null ? 0 : r.length();
   }
   this.totalSize = sz;
 }
  /**
   * Create a new {@link Reference} as copy of the source one.
   *
   * <p>
   *
   * <blockquote>
   *
   * <pre>
   * The source table looks like:
   *    1234/abc      (original file)
   *    5678/abc.1234 (reference file)
   *
   * After the clone operation looks like:
   *   wxyz/table=1234-abc
   *   stuv/table=1234-abc.wxyz
   *
   * NOTE that the region name in the clone changes (md5 of regioninfo)
   * and the reference should reflect that change.
   * </pre>
   *
   * </blockquote>
   *
   * @param familyDir destination directory for the store file
   * @param regionInfo destination region info for the table
   * @param hfileName reference file name
   */
  private void restoreReferenceFile(
      final Path familyDir, final HRegionInfo regionInfo, final String hfileName)
      throws IOException {
    // Extract the referred information (hfile name and parent region)
    String tableName = snapshotDesc.getTable();
    Path refPath =
        StoreFile.getReferredToFile(
            new Path(
                new Path(new Path(tableName, regionInfo.getEncodedName()), familyDir.getName()),
                hfileName));
    String snapshotRegionName = refPath.getParent().getParent().getName();
    String fileName = refPath.getName();

    // The new reference should have the cloned region name as parent, if it is a clone.
    String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
    if (clonedRegionName == null) clonedRegionName = snapshotRegionName;

    // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
    String refLink = fileName;
    if (!HFileLink.isHFileLink(fileName)) {
      refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
    }
    Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);

    // Create the new reference
    Path linkPath =
        new Path(
            familyDir,
            HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
    InputStream in = new HFileLink(conf, linkPath).open(fs);
    OutputStream out = fs.create(outPath);
    IOUtils.copyBytes(in, out, conf);
  }
 /**
  * Create a new {@link HFileLink} to reference the store file.
  *
  * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
  *
  * <ul>
  *   <li>hfile: abc -> table=region-abc
  *   <li>reference: abc.1234 -> table=region-abc.1234
  *   <li>hfilelink: table=region-hfile -> table=region-hfile
  * </ul>
  *
  * @param familyDir destination directory for the store file
  * @param regionInfo destination region info for the table
  * @param hfileName store file name (can be a Reference, HFileLink or simple HFile)
  */
 private void restoreStoreFile(
     final Path familyDir, final HRegionInfo regionInfo, final String hfileName)
     throws IOException {
   if (HFileLink.isHFileLink(hfileName)) {
     HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
   } else if (StoreFile.isReference(hfileName)) {
     restoreReferenceFile(familyDir, regionInfo, hfileName);
   } else {
     HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
   }
 }
 @Override
 public void postFlush(
     ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile)
     throws IOException {
   // Register HFiles for incremental backup
   SpliceLogUtils.info(LOG, "Flushing region %s.%s", tableName, regionName);
   try {
     if (namespace.compareTo("splice") != 0) return;
     BackupUtils.captureIncrementalChanges(
         conf,
         region,
         path,
         fs,
         rootDir,
         backupDir,
         tableName,
         resultFile.getPath().getName(),
         preparing);
   } catch (Exception ex) {
     throw new IOException(ex);
   }
 }
Пример #6
0
  /**
   * Extracts some details about the files to compact that are commonly needed by compactors.
   *
   * @param filesToCompact Files.
   * @param allFiles Whether all files are included for compaction
   * @return The result.
   */
  protected FileDetails getFileDetails(Collection<StoreFile> filesToCompact, boolean allFiles)
      throws IOException {
    FileDetails fd = new FileDetails();
    long oldestHFileTimeStampToKeepMVCC =
        System.currentTimeMillis() - (1000L * 60 * 60 * 24 * this.keepSeqIdPeriod);

    for (StoreFile file : filesToCompact) {
      if (allFiles && (file.getModificationTimeStamp() < oldestHFileTimeStampToKeepMVCC)) {
        // when isAllFiles is true, all files are compacted so we can calculate the smallest
        // MVCC value to keep
        if (fd.minSeqIdToKeep < file.getMaxMemstoreTS()) {
          fd.minSeqIdToKeep = file.getMaxMemstoreTS();
        }
      }
      long seqNum = file.getMaxSequenceId();
      fd.maxSeqId = Math.max(fd.maxSeqId, seqNum);
      StoreFile.Reader r = file.getReader();
      if (r == null) {
        LOG.warn("Null reader for " + file.getPath());
        continue;
      }
      // NOTE: getFilterEntries could cause under-sized blooms if the user
      // switches bloom type (e.g. from ROW to ROWCOL)
      long keyCount =
          (r.getBloomFilterType() == store.getFamily().getBloomFilterType())
              ? r.getFilterEntries()
              : r.getEntries();
      fd.maxKeyCount += keyCount;
      // calculate the latest MVCC readpoint in any of the involved store files
      Map<byte[], byte[]> fileInfo = r.loadFileInfo();
      byte tmp[] = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
      if (tmp != null) {
        fd.maxMVCCReadpoint = Math.max(fd.maxMVCCReadpoint, Bytes.toLong(tmp));
      }
      tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
      if (tmp != null) {
        fd.maxTagsLength = Math.max(fd.maxTagsLength, Bytes.toInt(tmp));
      }
      // If required, calculate the earliest put timestamp of all involved storefiles.
      // This is used to remove family delete marker during compaction.
      long earliestPutTs = 0;
      if (allFiles) {
        tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
        if (tmp == null) {
          // There's a file with no information, must be an old one
          // assume we have very old puts
          fd.earliestPutTs = earliestPutTs = HConstants.OLDEST_TIMESTAMP;
        } else {
          earliestPutTs = Bytes.toLong(tmp);
          fd.earliestPutTs = Math.min(fd.earliestPutTs, earliestPutTs);
        }
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug(
            "Compacting "
                + file
                + ", keycount="
                + keyCount
                + ", bloomtype="
                + r.getBloomFilterType().toString()
                + ", size="
                + StringUtils.humanReadableInt(r.length())
                + ", encoding="
                + r.getHFileReader().getDataBlockEncoding()
                + ", seqNum="
                + seqNum
                + (allFiles ? ", earliestPutTs=" + earliestPutTs : ""));
      }
    }
    return fd;
  }