@Override
 public Collection<StoreFile> getUnneededFiles(long maxTs, List<StoreFile> filesCompacting) {
   Collection<StoreFile> expiredStoreFiles = null;
   ImmutableList<StoreFile> files = storefiles;
   // 1) We can never get rid of the last file which has the maximum seqid.
   // 2) Files that are not the latest can't become one due to (1), so the rest are fair game.
   for (int i = 0; i < files.size() - 1; ++i) {
     StoreFile sf = files.get(i);
     long fileTs = sf.getReader().getMaxTimestamp();
     if (fileTs < maxTs && !filesCompacting.contains(sf)) {
       LOG.info(
           "Found an expired store file: "
               + sf.getPath()
               + " whose maxTimeStamp is "
               + fileTs
               + ", which is below "
               + maxTs);
       if (expiredStoreFiles == null) {
         expiredStoreFiles = new ArrayList<StoreFile>();
       }
       expiredStoreFiles.add(sf);
     }
   }
   return expiredStoreFiles;
 }
Ejemplo n.º 2
0
 /**
  * Moves multiple store files to the relative region's family store directory.
  *
  * @param storeFiles list of store files divided by family
  * @throws IOException
  */
 void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
   for (Map.Entry<byte[], List<StoreFile>> es : storeFiles.entrySet()) {
     String familyName = Bytes.toString(es.getKey());
     for (StoreFile sf : es.getValue()) {
       commitStoreFile(familyName, sf.getPath());
     }
   }
 }
Ejemplo n.º 3
0
 private Pair<Path, Path> splitStoreFile(final byte[] family, final StoreFile sf)
     throws IOException {
   if (LOG.isDebugEnabled()) {
     LOG.debug(
         "Splitting started for store file: " + sf.getPath() + " for region: " + this.parent);
   }
   HRegionFileSystem fs = this.parent.getRegionFileSystem();
   String familyName = Bytes.toString(family);
   Path path_a =
       fs.splitStoreFile(
           this.hri_a, familyName, sf, this.splitrow, false, this.parent.getSplitPolicy());
   Path path_b =
       fs.splitStoreFile(
           this.hri_b, familyName, sf, this.splitrow, true, this.parent.getSplitPolicy());
   if (LOG.isDebugEnabled()) {
     LOG.debug(
         "Splitting complete for store file: " + sf.getPath() + " for region: " + this.parent);
   }
   return new Pair<Path, Path>(path_a, path_b);
 }
Ejemplo n.º 4
0
 /**
  * Write out a merge reference under the given merges directory. Package local so it doesnt leak
  * out of regionserver.
  *
  * @param mergedRegion {@link HRegionInfo} of the merged region
  * @param familyName Column Family Name
  * @param f File to create reference.
  * @param mergedDir
  * @return Path to created reference.
  * @throws IOException
  */
 Path mergeStoreFile(
     final HRegionInfo mergedRegion,
     final String familyName,
     final StoreFile f,
     final Path mergedDir)
     throws IOException {
   Path referenceDir = new Path(new Path(mergedDir, mergedRegion.getEncodedName()), familyName);
   // A whole reference to the store file.
   Reference r = Reference.createTopReference(regionInfo.getStartKey());
   // Add the referred-to regions name as a dot separated suffix.
   // See REF_NAME_REGEX regex above. The referred-to regions name is
   // up in the path of the passed in <code>f</code> -- parentdir is family,
   // then the directory above is the region name.
   String mergingRegionName = regionInfo.getEncodedName();
   // Write reference with same file id only with the other region name as
   // suffix and into the new region location (under same family).
   Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName);
   return r.write(fs, p);
 }
 @Override
 public void postFlush(
     ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile)
     throws IOException {
   // Register HFiles for incremental backup
   SpliceLogUtils.info(LOG, "Flushing region %s.%s", tableName, regionName);
   try {
     if (namespace.compareTo("splice") != 0) return;
     BackupUtils.captureIncrementalChanges(
         conf,
         region,
         path,
         fs,
         rootDir,
         backupDir,
         tableName,
         resultFile.getPath().getName(),
         preparing);
   } catch (Exception ex) {
     throw new IOException(ex);
   }
 }
Ejemplo n.º 6
0
  /**
   * Do a minor/major compaction on an explicit set of storefiles from a Store.
   *
   * @param request the requested compaction that contains all necessary information to complete the
   *     compaction (i.e. the store, the files, etc.)
   * @return Product of compaction or null if all cells expired or deleted and nothing made it
   *     through the compaction.
   * @throws IOException
   */
  StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOException {
    // Calculate maximum key count after compaction (for blooms)
    // Also calculate earliest put timestamp if major compaction
    int maxKeyCount = 0;
    long earliestPutTs = HConstants.LATEST_TIMESTAMP;
    long maxMVCCReadpoint = 0;

    // pull out the interesting things from the CR for ease later
    final Store store = request.getStore();
    final boolean majorCompaction = request.isMajor();
    final List<StoreFile> filesToCompact = request.getFiles();

    for (StoreFile file : filesToCompact) {
      StoreFile.Reader r = file.getReader();
      if (r == null) {
        LOG.warn("Null reader for " + file.getPath());
        continue;
      }
      // NOTE: getFilterEntries could cause under-sized blooms if the user
      //       switches bloom type (e.g. from ROW to ROWCOL)
      long keyCount =
          (r.getBloomFilterType() == store.getFamily().getBloomFilterType())
              ? r.getFilterEntries()
              : r.getEntries();
      maxKeyCount += keyCount;
      // Calculate the maximum MVCC readpoint used in any of the involved files
      Map<byte[], byte[]> fileInfo = r.loadFileInfo();
      byte[] tmp = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
      if (tmp != null) {
        maxMVCCReadpoint = Math.max(maxMVCCReadpoint, Bytes.toLong(tmp));
      }
      // For major compactions calculate the earliest put timestamp
      // of all involved storefiles. This is used to remove
      // family delete marker during the compaction.
      if (majorCompaction) {
        tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
        if (tmp == null) {
          // There's a file with no information, must be an old one
          // assume we have very old puts
          earliestPutTs = HConstants.OLDEST_TIMESTAMP;
        } else {
          earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
        }
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug(
            "Compacting "
                + file
                + ", keycount="
                + keyCount
                + ", bloomtype="
                + r.getBloomFilterType().toString()
                + ", size="
                + StringUtils.humanReadableInt(r.length())
                + ", encoding="
                + r.getHFileReader().getEncodingOnDisk()
                + (majorCompaction ? ", earliestPutTs=" + earliestPutTs : ""));
      }
    }

    // keep track of compaction progress
    this.progress = new CompactionProgress(maxKeyCount);
    // Get some configs
    int compactionKVMax = getConf().getInt("hbase.hstore.compaction.kv.max", 10);
    Compression.Algorithm compression = store.getFamily().getCompression();
    // Avoid overriding compression setting for major compactions if the user
    // has not specified it separately
    Compression.Algorithm compactionCompression =
        (store.getFamily().getCompactionCompression() != Compression.Algorithm.NONE)
            ? store.getFamily().getCompactionCompression()
            : compression;

    // For each file, obtain a scanner:
    List<StoreFileScanner> scanners =
        StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, false, true);

    // Make the instantiation lazy in case compaction produces no product; i.e.
    // where all source cells are expired or deleted.
    StoreFile.Writer writer = null;
    // Find the smallest read point across all the Scanners.
    long smallestReadPoint = store.getHRegion().getSmallestReadPoint();
    MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
    try {
      InternalScanner scanner = null;
      try {
        if (store.getHRegion().getCoprocessorHost() != null) {
          scanner =
              store
                  .getHRegion()
                  .getCoprocessorHost()
                  .preCompactScannerOpen(
                      store,
                      scanners,
                      majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
                      earliestPutTs,
                      request);
        }
        if (scanner == null) {
          Scan scan = new Scan();
          scan.setMaxVersions(store.getFamily().getMaxVersions());
          /* Include deletes, unless we are doing a major compaction */
          scanner =
              new StoreScanner(
                  store,
                  store.getScanInfo(),
                  scan,
                  scanners,
                  majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
                  smallestReadPoint,
                  earliestPutTs);
        }
        if (store.getHRegion().getCoprocessorHost() != null) {
          InternalScanner cpScanner =
              store.getHRegion().getCoprocessorHost().preCompact(store, scanner, request);
          // NULL scanner returned from coprocessor hooks means skip normal processing
          if (cpScanner == null) {
            return null;
          }
          scanner = cpScanner;
        }

        int bytesWritten = 0;
        // since scanner.next() can return 'false' but still be delivering data,
        // we have to use a do/while loop.
        List<KeyValue> kvs = new ArrayList<KeyValue>();
        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
        boolean hasMore;
        do {
          hasMore = scanner.next(kvs, compactionKVMax);
          if (writer == null && !kvs.isEmpty()) {
            writer =
                store.createWriterInTmp(
                    maxKeyCount,
                    compactionCompression,
                    true,
                    maxMVCCReadpoint >= smallestReadPoint);
          }
          if (writer != null) {
            // output to writer:
            for (KeyValue kv : kvs) {
              if (kv.getMemstoreTS() <= smallestReadPoint) {
                kv.setMemstoreTS(0);
              }
              writer.append(kv);
              // update progress per key
              ++progress.currentCompactedKVs;

              // check periodically to see if a system stop is requested
              if (Store.closeCheckInterval > 0) {
                bytesWritten += kv.getLength();
                if (bytesWritten > Store.closeCheckInterval) {
                  bytesWritten = 0;
                  isInterrupted(store, writer);
                }
              }
            }
          }
          kvs.clear();
        } while (hasMore);
      } finally {
        if (scanner != null) {
          scanner.close();
        }
      }
    } finally {
      if (writer != null) {
        writer.appendMetadata(maxId, majorCompaction);
        writer.close();
      }
    }
    return writer;
  }
Ejemplo n.º 7
0
  /**
   * Write out a split reference. Package local so it doesnt leak out of regionserver.
   *
   * @param hri {@link HRegionInfo} of the destination
   * @param familyName Column Family Name
   * @param f File to split.
   * @param splitRow Split Row
   * @param top True if we are referring to the top half of the hfile.
   * @return Path to created reference.
   * @throws IOException
   */
  Path splitStoreFile(
      final HRegionInfo hri,
      final String familyName,
      final StoreFile f,
      final byte[] splitRow,
      final boolean top)
      throws IOException {

    // Check whether the split row lies in the range of the store file
    // If it is outside the range, return directly.
    if (!isIndexTable()) {
      if (top) {
        // check if larger than last key.
        KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
        byte[] lastKey = f.createReader().getLastKey();
        // If lastKey is null means storefile is empty.
        if (lastKey == null) return null;
        if (f.getReader()
                .getComparator()
                .compareFlatKey(
                    splitKey.getBuffer(),
                    splitKey.getKeyOffset(),
                    splitKey.getKeyLength(),
                    lastKey,
                    0,
                    lastKey.length)
            > 0) {
          return null;
        }
      } else {
        // check if smaller than first key
        KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
        byte[] firstKey = f.createReader().getFirstKey();
        // If firstKey is null means storefile is empty.
        if (firstKey == null) return null;
        if (f.getReader()
                .getComparator()
                .compareFlatKey(
                    splitKey.getBuffer(),
                    splitKey.getKeyOffset(),
                    splitKey.getKeyLength(),
                    firstKey,
                    0,
                    firstKey.length)
            < 0) {
          return null;
        }
      }

      f.getReader().close(true);
    }

    Path splitDir = new Path(getSplitsDir(hri), familyName);
    // A reference to the bottom half of the hsf store file.
    Reference r =
        top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow);
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above.  The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String parentRegionName = regionInfo.getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
    return r.write(fs, p);
  }