@Override
 public Collection<StoreFile> getUnneededFiles(long maxTs, List<StoreFile> filesCompacting) {
   Collection<StoreFile> expiredStoreFiles = null;
   ImmutableList<StoreFile> files = storefiles;
   // 1) We can never get rid of the last file which has the maximum seqid.
   // 2) Files that are not the latest can't become one due to (1), so the rest are fair game.
   for (int i = 0; i < files.size() - 1; ++i) {
     StoreFile sf = files.get(i);
     long fileTs = sf.getReader().getMaxTimestamp();
     if (fileTs < maxTs && !filesCompacting.contains(sf)) {
       LOG.info(
           "Found an expired store file: "
               + sf.getPath()
               + " whose maxTimeStamp is "
               + fileTs
               + ", which is below "
               + maxTs);
       if (expiredStoreFiles == null) {
         expiredStoreFiles = new ArrayList<StoreFile>();
       }
       expiredStoreFiles.add(sf);
     }
   }
   return expiredStoreFiles;
 }
  /**
   * @return the total key count in the files being merged
   * @throws IOException
   */
  private long prepareForMerge() throws IOException {
    LOG.info("Merging " + inputFileNames);
    LOG.info("Using block size: " + blockSize);
    inputStoreFiles = new ArrayList<StoreFile>();

    long maxKeyCount = 0;
    for (String fileName : inputFileNames) {
      Path filePath = new Path(fileName);

      // Open without caching.
      StoreFile sf = openStoreFile(filePath, false);
      sf.createReader();
      inputStoreFiles.add(sf);

      StoreFile.Reader r = sf.getReader();
      if (r != null) {
        long keyCount = r.getFilterEntries();
        maxKeyCount += keyCount;
        LOG.info(
            "Compacting: "
                + sf
                + "; keyCount = "
                + keyCount
                + "; Bloom Type = "
                + r.getBloomFilterType().toString()
                + "; Size = "
                + StringUtils.humanReadableInt(r.length()));
      }
    }
    return maxKeyCount;
  }
Esempio n. 3
0
 /**
  * Moves multiple store files to the relative region's family store directory.
  *
  * @param storeFiles list of store files divided by family
  * @throws IOException
  */
 void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
   for (Map.Entry<byte[], List<StoreFile>> es : storeFiles.entrySet()) {
     String familyName = Bytes.toString(es.getKey());
     for (StoreFile sf : es.getValue()) {
       commitStoreFile(familyName, sf.getPath());
     }
   }
 }
Esempio n. 4
0
 private static CompactionRequest createDummyRequest() throws Exception {
   // "Files" are totally unused, it's Scanner class below that gives compactor fake KVs.
   // But compaction depends on everything under the sun, so stub everything with dummies.
   StoreFile sf = mock(StoreFile.class);
   StoreFile.Reader r = mock(StoreFile.Reader.class);
   when(r.length()).thenReturn(1L);
   when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
   when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
   when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong()))
       .thenReturn(mock(StoreFileScanner.class));
   when(sf.getReader()).thenReturn(r);
   when(sf.createReader()).thenReturn(r);
   return new CompactionRequest(Arrays.asList(sf));
 }
Esempio n. 5
0
 /**
  * Return an array of scanners corresponding to the given set of store files, And set the
  * ScanQueryMatcher for each store file scanner for further optimization
  */
 public static List<StoreFileScanner> getScannersForStoreFiles(
     Collection<StoreFile> files,
     boolean cacheBlocks,
     boolean isCompaction,
     ScanQueryMatcher matcher)
     throws IOException {
   List<StoreFileScanner> scanners = new ArrayList<StoreFileScanner>(files.size());
   for (StoreFile file : files) {
     StoreFile.Reader r = file.createReader();
     StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, isCompaction);
     scanner.setScanQueryMatcher(matcher);
     scanners.add(scanner);
   }
   return scanners;
 }
Esempio n. 6
0
 private Pair<Path, Path> splitStoreFile(final byte[] family, final StoreFile sf)
     throws IOException {
   if (LOG.isDebugEnabled()) {
     LOG.debug(
         "Splitting started for store file: " + sf.getPath() + " for region: " + this.parent);
   }
   HRegionFileSystem fs = this.parent.getRegionFileSystem();
   String familyName = Bytes.toString(family);
   Path path_a =
       fs.splitStoreFile(
           this.hri_a, familyName, sf, this.splitrow, false, this.parent.getSplitPolicy());
   Path path_b =
       fs.splitStoreFile(
           this.hri_b, familyName, sf, this.splitrow, true, this.parent.getSplitPolicy());
   if (LOG.isDebugEnabled()) {
     LOG.debug(
         "Splitting complete for store file: " + sf.getPath() + " for region: " + this.parent);
   }
   return new Pair<Path, Path>(path_a, path_b);
 }
Esempio n. 7
0
 /**
  * Write out a merge reference under the given merges directory. Package local so it doesnt leak
  * out of regionserver.
  *
  * @param mergedRegion {@link HRegionInfo} of the merged region
  * @param familyName Column Family Name
  * @param f File to create reference.
  * @param mergedDir
  * @return Path to created reference.
  * @throws IOException
  */
 Path mergeStoreFile(
     final HRegionInfo mergedRegion,
     final String familyName,
     final StoreFile f,
     final Path mergedDir)
     throws IOException {
   Path referenceDir = new Path(new Path(mergedDir, mergedRegion.getEncodedName()), familyName);
   // A whole reference to the store file.
   Reference r = Reference.createTopReference(regionInfo.getStartKey());
   // Add the referred-to regions name as a dot separated suffix.
   // See REF_NAME_REGEX regex above. The referred-to regions name is
   // up in the path of the passed in <code>f</code> -- parentdir is family,
   // then the directory above is the region name.
   String mergingRegionName = regionInfo.getEncodedName();
   // Write reference with same file id only with the other region name as
   // suffix and into the new region location (under same family).
   Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName);
   return r.write(fs, p);
 }
 @Override
 public void postFlush(
     ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile)
     throws IOException {
   // Register HFiles for incremental backup
   SpliceLogUtils.info(LOG, "Flushing region %s.%s", tableName, regionName);
   try {
     if (namespace.compareTo("splice") != 0) return;
     BackupUtils.captureIncrementalChanges(
         conf,
         region,
         path,
         fs,
         rootDir,
         backupDir,
         tableName,
         resultFile.getPath().getName(),
         preparing);
   } catch (Exception ex) {
     throw new IOException(ex);
   }
 }
  @Override
  public void setUp() throws Exception {
    // setup config values necessary for store
    this.conf = TEST_UTIL.getConfiguration();
    this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
    this.conf.setInt("hbase.hstore.compaction.min", minFiles);
    this.conf.setInt("hbase.hstore.compaction.max", maxFiles);
    this.conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, minSize);
    this.conf.setLong("hbase.hstore.compaction.max.size", maxSize);
    this.conf.setFloat("hbase.hstore.compaction.ratio", 1.0F);

    // Setting up a Store
    Path basedir = new Path(DIR);
    String logName = "logs";
    Path logdir = new Path(DIR, logName);
    Path oldLogDir = new Path(basedir, HConstants.HREGION_OLDLOGDIR_NAME);
    HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family"));
    FileSystem fs = FileSystem.get(conf);

    fs.delete(logdir, true);

    HTableDescriptor htd = new HTableDescriptor(Bytes.toBytes("table"));
    htd.addFamily(hcd);
    HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);

    hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
    region = HRegion.createHRegion(info, basedir, conf, htd);
    HRegion.closeHRegion(region);
    Path tableDir = new Path(basedir, Bytes.toString(htd.getName()));
    region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);

    store = new HStore(basedir, region, hcd, fs, conf);
    manager = store.compactionPolicy;

    TEST_FILE = StoreFile.getRandomFilename(fs, store.getHomedir());
    fs.create(TEST_FILE);
  }
  public boolean runRandomReadWorkload() throws IOException {
    if (inputFileNames.size() != 1) {
      throw new IOException("Need exactly one input file for random reads: " + inputFileNames);
    }

    Path inputPath = new Path(inputFileNames.get(0));

    // Make sure we are using caching.
    StoreFile storeFile = openStoreFile(inputPath, true);

    StoreFile.Reader reader = storeFile.createReader();

    LOG.info("First key: " + Bytes.toStringBinary(reader.getFirstKey()));
    LOG.info("Last key: " + Bytes.toStringBinary(reader.getLastKey()));

    KeyValue firstKV = KeyValue.createKeyValueFromKey(reader.getFirstKey());
    firstRow = firstKV.getRow();

    KeyValue lastKV = KeyValue.createKeyValueFromKey(reader.getLastKey());
    lastRow = lastKV.getRow();

    byte[] family = firstKV.getFamily();
    if (!Bytes.equals(family, lastKV.getFamily())) {
      LOG.error(
          "First and last key have different families: "
              + Bytes.toStringBinary(family)
              + " and "
              + Bytes.toStringBinary(lastKV.getFamily()));
      return false;
    }

    if (Bytes.equals(firstRow, lastRow)) {
      LOG.error(
          "First and last row are the same, cannot run read workload: "
              + "firstRow="
              + Bytes.toStringBinary(firstRow)
              + ", "
              + "lastRow="
              + Bytes.toStringBinary(lastRow));
      return false;
    }

    ExecutorService exec = Executors.newFixedThreadPool(numReadThreads + 1);
    int numCompleted = 0;
    int numFailed = 0;
    try {
      ExecutorCompletionService<Boolean> ecs = new ExecutorCompletionService<Boolean>(exec);
      endTime = System.currentTimeMillis() + 1000 * durationSec;
      boolean pread = true;
      for (int i = 0; i < numReadThreads; ++i) ecs.submit(new RandomReader(i, reader, pread));
      ecs.submit(new StatisticsPrinter());
      Future<Boolean> result;
      while (true) {
        try {
          result = ecs.poll(endTime + 1000 - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
          if (result == null) break;
          try {
            if (result.get()) {
              ++numCompleted;
            } else {
              ++numFailed;
            }
          } catch (ExecutionException e) {
            LOG.error("Worker thread failure", e.getCause());
            ++numFailed;
          }
        } catch (InterruptedException ex) {
          LOG.error("Interrupted after " + numCompleted + " workers completed");
          Thread.currentThread().interrupt();
          continue;
        }
      }
    } finally {
      storeFile.closeReader(true);
      exec.shutdown();

      BlockCache c = cacheConf.getBlockCache();
      if (c != null) {
        c.shutdown();
      }
    }
    LOG.info("Worker threads completed: " + numCompleted);
    LOG.info("Worker threads failed: " + numFailed);
    return true;
  }
 // Mark the files as compactedAway once the storefiles and compactedfiles list is finalised
 // Let a background thread close the actual reader on these compacted files and also
 // ensure to evict the blocks from block cache so that they are no longer in
 // cache
 private void markCompactedAway(Collection<StoreFile> compactedFiles) {
   for (StoreFile file : compactedFiles) {
     file.markCompactedAway();
   }
 }
  /**
   * Do a minor/major compaction on an explicit set of storefiles from a Store.
   *
   * @param request the requested compaction that contains all necessary information to complete the
   *     compaction (i.e. the store, the files, etc.)
   * @return Product of compaction or null if all cells expired or deleted and nothing made it
   *     through the compaction.
   * @throws IOException
   */
  StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOException {
    // Calculate maximum key count after compaction (for blooms)
    // Also calculate earliest put timestamp if major compaction
    int maxKeyCount = 0;
    long earliestPutTs = HConstants.LATEST_TIMESTAMP;
    long maxMVCCReadpoint = 0;

    // pull out the interesting things from the CR for ease later
    final Store store = request.getStore();
    final boolean majorCompaction = request.isMajor();
    final List<StoreFile> filesToCompact = request.getFiles();

    for (StoreFile file : filesToCompact) {
      StoreFile.Reader r = file.getReader();
      if (r == null) {
        LOG.warn("Null reader for " + file.getPath());
        continue;
      }
      // NOTE: getFilterEntries could cause under-sized blooms if the user
      //       switches bloom type (e.g. from ROW to ROWCOL)
      long keyCount =
          (r.getBloomFilterType() == store.getFamily().getBloomFilterType())
              ? r.getFilterEntries()
              : r.getEntries();
      maxKeyCount += keyCount;
      // Calculate the maximum MVCC readpoint used in any of the involved files
      Map<byte[], byte[]> fileInfo = r.loadFileInfo();
      byte[] tmp = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
      if (tmp != null) {
        maxMVCCReadpoint = Math.max(maxMVCCReadpoint, Bytes.toLong(tmp));
      }
      // For major compactions calculate the earliest put timestamp
      // of all involved storefiles. This is used to remove
      // family delete marker during the compaction.
      if (majorCompaction) {
        tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
        if (tmp == null) {
          // There's a file with no information, must be an old one
          // assume we have very old puts
          earliestPutTs = HConstants.OLDEST_TIMESTAMP;
        } else {
          earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
        }
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug(
            "Compacting "
                + file
                + ", keycount="
                + keyCount
                + ", bloomtype="
                + r.getBloomFilterType().toString()
                + ", size="
                + StringUtils.humanReadableInt(r.length())
                + ", encoding="
                + r.getHFileReader().getEncodingOnDisk()
                + (majorCompaction ? ", earliestPutTs=" + earliestPutTs : ""));
      }
    }

    // keep track of compaction progress
    this.progress = new CompactionProgress(maxKeyCount);
    // Get some configs
    int compactionKVMax = getConf().getInt("hbase.hstore.compaction.kv.max", 10);
    Compression.Algorithm compression = store.getFamily().getCompression();
    // Avoid overriding compression setting for major compactions if the user
    // has not specified it separately
    Compression.Algorithm compactionCompression =
        (store.getFamily().getCompactionCompression() != Compression.Algorithm.NONE)
            ? store.getFamily().getCompactionCompression()
            : compression;

    // For each file, obtain a scanner:
    List<StoreFileScanner> scanners =
        StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, false, true);

    // Make the instantiation lazy in case compaction produces no product; i.e.
    // where all source cells are expired or deleted.
    StoreFile.Writer writer = null;
    // Find the smallest read point across all the Scanners.
    long smallestReadPoint = store.getHRegion().getSmallestReadPoint();
    MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
    try {
      InternalScanner scanner = null;
      try {
        if (store.getHRegion().getCoprocessorHost() != null) {
          scanner =
              store
                  .getHRegion()
                  .getCoprocessorHost()
                  .preCompactScannerOpen(
                      store,
                      scanners,
                      majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
                      earliestPutTs,
                      request);
        }
        if (scanner == null) {
          Scan scan = new Scan();
          scan.setMaxVersions(store.getFamily().getMaxVersions());
          /* Include deletes, unless we are doing a major compaction */
          scanner =
              new StoreScanner(
                  store,
                  store.getScanInfo(),
                  scan,
                  scanners,
                  majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
                  smallestReadPoint,
                  earliestPutTs);
        }
        if (store.getHRegion().getCoprocessorHost() != null) {
          InternalScanner cpScanner =
              store.getHRegion().getCoprocessorHost().preCompact(store, scanner, request);
          // NULL scanner returned from coprocessor hooks means skip normal processing
          if (cpScanner == null) {
            return null;
          }
          scanner = cpScanner;
        }

        int bytesWritten = 0;
        // since scanner.next() can return 'false' but still be delivering data,
        // we have to use a do/while loop.
        List<KeyValue> kvs = new ArrayList<KeyValue>();
        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
        boolean hasMore;
        do {
          hasMore = scanner.next(kvs, compactionKVMax);
          if (writer == null && !kvs.isEmpty()) {
            writer =
                store.createWriterInTmp(
                    maxKeyCount,
                    compactionCompression,
                    true,
                    maxMVCCReadpoint >= smallestReadPoint);
          }
          if (writer != null) {
            // output to writer:
            for (KeyValue kv : kvs) {
              if (kv.getMemstoreTS() <= smallestReadPoint) {
                kv.setMemstoreTS(0);
              }
              writer.append(kv);
              // update progress per key
              ++progress.currentCompactedKVs;

              // check periodically to see if a system stop is requested
              if (Store.closeCheckInterval > 0) {
                bytesWritten += kv.getLength();
                if (bytesWritten > Store.closeCheckInterval) {
                  bytesWritten = 0;
                  isInterrupted(store, writer);
                }
              }
            }
          }
          kvs.clear();
        } while (hasMore);
      } finally {
        if (scanner != null) {
          scanner.close();
        }
      }
    } finally {
      if (writer != null) {
        writer.appendMetadata(maxId, majorCompaction);
        writer.close();
      }
    }
    return writer;
  }
Esempio n. 13
0
  /**
   * Write out a split reference. Package local so it doesnt leak out of regionserver.
   *
   * @param hri {@link HRegionInfo} of the destination
   * @param familyName Column Family Name
   * @param f File to split.
   * @param splitRow Split Row
   * @param top True if we are referring to the top half of the hfile.
   * @return Path to created reference.
   * @throws IOException
   */
  Path splitStoreFile(
      final HRegionInfo hri,
      final String familyName,
      final StoreFile f,
      final byte[] splitRow,
      final boolean top)
      throws IOException {

    // Check whether the split row lies in the range of the store file
    // If it is outside the range, return directly.
    if (!isIndexTable()) {
      if (top) {
        // check if larger than last key.
        KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
        byte[] lastKey = f.createReader().getLastKey();
        // If lastKey is null means storefile is empty.
        if (lastKey == null) return null;
        if (f.getReader()
                .getComparator()
                .compareFlatKey(
                    splitKey.getBuffer(),
                    splitKey.getKeyOffset(),
                    splitKey.getKeyLength(),
                    lastKey,
                    0,
                    lastKey.length)
            > 0) {
          return null;
        }
      } else {
        // check if smaller than first key
        KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
        byte[] firstKey = f.createReader().getFirstKey();
        // If firstKey is null means storefile is empty.
        if (firstKey == null) return null;
        if (f.getReader()
                .getComparator()
                .compareFlatKey(
                    splitKey.getBuffer(),
                    splitKey.getKeyOffset(),
                    splitKey.getKeyLength(),
                    firstKey,
                    0,
                    firstKey.length)
            < 0) {
          return null;
        }
      }

      f.getReader().close(true);
    }

    Path splitDir = new Path(getSplitsDir(hri), familyName);
    // A reference to the bottom half of the hsf store file.
    Reference r =
        top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow);
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above.  The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String parentRegionName = regionInfo.getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
    return r.write(fs, p);
  }