Example #1
0
  private void finishCompactionOutputFile(CompactionState compactionState) throws IOException {
    Preconditions.checkNotNull(compactionState, "compactionState is null");
    Preconditions.checkArgument(compactionState.outfile != null);
    Preconditions.checkArgument(compactionState.builder != null);

    long outputNumber = compactionState.currentFileNumber;
    Preconditions.checkArgument(outputNumber != 0);

    long currentEntries = compactionState.builder.getEntryCount();
    compactionState.builder.finish();

    long currentBytes = compactionState.builder.getFileSize();
    compactionState.currentFileSize = currentBytes;
    compactionState.totalBytes += currentBytes;

    FileMetaData currentFileMetaData =
        new FileMetaData(
            compactionState.currentFileNumber,
            compactionState.currentFileSize,
            compactionState.currentSmallest,
            compactionState.currentLargest);
    compactionState.outputs.add(currentFileMetaData);

    compactionState.builder = null;

    compactionState.outfile.force(true);
    compactionState.outfile.close();
    compactionState.outfile = null;

    if (currentEntries > 0) {
      // Verify that the table is usable
      tableCache.newIterator(outputNumber);
    }
  }
Example #2
0
  private void openCompactionOutputFile(CompactionState compactionState)
      throws FileNotFoundException {
    Preconditions.checkNotNull(compactionState, "compactionState is null");
    Preconditions.checkArgument(
        compactionState.builder == null, "compactionState builder is not null");

    mutex.lock();
    try {
      long fileNumber = versions.getNextFileNumber();
      pendingOutputs.add(fileNumber);
      compactionState.currentFileNumber = fileNumber;
      compactionState.currentFileSize = 0;
      compactionState.currentSmallest = null;
      compactionState.currentLargest = null;

      File file = new File(databaseDir, Filename.tableFileName(fileNumber));
      compactionState.outfile = new FileOutputStream(file).getChannel();
      compactionState.builder =
          new TableBuilder(
              options, compactionState.outfile, new InternalUserComparator(internalKeyComparator));
    } finally {
      mutex.unlock();
    }
  }
Example #3
0
  private void doCompactionWork(CompactionState compactionState) throws IOException {
    Preconditions.checkState(mutex.isHeldByCurrentThread());
    Preconditions.checkArgument(
        versions.numberOfBytesInLevel(compactionState.getCompaction().getLevel()) > 0);
    Preconditions.checkArgument(compactionState.builder == null);
    Preconditions.checkArgument(compactionState.outfile == null);

    // todo track snapshots
    compactionState.smallestSnapshot = versions.getLastSequence();

    // Release mutex while we're actually doing the compaction work
    mutex.unlock();
    try {
      MergingIterator iterator = versions.makeInputIterator(compactionState.compaction);

      Slice currentUserKey = null;
      boolean hasCurrentUserKey = false;

      long lastSequenceForKey = MAX_SEQUENCE_NUMBER;
      while (iterator.hasNext() && !shuttingDown.get()) {
        // always give priority to compacting the current mem table
        mutex.lock();
        try {
          compactMemTableInternal();
        } finally {
          mutex.unlock();
        }

        InternalKey key = iterator.peek().getKey();
        if (compactionState.compaction.shouldStopBefore(key) && compactionState.builder != null) {
          finishCompactionOutputFile(compactionState);
        }

        // Handle key/value, add to state, etc.
        boolean drop = false;
        // todo if key doesn't parse (it is corrupted),
        if (false /*!ParseInternalKey(key, &ikey)*/) {
          // do not hide error keys
          currentUserKey = null;
          hasCurrentUserKey = false;
          lastSequenceForKey = MAX_SEQUENCE_NUMBER;
        } else {
          if (!hasCurrentUserKey
              || internalKeyComparator.getUserComparator().compare(key.getUserKey(), currentUserKey)
                  != 0) {
            // First occurrence of this user key
            currentUserKey = key.getUserKey();
            hasCurrentUserKey = true;
            lastSequenceForKey = MAX_SEQUENCE_NUMBER;
          }

          if (lastSequenceForKey <= compactionState.smallestSnapshot) {
            // Hidden by an newer entry for same user key
            drop = true; // (A)
          } else if (key.getValueType() == ValueType.DELETION
              && key.getSequenceNumber() <= compactionState.smallestSnapshot
              && compactionState.compaction.isBaseLevelForKey(key.getUserKey())) {

            // For this user key:
            // (1) there is no data in higher levels
            // (2) data in lower levels will have larger sequence numbers
            // (3) data in layers that are being compacted here and have
            //     smaller sequence numbers will be dropped in the next
            //     few iterations of this loop (by rule (A) above).
            // Therefore this deletion marker is obsolete and can be dropped.
            drop = true;
          }

          lastSequenceForKey = key.getSequenceNumber();
        }

        if (!drop) {
          // Open output file if necessary
          if (compactionState.builder == null) {
            openCompactionOutputFile(compactionState);
          }
          if (compactionState.builder.getEntryCount() == 0) {
            compactionState.currentSmallest = key;
          }
          compactionState.currentLargest = key;
          compactionState.builder.add(key.encode(), iterator.peek().getValue());

          // Close output file if it is big enough
          if (compactionState.builder.getFileSize()
              >= compactionState.compaction.getMaxOutputFileSize()) {
            finishCompactionOutputFile(compactionState);
          }
        }
        iterator.next();
      }

      if (shuttingDown.get()) {
        throw new DatabaseShutdownException("DB shutdown during compaction");
      }
      if (compactionState.builder != null) {
        finishCompactionOutputFile(compactionState);
      }
    } finally {
      mutex.lock();
    }

    // todo port CompactionStats code

    installCompactionResults(compactionState);
  }