Example #1
0
 /**
  * Purge the log up to and excluding the provided key.
  *
  * @param purgeKey the key up to which purging must happen
  * @return the oldest non purged record, or {@code null} if no record was purged
  * @throws ChangelogException if a database problem occurs.
  */
 public Record<K, V> purgeUpTo(final K purgeKey) throws ChangelogException {
   exclusiveLock.lock();
   try {
     if (isClosed) {
       return null;
     }
     final SortedMap<K, LogFile<K, V>> logFilesToPurge = logFiles.headMap(purgeKey);
     if (logFilesToPurge.isEmpty()) {
       return null;
     }
     final List<String> undeletableFiles = new ArrayList<>();
     final Iterator<LogFile<K, V>> entriesToPurge = logFilesToPurge.values().iterator();
     while (entriesToPurge.hasNext()) {
       final LogFile<K, V> logFile = entriesToPurge.next();
       try {
         abortCursorsOpenOnLogFile(logFile);
         logFile.close();
         logFile.delete();
         entriesToPurge.remove();
       } catch (ChangelogException e) {
         // The deletion of log file on file system has failed
         undeletableFiles.add(logFile.getFile().getPath());
       }
     }
     if (!undeletableFiles.isEmpty()) {
       throw new ChangelogException(
           ERR_CHANGELOG_UNABLE_TO_DELETE_LOG_FILE_WHILE_PURGING.get(
               Utils.joinAsString(", ", undeletableFiles)));
     }
     return getOldestRecord();
   } finally {
     exclusiveLock.unlock();
   }
 }
Example #2
0
 private void openHeadLogFile() throws ChangelogException {
   final LogFile<K, V> head =
       LogFile.newAppendableLogFile(new File(logPath, HEAD_LOG_FILE_NAME), recordParser);
   final Record<K, V> newestRecord = head.getNewestRecord();
   lastAppendedKey = newestRecord != null ? newestRecord.getKey() : null;
   logFiles.put(recordParser.getMaxKey(), head);
 }
Example #3
0
  /**
   * Add the provided record at the end of this log.
   *
   * <p>The record must have a key strictly higher than the key of the last record added. If it is
   * not the case, the record is not appended and the method returns immediately.
   *
   * <p>In order to ensure that record is written out of buffers and persisted to file system, it is
   * necessary to explicitely call the {@code syncToFileSystem()} method.
   *
   * @param record The record to add.
   * @throws ChangelogException If an error occurs while adding the record to the log.
   */
  public void append(final Record<K, V> record) throws ChangelogException {
    // If this exclusive lock happens to be a bottleneck :
    // 1. use a shared lock for appending the record first
    // 2. switch to an exclusive lock only if rotation is needed
    // See http://sources.forgerock.org/cru/CR-3548#c27521 for full detail
    exclusiveLock.lock();
    try {
      if (isClosed) {
        return;
      }
      if (recordIsBreakingKeyOrdering(record)) {
        logger.info(
            LocalizableMessage.raw(
                "Rejecting append to log '%s' for record: [%s], last key appended: [%s]",
                logPath.getPath(), record, lastAppendedKey != null ? lastAppendedKey : "null"));
        return;
      }
      LogFile<K, V> headLogFile = getHeadLogFile();
      if (mustRotate(headLogFile)) {
        logger.trace(
            INFO_CHANGELOG_LOG_FILE_ROTATION.get(logPath.getPath(), headLogFile.getSizeInBytes()));

        rotateHeadLogFile();
        headLogFile = getHeadLogFile();
      }
      headLogFile.append(record);
      lastAppendedKey = record.getKey();
    } finally {
      exclusiveLock.unlock();
    }
  }
Example #4
0
 /**
  * Returns the file name to use for the read-only version of the provided log file.
  *
  * <p>The file name is based on the lowest and highest key in the log file.
  *
  * @return the name to use for the read-only version of the log file
  * @throws ChangelogException If an error occurs.
  */
 private String generateReadOnlyFileName(final LogFile<K, V> logFile) throws ChangelogException {
   final K lowestKey = logFile.getOldestRecord().getKey();
   final K highestKey = logFile.getNewestRecord().getKey();
   return recordParser.encodeKeyToString(lowestKey)
       + LOG_FILE_NAME_SEPARATOR
       + recordParser.encodeKeyToString(highestKey)
       + LOG_FILE_SUFFIX;
 }
Example #5
0
 /**
  * Returns the number of records in the log.
  *
  * @return the number of records
  * @throws ChangelogException If a problem occurs.
  */
 public long getNumberOfRecords() throws ChangelogException {
   long count = 0;
   sharedLock.lock();
   try {
     for (final LogFile<K, V> logFile : logFiles.values()) {
       count += logFile.getNumberOfRecords();
     }
     return count;
   } finally {
     sharedLock.unlock();
   }
 }
Example #6
0
 /**
  * Returns the key bounds for the provided log file.
  *
  * @return the pair of (lowest key, highest key) that correspond to records stored in the
  *     corresponding log file.
  * @throws ChangelogException if an error occurs while retrieving the keys
  */
 private Pair<K, K> getKeyBounds(final LogFile<K, V> logFile) throws ChangelogException {
   try {
     final String name = logFile.getFile().getName();
     final String[] keys =
         name.substring(0, name.length() - Log.LOG_FILE_SUFFIX.length())
             .split(LOG_FILE_NAME_SEPARATOR);
     return Pair.of(
         recordParser.decodeKeyFromString(keys[0]), recordParser.decodeKeyFromString(keys[1]));
   } catch (Exception e) {
     throw new ChangelogException(
         ERR_CHANGELOG_UNABLE_TO_RETRIEVE_KEY_BOUNDS_FROM_FILE.get(logFile.getFile().getPath()),
         e);
   }
 }
Example #7
0
 /**
  * Find the highest key that corresponds to a record that is the oldest (or first) of a read-only
  * log file and where value mapped from the record is lower or equals to provided limit value.
  *
  * <p>Example<br>
  * Given a log with 3 log files, with Record<Int, String> and Mapper<String, Long> mapping a
  * string to its long value
  *
  * <ul>
  *   <li>1_10.log where oldest record is (key=1, value="50")
  *   <li>11_20.log where oldest record is (key=11, value="150")
  *   <li>head.log where oldest record is (key=25, value="250")
  * </ul>
  *
  * Then
  *
  * <ul>
  *   <li>findBoundaryKeyFromRecord(mapper, 20) => null
  *   <li>findBoundaryKeyFromRecord(mapper, 50) => 1
  *   <li>findBoundaryKeyFromRecord(mapper, 100) => 1
  *   <li>findBoundaryKeyFromRecord(mapper, 150) => 11
  *   <li>findBoundaryKeyFromRecord(mapper, 200) => 11
  *   <li>findBoundaryKeyFromRecord(mapper, 250) => 25
  *   <li>findBoundaryKeyFromRecord(mapper, 300) => 25
  * </ul>
  *
  * @param <V2> Type of the value extracted from the record
  * @param mapper The mapper to extract a value from a record. It is expected that extracted values
  *     are ordered according to an order consistent with this log ordering, i.e. for two records,
  *     if key(R1) > key(R2) then extractedValue(R1) > extractedValue(R2).
  * @param limitValue The limit value to search for
  * @return the key or {@code null} if no key corresponds
  * @throws ChangelogException If a problem occurs
  */
 <V2 extends Comparable<V2>> K findBoundaryKeyFromRecord(
     Record.Mapper<V, V2> mapper, V2 limitValue) throws ChangelogException {
   sharedLock.lock();
   try {
     K key = null;
     for (LogFile<K, V> logFile : logFiles.values()) {
       final Record<K, V> record = logFile.getOldestRecord();
       final V2 oldestValue = mapper.map(record.getValue());
       if (oldestValue.compareTo(limitValue) > 0) {
         return key;
       }
       key = record.getKey();
     }
     return key;
   } finally {
     sharedLock.unlock();
   }
 }
Example #8
0
  /**
   * Rotate the head log file to a read-only log file, and open a new empty head log file to write
   * in.
   *
   * <p>All cursors opened on this log are temporarily disabled (closing underlying resources) and
   * then re-open with their previous state.
   */
  private void rotateHeadLogFile() throws ChangelogException {
    // Temporarily disable cursors opened on head, saving their state
    final List<Pair<AbortableLogCursor<K, V>, CursorState<K, V>>> cursorsOnHead =
        disableOpenedCursorsOnHead();

    final LogFile<K, V> headLogFile = getHeadLogFile();
    final File readOnlyLogFile = new File(logPath, generateReadOnlyFileName(headLogFile));
    headLogFile.close();
    renameHeadLogFileTo(readOnlyLogFile);

    openHeadLogFile();
    openReadOnlyLogFile(readOnlyLogFile);

    // Re-enable cursors previously opened on head, with the saved state
    updateOpenedCursorsOnHeadAfterRotation(cursorsOnHead);

    // Notify even if time-based rotation is not enabled, as it could be enabled at any time
    replicationEnv.notifyLogFileRotation(this);
    lastRotationTime = timeService.now();
  }
Example #9
0
  /**
   * Empties the log, discarding all records it contains.
   *
   * <p>All cursors open on the log are aborted.
   *
   * @throws ChangelogException If cursors are opened on this log, or if a problem occurs during
   *     clearing operation.
   */
  public void clear() throws ChangelogException {
    exclusiveLock.lock();
    try {
      if (isClosed) {
        return;
      }
      if (!openCursors.isEmpty()) {
        // All open cursors are aborted, which means the change number indexer thread
        // should manage AbortedChangelogCursorException specifically to avoid being
        // stopped
        abortAllOpenCursors();
      }

      // delete all log files
      final List<String> undeletableFiles = new ArrayList<>();
      for (LogFile<K, V> logFile : logFiles.values()) {
        try {
          logFile.close();
          logFile.delete();
        } catch (ChangelogException e) {
          undeletableFiles.add(logFile.getFile().getPath());
        }
      }
      if (!undeletableFiles.isEmpty()) {
        throw new ChangelogException(
            ERR_CHANGELOG_UNABLE_TO_DELETE_LOG_FILE.get(
                Utils.joinAsString(", ", undeletableFiles)));
      }
      logFiles.clear();

      // recreate an empty head log file
      openHeadLogFile();
    } catch (Exception e) {
      throw new ChangelogException(
          ERR_ERROR_CLEARING_DB.get(logPath.getPath(), stackTraceToSingleLineString(e)));
    } finally {
      exclusiveLock.unlock();
    }
  }
Example #10
0
 private boolean mustRotate(LogFile<K, V> headLogFile) {
   if (lastAppendedKey == null) {
     // never rotate an empty file
     return false;
   }
   if (headLogFile.getSizeInBytes() > sizeLimitPerLogFileInBytes) {
     // rotate because file size exceeded threshold
     logger.trace(
         "Rotate log %s due to size: %s", logPath.getPath(), headLogFile.getSizeInBytes());
     return true;
   }
   if (rotationIntervalInMillis > 0) {
     // rotate if time limit is reached
     final long timeElapsed = timeService.since(lastRotationTime);
     boolean shouldRotate = timeElapsed > rotationIntervalInMillis;
     if (shouldRotate) {
       logger.trace(
           "Rotate log %s due to time: time elapsed %s, rotation interval: %s",
           logPath.getPath(), timeElapsed, rotationIntervalInMillis);
     }
     return shouldRotate;
   }
   return false;
 }
Example #11
0
 /**
  * Dump this log as a text files, intended for debugging purpose only.
  *
  * @param dumpDirectory Directory that will contains log files with text format and ".txt"
  *     extensions
  * @throws ChangelogException If an error occurs during dump
  */
 void dumpAsTextFile(File dumpDirectory) throws ChangelogException {
   for (LogFile<K, V> logFile : logFiles.values()) {
     logFile.dumpAsTextFile(new File(dumpDirectory, logFile.getFile().getName() + ".txt"));
   }
 }
Example #12
0
 @Override
 boolean isAccessingLogFile(LogFile<K, V> logFile) {
   return currentLogFile != null && currentLogFile.equals(logFile);
 }
Example #13
0
 @Override
 public String toString() {
   return String.format(
       "Cursor on log : %s, current log file: %s, current cursor: %s",
       log.logPath, currentLogFile.getFile().getName(), currentCursor);
 }
Example #14
0
 /** Switch the cursor to the provided log file. */
 private void switchToLogFile(final LogFile<K, V> logFile) throws ChangelogException {
   StaticUtils.close(currentCursor);
   currentLogFile = logFile;
   currentCursor = currentLogFile.getCursor();
 }
Example #15
0
 private void openReadOnlyLogFile(final File logFilePath) throws ChangelogException {
   final LogFile<K, V> logFile = LogFile.newReadOnlyLogFile(logFilePath, recordParser);
   final Pair<K, K> bounds = getKeyBounds(logFile);
   logFiles.put(bounds.getSecond(), logFile);
 }
Example #16
0
 private boolean isHeadLogFile(final LogFile<K, V> logFile) {
   return logFile.getFile().getName().equals(Log.HEAD_LOG_FILE_NAME);
 }
Example #17
0
 /** Reinitialize this cursor to the provided state. */
 @Override
 void reinitializeTo(final CursorState<K, V> cursorState) throws ChangelogException {
   currentLogFile = cursorState.logFile;
   currentCursor =
       currentLogFile.getCursorInitialisedTo(cursorState.record, cursorState.filePosition);
 }