/**
* Purge the log up to and excluding the provided key.
*
* @param purgeKey the key up to which purging must happen
* @return the oldest non purged record, or {@code null} if no record was purged
* @throws ChangelogException if a database problem occurs.
*/
public Record<K, V> purgeUpTo(final K purgeKey) throws ChangelogException {
exclusiveLock.lock();
try {
if (isClosed) {
return null;
}
final SortedMap<K, LogFile<K, V>> logFilesToPurge = logFiles.headMap(purgeKey);
if (logFilesToPurge.isEmpty()) {
return null;
}
final List<String> undeletableFiles = new ArrayList<>();
final Iterator<LogFile<K, V>> entriesToPurge = logFilesToPurge.values().iterator();
while (entriesToPurge.hasNext()) {
final LogFile<K, V> logFile = entriesToPurge.next();
try {
abortCursorsOpenOnLogFile(logFile);
logFile.close();
logFile.delete();
entriesToPurge.remove();
} catch (ChangelogException e) {
// The deletion of log file on file system has failed
undeletableFiles.add(logFile.getFile().getPath());
}
}
if (!undeletableFiles.isEmpty()) {
throw new ChangelogException(
ERR_CHANGELOG_UNABLE_TO_DELETE_LOG_FILE_WHILE_PURGING.get(
Utils.joinAsString(", ", undeletableFiles)));
}
return getOldestRecord();
} finally {
exclusiveLock.unlock();
}
}
private void openHeadLogFile() throws ChangelogException {
final LogFile<K, V> head =
LogFile.newAppendableLogFile(new File(logPath, HEAD_LOG_FILE_NAME), recordParser);
final Record<K, V> newestRecord = head.getNewestRecord();
lastAppendedKey = newestRecord != null ? newestRecord.getKey() : null;
logFiles.put(recordParser.getMaxKey(), head);
}
/**
* Add the provided record at the end of this log.
*
* <p>The record must have a key strictly higher than the key of the last record added. If it is
* not the case, the record is not appended and the method returns immediately.
*
* <p>In order to ensure that record is written out of buffers and persisted to file system, it is
* necessary to explicitely call the {@code syncToFileSystem()} method.
*
* @param record The record to add.
* @throws ChangelogException If an error occurs while adding the record to the log.
*/
public void append(final Record<K, V> record) throws ChangelogException {
// If this exclusive lock happens to be a bottleneck :
// 1. use a shared lock for appending the record first
// 2. switch to an exclusive lock only if rotation is needed
// See http://sources.forgerock.org/cru/CR-3548#c27521 for full detail
exclusiveLock.lock();
try {
if (isClosed) {
return;
}
if (recordIsBreakingKeyOrdering(record)) {
logger.info(
LocalizableMessage.raw(
"Rejecting append to log '%s' for record: [%s], last key appended: [%s]",
logPath.getPath(), record, lastAppendedKey != null ? lastAppendedKey : "null"));
return;
}
LogFile<K, V> headLogFile = getHeadLogFile();
if (mustRotate(headLogFile)) {
logger.trace(
INFO_CHANGELOG_LOG_FILE_ROTATION.get(logPath.getPath(), headLogFile.getSizeInBytes()));
rotateHeadLogFile();
headLogFile = getHeadLogFile();
}
headLogFile.append(record);
lastAppendedKey = record.getKey();
} finally {
exclusiveLock.unlock();
}
}
/**
* Returns the file name to use for the read-only version of the provided log file.
*
* <p>The file name is based on the lowest and highest key in the log file.
*
* @return the name to use for the read-only version of the log file
* @throws ChangelogException If an error occurs.
*/
private String generateReadOnlyFileName(final LogFile<K, V> logFile) throws ChangelogException {
final K lowestKey = logFile.getOldestRecord().getKey();
final K highestKey = logFile.getNewestRecord().getKey();
return recordParser.encodeKeyToString(lowestKey)
+ LOG_FILE_NAME_SEPARATOR
+ recordParser.encodeKeyToString(highestKey)
+ LOG_FILE_SUFFIX;
}
/**
* Returns the number of records in the log.
*
* @return the number of records
* @throws ChangelogException If a problem occurs.
*/
public long getNumberOfRecords() throws ChangelogException {
long count = 0;
sharedLock.lock();
try {
for (final LogFile<K, V> logFile : logFiles.values()) {
count += logFile.getNumberOfRecords();
}
return count;
} finally {
sharedLock.unlock();
}
}
/**
* Returns the key bounds for the provided log file.
*
* @return the pair of (lowest key, highest key) that correspond to records stored in the
* corresponding log file.
* @throws ChangelogException if an error occurs while retrieving the keys
*/
private Pair<K, K> getKeyBounds(final LogFile<K, V> logFile) throws ChangelogException {
try {
final String name = logFile.getFile().getName();
final String[] keys =
name.substring(0, name.length() - Log.LOG_FILE_SUFFIX.length())
.split(LOG_FILE_NAME_SEPARATOR);
return Pair.of(
recordParser.decodeKeyFromString(keys[0]), recordParser.decodeKeyFromString(keys[1]));
} catch (Exception e) {
throw new ChangelogException(
ERR_CHANGELOG_UNABLE_TO_RETRIEVE_KEY_BOUNDS_FROM_FILE.get(logFile.getFile().getPath()),
e);
}
}
/**
* Find the highest key that corresponds to a record that is the oldest (or first) of a read-only
* log file and where value mapped from the record is lower or equals to provided limit value.
*
* <p>Example<br>
* Given a log with 3 log files, with Record<Int, String> and Mapper<String, Long> mapping a
* string to its long value
*
* <ul>
* <li>1_10.log where oldest record is (key=1, value="50")
* <li>11_20.log where oldest record is (key=11, value="150")
* <li>head.log where oldest record is (key=25, value="250")
* </ul>
*
* Then
*
* <ul>
* <li>findBoundaryKeyFromRecord(mapper, 20) => null
* <li>findBoundaryKeyFromRecord(mapper, 50) => 1
* <li>findBoundaryKeyFromRecord(mapper, 100) => 1
* <li>findBoundaryKeyFromRecord(mapper, 150) => 11
* <li>findBoundaryKeyFromRecord(mapper, 200) => 11
* <li>findBoundaryKeyFromRecord(mapper, 250) => 25
* <li>findBoundaryKeyFromRecord(mapper, 300) => 25
* </ul>
*
* @param <V2> Type of the value extracted from the record
* @param mapper The mapper to extract a value from a record. It is expected that extracted values
* are ordered according to an order consistent with this log ordering, i.e. for two records,
* if key(R1) > key(R2) then extractedValue(R1) > extractedValue(R2).
* @param limitValue The limit value to search for
* @return the key or {@code null} if no key corresponds
* @throws ChangelogException If a problem occurs
*/
<V2 extends Comparable<V2>> K findBoundaryKeyFromRecord(
Record.Mapper<V, V2> mapper, V2 limitValue) throws ChangelogException {
sharedLock.lock();
try {
K key = null;
for (LogFile<K, V> logFile : logFiles.values()) {
final Record<K, V> record = logFile.getOldestRecord();
final V2 oldestValue = mapper.map(record.getValue());
if (oldestValue.compareTo(limitValue) > 0) {
return key;
}
key = record.getKey();
}
return key;
} finally {
sharedLock.unlock();
}
}
/**
* Rotate the head log file to a read-only log file, and open a new empty head log file to write
* in.
*
* <p>All cursors opened on this log are temporarily disabled (closing underlying resources) and
* then re-open with their previous state.
*/
private void rotateHeadLogFile() throws ChangelogException {
// Temporarily disable cursors opened on head, saving their state
final List<Pair<AbortableLogCursor<K, V>, CursorState<K, V>>> cursorsOnHead =
disableOpenedCursorsOnHead();
final LogFile<K, V> headLogFile = getHeadLogFile();
final File readOnlyLogFile = new File(logPath, generateReadOnlyFileName(headLogFile));
headLogFile.close();
renameHeadLogFileTo(readOnlyLogFile);
openHeadLogFile();
openReadOnlyLogFile(readOnlyLogFile);
// Re-enable cursors previously opened on head, with the saved state
updateOpenedCursorsOnHeadAfterRotation(cursorsOnHead);
// Notify even if time-based rotation is not enabled, as it could be enabled at any time
replicationEnv.notifyLogFileRotation(this);
lastRotationTime = timeService.now();
}
/**
* Empties the log, discarding all records it contains.
*
* <p>All cursors open on the log are aborted.
*
* @throws ChangelogException If cursors are opened on this log, or if a problem occurs during
* clearing operation.
*/
public void clear() throws ChangelogException {
exclusiveLock.lock();
try {
if (isClosed) {
return;
}
if (!openCursors.isEmpty()) {
// All open cursors are aborted, which means the change number indexer thread
// should manage AbortedChangelogCursorException specifically to avoid being
// stopped
abortAllOpenCursors();
}
// delete all log files
final List<String> undeletableFiles = new ArrayList<>();
for (LogFile<K, V> logFile : logFiles.values()) {
try {
logFile.close();
logFile.delete();
} catch (ChangelogException e) {
undeletableFiles.add(logFile.getFile().getPath());
}
}
if (!undeletableFiles.isEmpty()) {
throw new ChangelogException(
ERR_CHANGELOG_UNABLE_TO_DELETE_LOG_FILE.get(
Utils.joinAsString(", ", undeletableFiles)));
}
logFiles.clear();
// recreate an empty head log file
openHeadLogFile();
} catch (Exception e) {
throw new ChangelogException(
ERR_ERROR_CLEARING_DB.get(logPath.getPath(), stackTraceToSingleLineString(e)));
} finally {
exclusiveLock.unlock();
}
}
private boolean mustRotate(LogFile<K, V> headLogFile) {
if (lastAppendedKey == null) {
// never rotate an empty file
return false;
}
if (headLogFile.getSizeInBytes() > sizeLimitPerLogFileInBytes) {
// rotate because file size exceeded threshold
logger.trace(
"Rotate log %s due to size: %s", logPath.getPath(), headLogFile.getSizeInBytes());
return true;
}
if (rotationIntervalInMillis > 0) {
// rotate if time limit is reached
final long timeElapsed = timeService.since(lastRotationTime);
boolean shouldRotate = timeElapsed > rotationIntervalInMillis;
if (shouldRotate) {
logger.trace(
"Rotate log %s due to time: time elapsed %s, rotation interval: %s",
logPath.getPath(), timeElapsed, rotationIntervalInMillis);
}
return shouldRotate;
}
return false;
}
/**
* Dump this log as a text files, intended for debugging purpose only.
*
* @param dumpDirectory Directory that will contains log files with text format and ".txt"
* extensions
* @throws ChangelogException If an error occurs during dump
*/
void dumpAsTextFile(File dumpDirectory) throws ChangelogException {
for (LogFile<K, V> logFile : logFiles.values()) {
logFile.dumpAsTextFile(new File(dumpDirectory, logFile.getFile().getName() + ".txt"));
}
}
@Override
boolean isAccessingLogFile(LogFile<K, V> logFile) {
return currentLogFile != null && currentLogFile.equals(logFile);
}
@Override
public String toString() {
return String.format(
"Cursor on log : %s, current log file: %s, current cursor: %s",
log.logPath, currentLogFile.getFile().getName(), currentCursor);
}
/** Switch the cursor to the provided log file. */
private void switchToLogFile(final LogFile<K, V> logFile) throws ChangelogException {
StaticUtils.close(currentCursor);
currentLogFile = logFile;
currentCursor = currentLogFile.getCursor();
}
private void openReadOnlyLogFile(final File logFilePath) throws ChangelogException {
final LogFile<K, V> logFile = LogFile.newReadOnlyLogFile(logFilePath, recordParser);
final Pair<K, K> bounds = getKeyBounds(logFile);
logFiles.put(bounds.getSecond(), logFile);
}
private boolean isHeadLogFile(final LogFile<K, V> logFile) {
return logFile.getFile().getName().equals(Log.HEAD_LOG_FILE_NAME);
}
/** Reinitialize this cursor to the provided state. */
@Override
void reinitializeTo(final CursorState<K, V> cursorState) throws ChangelogException {
currentLogFile = cursorState.logFile;
currentCursor =
currentLogFile.getCursorInitialisedTo(cursorState.record, cursorState.filePosition);
}