/*
* Reposition to the specified file, and fill starting at
* startOffset. Position the window's buffer to point at the log entry
* indicated by targetOffset
*/
public void slideAndFill(
long windowfileNum, long windowStartOffset, long targetOffset, boolean forward)
throws ChecksumException, FileNotFoundException, DatabaseException {
FileHandle fileHandle = fileManager.getFileHandle(windowfileNum);
try {
startOffset = windowStartOffset;
setFileNum(windowfileNum, fileHandle.getLogVersion());
boolean foundData = fillFromFile(fileHandle, targetOffset);
/*
* When reading backwards, we need to guarantee there is no log
* gap, throws out an EnvironmentFailreException if it exists.
*/
if (!foundData && !forward) {
throw EnvironmentFailureException.unexpectedState(
"Detected a log file gap when reading backwards. "
+ "Target position = "
+ DbLsn.getNoFormatString(DbLsn.makeLsn(windowfileNum, targetOffset))
+ " starting position = "
+ DbLsn.getNoFormatString(DbLsn.makeLsn(windowfileNum, windowStartOffset))
+ " end position = "
+ DbLsn.getNoFormatString(DbLsn.makeLsn(windowfileNum, endOffset)));
}
} finally {
fileHandle.release();
}
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
long start = DbLsn.makeLsn(fileNum, startOffset);
long end = DbLsn.makeLsn(fileNum, endOffset);
sb.append("window covers ");
sb.append(DbLsn.getNoFormatString(start)).append(" to ");
sb.append(DbLsn.getNoFormatString(end));
sb.append(" positioned at ");
long target = DbLsn.makeLsn(fileNum, startOffset + readBuffer.position());
sb.append(DbLsn.getNoFormatString(target));
return sb.toString();
}
void setPosition(long startLsn)
throws ChecksumException, FileNotFoundException, DatabaseException {
if (startLsn == DbLsn.NULL_LSN) {
return;
}
/*
* An assertion: a reposition should never make the reader lose ground.
*/
if (forward) {
if (DbLsn.compareTo(getLastLsn(), startLsn) > 0) {
throw EnvironmentFailureException.unexpectedState(
"Feeder forward scanning should not be repositioned to "
+ " a position earlier than the current position. Current"
+ " lsn = "
+ DbLsn.getNoFormatString(getLastLsn())
+ " reposition = "
+ DbLsn.getNoFormatString(startLsn));
}
} else {
if (DbLsn.compareTo(getLastLsn(), startLsn) < 0) {
throw EnvironmentFailureException.unexpectedState(
"Feeder backward scanning should not be repositioned to "
+ " a position later than the current position. Current"
+ " lsn = "
+ DbLsn.getNoFormatString(getLastLsn())
+ " reposition = "
+ DbLsn.getNoFormatString(startLsn));
}
}
long fileNum = DbLsn.getFileNumber(startLsn);
long offset = DbLsn.getFileOffset(startLsn);
if (window.containsLsn(fileNum, offset)) {
window.positionBuffer(offset);
} else {
window.slideAndFill(fileNum, offset, offset);
}
if (forward) {
nextEntryOffset = offset;
} else {
currentEntryPrevOffset = offset;
}
nReposition++;
}
/** Print out the contents of an entry. */
@Override
public StringBuilder dumpEntry(StringBuilder sb, boolean verbose) {
in.dumpLog(sb, verbose);
dbId.dumpLog(sb, verbose);
if (prevFullLsn != DbLsn.NULL_LSN) {
sb.append("<prevFullLsn>");
sb.append(DbLsn.getNoFormatString(prevFullLsn));
sb.append("</prevFullLsn>");
}
if (prevDeltaLsn != DbLsn.NULL_LSN) {
sb.append("<prevDeltaLsn>");
sb.append(DbLsn.getNoFormatString(prevDeltaLsn));
sb.append("</prevDeltaLsn>");
}
return sb;
}
/* Position the readBuffer to the targetOffset. */
public void positionBuffer(long targetOffset) {
assert containsOffset(targetOffset)
: this + " doesn't contain " + DbLsn.getNoFormatString(targetOffset);
threadSafeBufferPosition(readBuffer, (int) (targetOffset - startOffset));
}
/* Wrap the call to logger to reduce runtime overhead. */
private void logFinest(long lsn, UndoReader undo, RevertInfo revertTo) {
if ((logger != null) && (logger.isLoggable(Level.FINEST))) {
LoggerUtils.finest(
logger,
envImpl,
"undoLsn=" + DbLsn.getNoFormatString(lsn) + " undo=" + undo + " revertInfo=" + revertTo);
}
}
/** @see Loggable#dumpLog */
public void dumpLog(StringBuilder sb, boolean verbose) {
sb.append(" matchpointVLSN=").append(matchpointVLSN.getSequence());
sb.append(" matchpointLSN=");
sb.append(DbLsn.getNoFormatString(matchpointLSN));
/* Make sure the active txns are listed in order, partially for the sake
* of the LoggableTest unit test, which expects the toString() for two
* equivalent objects to always display the same, and partially for
* ease of debugging.
*/
List<Long> displayTxnIds = new ArrayList<Long>(activeTxnIds);
Collections.sort(displayTxnIds);
sb.append(" activeTxnIds=").append(displayTxnIds);
sb.append("\" time=\"").append(time);
}
/**
* Converts the bin/index slot, whether a singleton LN or a DIN root.
*
* <p>Enter/leave with bin field latched, although bin field may change.
*
* <p>When a singleton LN is converted, leaves with bin/index fields unchanged.
*
* <p>When a dup tree is converted, leaves with bin/index fields set to last inserted slot. This
* is the slot of the highest key in the dup tree.
*/
private void convertBinSlot() {
if (DEBUG) {
System.out.println(
"DupConvert BIN LSN "
+ DbLsn.getNoFormatString(bin.getLsn(index))
+ " index "
+ index
+ " nEntries "
+ bin.getNEntries());
}
/* Delete slot if LN is deleted. */
final boolean isDeleted;
if (isLNDeleted(bin, index)) {
deleteSlot();
return;
}
final Node node = bin.fetchLNOrDIN(index, CacheMode.DEFAULT);
if (!node.containsDuplicates()) {
if (DEBUG) {
System.out.println("DupConvert BIN LN " + Key.dumpString(bin.getKey(index), 0));
}
/* Fetching a non-deleted LN updates the slot key; we're done. */
assert node instanceof LN;
nConverted += 1;
return;
}
/*
* Delete the slot containing the DIN before re-inserting the dup tree,
* so that the DIN slot key doesn't interfere with insertions.
*
* The DIN is evicted and memory usage is decremented. This is not
* exactly correct because we keep a local reference to the DIN until
* the dup tree is converted, but we tolerate this temporary
* inaccuracy.
*/
final byte[] binKey = bin.getKey(index);
final DIN din = (DIN) node;
deleteSlot();
convertDin(din, binKey);
}
/**
* Transfer a lock from another transaction to this one. Used for master-> replica transitions,
* when a node has to transform a MasterTxn into a ReplayTxn. Another approach would be to have
* this importunate ReplayTxn call lock() on the lsn, but that path is not available because we do
* not have a handle on a databaseImpl.
*/
public void stealLockFromMasterTxn(Long lsn) {
LockAttemptResult result = lockManager.stealLock(lsn, this, LockType.WRITE);
/*
* Assert, and if something strange happened, opt to invalidate
* the environment and wipe the slate clean.
*/
if (!result.success) {
throw EnvironmentFailureException.unexpectedState(
envImpl,
"Transferring from master to replica state, txn "
+ getId()
+ " was unable to transfer lock for "
+ DbLsn.getNoFormatString(lsn)
+ ", lock grant type="
+ result.lockGrant);
}
addLock(Long.valueOf(lsn), LockType.WRITE, result.lockGrant);
addLogInfo(lsn);
}
@Override
public String toString() {
return "lsn=" + DbLsn.getNoFormatString(lsn) + " node=" + nodeId;
}
/**
* Ensure that the next target is in the window. The default behavior is that the next target is
* the next previous entry.
*
* @throws DatabaseException
*/
protected void setBackwardPosition()
throws ChecksumException, FileNotFoundException, EOFException, DatabaseException {
/*
* currentEntryPrevOffset is the entry before the current entry.
* currentEntryOffset is the entry we just read (or the end of the
* file if we're starting out.
*/
if ((currentEntryPrevOffset != 0) && window.containsOffset(currentEntryPrevOffset)) {
/* The next log entry has passed the start LSN. */
long nextLsn = DbLsn.makeLsn(window.currentFileNum(), currentEntryPrevOffset);
if (finishLsn != DbLsn.NULL_LSN) {
if (DbLsn.compareTo(nextLsn, finishLsn) == -1) {
throw new EOFException(
"finish="
+ DbLsn.getNoFormatString(finishLsn)
+ "next="
+ DbLsn.getNoFormatString(nextLsn));
}
}
/* This log entry starts in this buffer, just reposition. */
window.positionBuffer(currentEntryPrevOffset);
} else {
/*
* The start of the log entry is not in this read buffer so
* we must fill the buffer again.
*
* 1) The target log entry is in a different file from the
* current window's file. Move the window to the previous
* file and start the read from the target LSN.
*
* 2) The target log entry is the same file but the log entry
* is larger than the read chunk size. Start the next read
* buffer from the target LSN. It's going to take multiple
* reads to get the log entry, and we might as well get as
* much as possible.
*
* 3) In the same file, and the log entry fits within one
* read buffer. Try to position the next buffer chunk so the
* target entry is held within the buffer, all the way at the
* end. That way, since we're reading backwards, there will be
* more buffered data available for following reads.
*/
long nextFile;
long nextWindowStart;
long nextTarget;
if (currentEntryPrevOffset == 0) {
/* Case 1: Go to another file. */
currentEntryPrevOffset = fileManager.getFileHeaderPrevOffset(window.currentFileNum());
Long prevFileNum = fileManager.getFollowingFileNum(window.currentFileNum(), false);
if (prevFileNum == null) {
throw new EOFException("No file following " + window.currentFileNum());
}
/*
* Check finishLSN before proceeding, in case we should stop
* the search before attempting to set the file reader to a
* position in the previous file. In [#22407] we threw a
* spurious EFE complaining that we cannot read backwards over
* a cleaned file because the previous file had been cleaned
* away.
*/
if (finishLsn != DbLsn.NULL_LSN && prevFileNum < DbLsn.getFileNumber(finishLsn)) {
throw new EOFException(
"finish="
+ DbLsn.getNoFormatString(finishLsn)
+ " nextFile=0x"
+ Long.toHexString(prevFileNum));
}
if (window.currentFileNum() - prevFileNum.longValue() != 1) {
handleGapInBackwardsScan(prevFileNum);
}
nextFile = prevFileNum;
nextWindowStart = currentEntryPrevOffset;
nextTarget = currentEntryPrevOffset;
} else if ((currentEntryOffset - currentEntryPrevOffset) > window.capacity()) {
/*
* Case 2: The entry is in the same file, but is bigger
* than one buffer. Position it at the front of the buffer.
*/
nextFile = window.currentFileNum();
nextWindowStart = currentEntryPrevOffset;
nextTarget = currentEntryPrevOffset;
} else {
/*
* Case 3: In same file, but not in this buffer. The target
* entry will fit in one buffer.
*/
nextFile = window.currentFileNum();
long newPosition = currentEntryOffset - window.capacity();
nextWindowStart = (newPosition < 0) ? 0 : newPosition;
nextTarget = currentEntryPrevOffset;
}
/* The next log entry has passed the start LSN. */
long nextLsn = DbLsn.makeLsn(nextFile, currentEntryPrevOffset);
if (finishLsn != DbLsn.NULL_LSN) {
if (DbLsn.compareTo(nextLsn, finishLsn) == -1) {
throw new EOFException(
"finish="
+ DbLsn.getNoFormatString(finishLsn)
+ " next="
+ DbLsn.getNoFormatString(nextLsn));
}
}
window.slideAndFill(nextFile, nextWindowStart, nextTarget, forward);
}
/* The current entry will start at this offset. */
currentEntryOffset = currentEntryPrevOffset;
}
private void summarizeCheckpointInfo(CheckpointInfoTextFormatter f) {
System.out.println("\nPer checkpoint interval info:");
/*
* Print out checkpoint interval info.
* If the log looks like this:
*
* start of log
* ckpt1 start
* ckpt1 end
* ckpt2 start
* ckpt2 end
* end of log
*
* There are 3 ckpt intervals
* start of log->ckpt1 end
* ckpt1 end -> ckpt2 end
* ckpt2 end -> end of log
*/
System.out.println(
f.format("lnTxn")
+ f.format("ln")
+ f.format("mapLNTxn")
+ f.format("mapLN")
+ f.format("end to end")
+ // ckpt n-1 end -> ckpt n end
f.format("end to start")
+ // ckpt n-1 end -> ckpt n start
f.format("start to end")
+ // ckpt n start -> ckpt n end
f.format("maxLNReplay")
+ f.format("ckptEnd"));
long logFileMax = envImpl.getConfigManager().getLong(EnvironmentParams.LOG_FILE_MAX);
Iterator<CheckpointCounter> iter = ckptList.iterator();
CheckpointCounter prevCounter = null;
while (iter.hasNext()) {
CheckpointCounter c = iter.next();
StringBuilder sb = new StringBuilder();
/* Entry type counts. */
int maxTxnLNs = c.preStartLNTxnCount + c.postStartLNTxnCount;
sb.append(f.format(maxTxnLNs));
int maxLNs = c.preStartLNCount + c.postStartLNCount;
sb.append(f.format(maxLNs));
sb.append(f.format(c.preStartMapLNTxnCount + c.postStartMapLNTxnCount));
sb.append(f.format(c.preStartMapLNCount + c.postStartMapLNCount));
/* Checkpoint interval distance. */
long end = (c.endCkptLsn == DbLsn.NULL_LSN) ? getLastLsn() : c.endCkptLsn;
long endToEndDistance = 0;
FileManager fileMgr = envImpl.getFileManager();
if (prevCounter == null) {
endToEndDistance = DbLsn.getWithCleaningDistance(end, fileMgr, firstLsnRead, logFileMax);
} else {
endToEndDistance =
DbLsn.getWithCleaningDistance(end, fileMgr, prevCounter.endCkptLsn, logFileMax);
}
sb.append(f.format(endToEndDistance));
/*
* Interval between last checkpoint end and this checkpoint start.
*/
long start = (c.startCkptLsn == DbLsn.NULL_LSN) ? getLastLsn() : c.startCkptLsn;
long endToStartDistance = 0;
if (prevCounter == null) {
endToStartDistance =
DbLsn.getWithCleaningDistance(start, fileMgr, firstLsnRead, logFileMax);
} else {
endToStartDistance =
DbLsn.getWithCleaningDistance(start, fileMgr, prevCounter.endCkptLsn, logFileMax);
}
sb.append(f.format(endToStartDistance));
/*
* Interval between ckpt start and ckpt end.
*/
long startToEndDistance = 0;
if ((c.startCkptLsn != DbLsn.NULL_LSN) && (c.endCkptLsn != DbLsn.NULL_LSN)) {
startToEndDistance =
DbLsn.getWithCleaningDistance(c.endCkptLsn, fileMgr, c.startCkptLsn, logFileMax);
}
sb.append(f.format(startToEndDistance));
/*
* The maximum number of LNs to replay includes the portion of LNs
* from checkpoint start to checkpoint end of the previous
* interval.
*/
int maxReplay = maxLNs + maxTxnLNs;
if (prevCounter != null) {
maxReplay += prevCounter.postStartLNTxnCount;
maxReplay += prevCounter.postStartLNCount;
}
sb.append(f.format(maxReplay));
if (c.endCkptLsn == DbLsn.NULL_LSN) {
sb.append(" ").append(DbLsn.getNoFormatString(getLastLsn()));
} else {
sb.append(" ").append(DbLsn.getNoFormatString(c.endCkptLsn));
}
System.out.println(sb.toString());
prevCounter = c;
}
}
/**
* Determine whether a checkpoint should be run. 1. If the force parameter is specified, always
* checkpoint. 2. If the config object specifies time or log size, use that. 3. If the environment
* is configured to use log size based checkpointing, check the log. 4. Lastly, use time based
* checking.
*/
private boolean isRunnable(CheckpointConfig config) throws DatabaseException {
/* Figure out if we're using log size or time to determine interval.*/
long useBytesInterval = 0;
long useTimeInterval = 0;
DbLsn nextLsn = null;
try {
if (config.getForce()) {
return true;
} else if (config.getKBytes() != 0) {
useBytesInterval = config.getKBytes() << 10;
} else if (config.getMinutes() != 0) {
// convert to millis
useTimeInterval = config.getMinutes() * 60 * 1000;
} else if (logSizeBytesInterval != 0) {
useBytesInterval = logSizeBytesInterval;
} else {
useTimeInterval = timeInterval;
}
/*
* If our checkpoint interval is defined by log size, check
* on how much log has grown since the last checkpoint.
*/
if (useBytesInterval != 0) {
nextLsn = envImpl.getFileManager().getNextLsn();
if (nextLsn.getNoCleaningDistance(lastCheckpointEnd, logFileMax) >= useBytesInterval) {
return true;
} else {
return false;
}
} else if (useTimeInterval != 0) {
/*
* Our checkpoint is determined by time. If enough
* time has passed and some log data has been written,
* do a checkpoint.
*/
DbLsn lastUsedLsn = envImpl.getFileManager().getLastUsedLsn();
if (((System.currentTimeMillis() - lastCheckpointMillis) >= useTimeInterval)
&& (lastUsedLsn.compareTo(lastCheckpointEnd) != 0)) {
return true;
} else {
return false;
}
} else {
return false;
}
} finally {
StringBuffer sb = new StringBuffer();
sb.append("size interval=").append(useBytesInterval);
if (nextLsn != null) {
sb.append(" nextLsn=").append(nextLsn.getNoFormatString());
}
if (lastCheckpointEnd != null) {
sb.append(" lastCkpt=");
sb.append(lastCheckpointEnd.getNoFormatString());
}
sb.append(" time interval=").append(useTimeInterval);
sb.append(" force=").append(config.getForce());
Tracer.trace(Level.FINEST, envImpl, sb.toString());
}
}
/**
* Rollback the changes to this txn's write locked nodes up to but not including the entry at the
* specified matchpoint. When we log a transactional entry, we record the LSN of the original,
* before-this-transaction version as the abort LSN. This means that if there are multiple updates
* to a given record in a single transaction, each update only references that original version
* and its true predecessor.
*
* <p>This was done to streamline abort processing, so that an undo reverts directly to the
* original version rather than stepping through all the intermediates. The intermediates are
* skipped. However, undo to a matchpoint may need to stop at an intermediate point, so we need to
* create a true chain of versions.
*
* <p>To do so, we read the transaction backwards from the last logged LSN to reconstruct a
* transaction chain that links intermediate versions of records. For example, suppose our
* transaction looks like this and that we are undoing up to LSN 250
*
* <p>lsn=100 node=A (version 1) lsn=200 node=B (version 1) <-- matchpointLsn lsn=300 node=C
* (version 1) lsn=400 node=A (version 2) lsn=500 node=B (version 2) lsn=600 node=A (version 3)
* lsn=700 node=A (version 4)
*
* <p>To setup the old versions, We walk from LSN 700 -> 100 700 (A) rolls back to 600 600 (A)
* rolls back to 400 500 (B) rolls back to 200 400 (A) rolls back to 100 300 (C) rolls back to an
* empty slot (NULL_LSN).
*
* <p>A partial rollback also requires resetting the lastLoggedLsn field, because these operations
* are no longer in the btree and their on-disk entries are no longer valid.
*
* <p>Lastly, the appropriate write locks must be released.
*
* @param matchpointLsn the rollback should go up to but not include this LSN.
*/
private void undoWrites(long matchpointLsn, List<Long> rollbackLsns) throws DatabaseException {
/*
* Generate a map of nodeId->List of intermediate LSNs for this node.
* to re-create the transaction chain.
*/
TreeLocation location = new TreeLocation();
Long undoLsn = lastLoggedLsn;
TxnChain chain = new TxnChain(undoLsn, id, matchpointLsn, undoDatabases, envImpl);
try {
while ((undoLsn != DbLsn.NULL_LSN) && DbLsn.compareTo(undoLsn, matchpointLsn) > 0) {
UndoReader undo = new UndoReader(envImpl, undoLsn, undoDatabases);
RevertInfo revertTo = chain.pop();
logFinest(undoLsn, undo, revertTo);
/*
* When we undo this log entry, we've logically truncated
* it from the log. Remove it from the btree and mark it
* obsolete.
*/
RecoveryManager.rollbackUndo(logger, Level.FINER, undo, revertTo, location, undoLsn);
countObsoleteInexact(undoLsn, undo);
rollbackLsns.add(undoLsn);
/*
* Move on to the previous log entry for this txn and update
* what is considered to be the end of the transaction chain.
*/
undoLsn = undo.logEntry.getUserTxn().getLastLsn();
lastLoggedLsn = undoLsn;
}
/*
* Correct the fields which hold LSN and VLSN state that may
* now be changed.
*/
lastApplied = chain.getLastValidVLSN();
if (!updateLoggedForTxn()) {
firstLoggedLsn = NULL_LSN;
}
} catch (DatabaseException e) {
LoggerUtils.traceAndLogException(
envImpl, "Txn", "undo", "For LSN=" + DbLsn.getNoFormatString(undoLsn), e);
throw e;
} catch (RuntimeException e) {
throw EnvironmentFailureException.unexpectedException(
"Txn undo for LSN=" + DbLsn.getNoFormatString(undoLsn), e);
}
if (lastLoggedLsn == DbLsn.NULL_LSN) {
/*
* The whole txn is rolled back, and it may not appear again. This
* is the equivalent of an abort. Do any delete processing for an
* abort which is needed.
*
* Set database state for deletes before releasing any write
* locks.
*/
setDeletedDatabaseState(false);
}
/* Clear any write locks that are no longer needed. */
clearWriteLocks(chain.getRemainingLockedNodes());
}