/** Logs the LN at the given index if it is dirty. */
private void logDirtyLN(int index, LN ln, boolean force) throws DatabaseException {
if (ln.isDirty() || force) {
DatabaseImpl dbImpl = getDatabase();
/* Only deferred write databases should have dirty LNs. */
assert dbImpl.isDeferredWriteMode();
/* Log the LN with the main tree key. */
byte[] key = containsDuplicates() ? getDupKey() : getKey(index);
/*
* No need to lock, this is non-txnal. This should never be part
* othe replication stream, because this is a deferred write db.
*/
long lsn =
ln.log(
dbImpl.getDbEnvironment(),
dbImpl,
key,
getLsn(index), // obsoleteLsn
null, // locker
true, // backgroundIO
ReplicationContext.NO_REPLICATE);
updateEntry(index, lsn);
}
}
/**
* Evict a single LN if allowed. The amount of memory freed is returned and must be subtracted
* from the memory budget by the caller.
*/
private long evictInternal(int index, Cleaner cleaner) throws DatabaseException {
Node n = getTarget(index);
if (n instanceof LN) {
LN ln = (LN) n;
/*
* Don't evict MapLNs for open databases (LN.isEvictable) [#13415].
* And don't strip LNs that the cleaner will be migrating
* (Cleaner.isEvictable).
*/
if (ln.isEvictable() && cleaner.isEvictable(this, index)) {
boolean force = getDatabase().isDeferredWriteMode() && getLsn(index) == DbLsn.NULL_LSN;
/* Log target if necessary. */
logDirtyLN(index, (LN) n, force);
/* Clear target. */
setTarget(index, null);
ln.releaseMemoryBudget();
return n.getMemorySizeIncludedByParent();
}
}
return 0;
}
/** @see LN#writeToLog */
public void writeToLog(ByteBuffer logBuffer) {
/*
* Add the tracked (live) summary to the base summary before writing it
* to the log, and reset the tracked summary. Do this even when
* deleting the LN, so that the tracked summary is cleared.
*/
if (trackedSummary != null) {
baseSummary.add(trackedSummary);
if (!isDeleted()) {
getOffsets();
}
/* Reset the totals to zero and clear the tracked offsets. */
trackedSummary.reset();
}
super.writeToLog(logBuffer);
if (!isDeleted()) {
baseSummary.writeToLog(logBuffer);
obsoleteOffsets.writeToLog(logBuffer);
}
}
/**
* Initialize a node that has been faulted in from the log. If this FSLN contains version 1
* offsets that can be incorrect when RMW was used, and if je.cleaner.rmwFix is enabled, discard
* the offsets. [#13158]
*/
public void postFetchInit(DatabaseImpl db, long sourceLsn) throws DatabaseException {
super.postFetchInit(db, sourceLsn);
memBudget = db.getDbEnvironment().getMemoryBudget();
if (entryVersion == 1 && db.getDbEnvironment().getUtilizationProfile().isRMWFixEnabled()) {
obsoleteOffsets = new PackedOffsets();
}
}
/** @see LN#readFromLog */
public void readFromLog(ByteBuffer itemBuffer, byte entryVersion) throws LogException {
this.entryVersion = entryVersion;
super.readFromLog(itemBuffer, entryVersion);
if (!isDeleted()) {
baseSummary.readFromLog(itemBuffer, entryVersion);
if (entryVersion > 0) {
obsoleteOffsets.readFromLog(itemBuffer, entryVersion);
}
}
}
/**
* Note that the IN may or may not be latched when this method is called. Returning the wrong
* answer is OK in that case (it will be called again later when latched), but an exception should
* not occur.
*/
@Override
int getChildEvictionType() {
Cleaner cleaner = getDatabase().getDbEnvironment().getCleaner();
for (int i = 0; i < getNEntries(); i++) {
Node node = getTarget(i);
if (node != null) {
if (node instanceof LN) {
LN ln = (LN) node;
/*
* If the LN is not evictable, we may neither strip the LN
* nor evict the node. isEvictableInexact is used here as
* a fast check, to avoid the overhead of acquiring a
* handle lock while selecting an IN for eviction. See
* evictInternal which will call LN.isEvictable to acquire
* an handle lock and guarantee that another thread cannot
* open the MapLN. [#13415]
*/
if (!ln.isEvictableInexact()) {
return MAY_NOT_EVICT;
}
/*
* If the cleaner allows eviction, then this LN may be
* stripped.
*/
if (cleaner.isEvictable(this, i)) {
return MAY_EVICT_LNS;
}
} else {
return MAY_NOT_EVICT;
}
}
}
return MAY_EVICT_NODE;
}
/**
* Mark this entry as deleted, using the delete flag. Only BINS may do this.
*
* @param index indicates target entry
*/
@Override
public void setKnownDeleted(int index) {
/*
* The target is cleared to save memory, since a known deleted entry
* will never be fetched. The migrate flag is also cleared since
* migration is never needed for known deleted entries either.
*/
super.setKnownDeleted(index);
/*
* We know it's an LN because we never call setKnownDeleted for
* an IN.
*/
LN oldLN = (LN) getTarget(index);
updateMemorySize(oldLN, null /* newNode */);
if (oldLN != null) {
oldLN.releaseMemoryBudget();
}
setMigrate(index, false);
super.setTarget(index, null);
setDirty(true);
}
/**
* Compress this BIN by removing any entries that are deleted. Deleted entries are those that have
* LN's marked deleted or if the knownDeleted flag is set. Caller is responsible for latching and
* unlatching this node.
*
* @param binRef is used to determine the set of keys to be checked for deletedness, or is null to
* check all keys.
* @param canFetch if false, don't fetch any non-resident children. We don't want some callers of
* compress, such as the evictor, to fault in other nodes.
* @return true if we had to requeue the entry because we were unable to get locks, false if all
* entries were processed and therefore any remaining deleted keys in the BINReference must
* now be in some other BIN because of a split.
*/
@Override
public boolean compress(
BINReference binRef, boolean canFetch, LocalUtilizationTracker localTracker)
throws DatabaseException {
boolean ret = false;
boolean setNewIdKey = false;
boolean anyLocksDenied = false;
DatabaseImpl db = getDatabase();
EnvironmentImpl envImpl = db.getDbEnvironment();
BasicLocker lockingTxn = BasicLocker.createBasicLocker(envImpl);
try {
for (int i = 0; i < getNEntries(); i++) {
/*
* We have to be able to lock the LN before we can compress the
* entry. If we can't, then, skip over it.
*
* We must lock the LN even if isKnownDeleted is true, because
* locks protect the aborts. (Aborts may execute multiple
* operations, where each operation latches and unlatches. It's
* the LN lock that protects the integrity of the whole
* multi-step process.)
*
* For example, during abort, there may be cases where we have
* deleted and then added an LN during the same txn. This
* means that to undo/abort it, we first delete the LN (leaving
* knownDeleted set), and then add it back into the tree. We
* want to make sure the entry is in the BIN when we do the
* insert back in.
*/
boolean deleteEntry = false;
Node n = null;
if (binRef == null
|| isEntryPendingDeleted(i)
|| isEntryKnownDeleted(i)
|| binRef.hasDeletedKey(new Key(getKey(i)))) {
if (canFetch) {
if (db.isDeferredWriteMode() && getLsn(i) == DbLsn.NULL_LSN) {
/* Null LSNs are ok in DW. [#15588] */
n = getTarget(i);
} else {
n = fetchTarget(i);
}
} else {
n = getTarget(i);
if (n == null) {
/* Punt, we don't know the state of this child. */
continue;
}
}
if (n == null) {
/* Cleaner deleted the log file. Compress this LN. */
deleteEntry = true;
} else if (isEntryKnownDeleted(i)) {
LockResult lockRet = lockingTxn.nonBlockingLock(n.getNodeId(), LockType.READ, db);
if (lockRet.getLockGrant() == LockGrantType.DENIED) {
anyLocksDenied = true;
continue;
}
deleteEntry = true;
} else {
if (!n.containsDuplicates()) {
LN ln = (LN) n;
LockResult lockRet = lockingTxn.nonBlockingLock(ln.getNodeId(), LockType.READ, db);
if (lockRet.getLockGrant() == LockGrantType.DENIED) {
anyLocksDenied = true;
continue;
}
if (ln.isDeleted()) {
deleteEntry = true;
}
}
}
/* Remove key from BINReference in case we requeue it. */
if (binRef != null) {
binRef.removeDeletedKey(new Key(getKey(i)));
}
}
/* At this point, we know we can delete. */
if (deleteEntry) {
boolean entryIsIdentifierKey =
Key.compareKeys(getKey(i), getIdentifierKey(), getKeyComparator()) == 0;
if (entryIsIdentifierKey) {
/*
* We're about to remove the entry with the idKey so
* the node will need a new idkey.
*/
setNewIdKey = true;
}
/*
* When deleting a deferred-write LN entry, we count the
* last logged LSN as obsolete.
*/
if (localTracker != null && db.isDeferredWriteMode() && n instanceof LN) {
LN ln = (LN) n;
long lsn = getLsn(i);
if (ln.isDirty() && lsn != DbLsn.NULL_LSN) {
localTracker.countObsoleteNode(lsn, ln.getLogType(), ln.getLastLoggedSize(), db);
}
}
boolean deleteSuccess = deleteEntry(i, true);
assert deleteSuccess;
/*
* Since we're deleting the current entry, bump the current
* index back down one.
*/
i--;
}
}
} finally {
if (lockingTxn != null) {
lockingTxn.operationEnd();
}
}
if (anyLocksDenied && binRef != null) {
db.getDbEnvironment().addToCompressorQueue(binRef, false);
ret = true;
}
if (getNEntries() != 0 && setNewIdKey) {
setIdentifierKey(getKey(0));
}
/* This BIN is empty and expendable. */
if (getNEntries() == 0) {
setGeneration(0);
}
return ret;
}
/** @see LN#readFromLog */
public void readFromLog(ByteBuffer itemBuffer, byte entryTypeVersion) throws LogException {
super.readFromLog(itemBuffer, entryTypeVersion);
databaseImpl.readFromLog(itemBuffer, entryTypeVersion);
deleted = LogUtils.readBoolean(itemBuffer);
}
/** @see LN#writeToLog */
public void writeToLog(ByteBuffer logBuffer) {
super.writeToLog(logBuffer);
databaseImpl.writeToLog(logBuffer);
LogUtils.writeBoolean(logBuffer, deleted);
}
/** Clear out the obsoleteOffsets to save memory when the LN is deleted. */
@Override
void makeDeleted() {
super.makeDeleted();
obsoleteOffsets = new PackedOffsets();
}
public void testEntryData() throws Throwable {
try {
ByteBuffer buffer = ByteBuffer.allocate(1000);
database = new DatabaseImpl("foo", new DatabaseId(1), env, new DatabaseConfig());
/*
* For each loggable object, can we write the entry data out?
*/
/*
* Tracer records.
*/
Tracer dMsg = new Tracer("Hello there");
writeAndRead(buffer, LogEntryType.LOG_TRACE, dMsg, new Tracer());
/*
* LNs
*/
String data = "abcdef";
LN ln = new LN(data.getBytes());
LN lnFromLog = new LN();
writeAndRead(buffer, LogEntryType.LOG_LN, ln, lnFromLog);
lnFromLog.verify(null);
assertTrue(LogEntryType.LOG_LN.marshallOutsideLatch());
FileSummaryLN fsLN = new FileSummaryLN(new FileSummary());
FileSummaryLN fsLNFromLog = new FileSummaryLN();
writeAndRead(buffer, LogEntryType.LOG_FILESUMMARYLN, fsLN, fsLNFromLog);
assertFalse(LogEntryType.LOG_FILESUMMARYLN.marshallOutsideLatch());
/*
* INs
*/
IN in = new IN(database, new byte[] {1, 0, 1, 0}, 7, 5);
in.latch();
in.insertEntry(new ChildReference(null, new byte[] {1, 0, 1, 0}, DbLsn.makeLsn(12, 200)));
in.insertEntry(new ChildReference(null, new byte[] {1, 1, 1, 0}, DbLsn.makeLsn(29, 300)));
in.insertEntry(new ChildReference(null, new byte[] {0, 0, 1, 0}, DbLsn.makeLsn(35, 400)));
/* Write it. */
IN inFromLog = new IN();
inFromLog.latch();
writeAndRead(buffer, LogEntryType.LOG_IN, in, inFromLog);
inFromLog.releaseLatch();
in.releaseLatch();
/*
* IN - long form
*/
in = new IN(database, new byte[] {1, 0, 1, 0}, 7, 5);
in.latch();
in.insertEntry(new ChildReference(null, new byte[] {1, 0, 1, 0}, DbLsn.makeLsn(12, 200)));
in.insertEntry(new ChildReference(null, new byte[] {1, 1, 1, 0}, DbLsn.makeLsn(29, 300)));
in.insertEntry(new ChildReference(null, new byte[] {0, 0, 1, 0}, DbLsn.makeLsn(1235, 400)));
in.insertEntry(
new ChildReference(null, new byte[] {0, 0, 1, 0}, DbLsn.makeLsn(0xFFFFFFF0L, 400)));
/* Write it. */
inFromLog = new IN();
inFromLog.latch();
writeAndRead(buffer, LogEntryType.LOG_IN, in, inFromLog);
inFromLog.releaseLatch();
in.releaseLatch();
/*
* BINs
*/
BIN bin = new BIN(database, new byte[] {3, 2, 1}, 8, 5);
bin.latch();
bin.insertEntry(new ChildReference(null, new byte[] {1, 0, 1, 0}, DbLsn.makeLsn(212, 200)));
bin.insertEntry(new ChildReference(null, new byte[] {1, 1, 1, 0}, DbLsn.makeLsn(229, 300)));
bin.insertEntry(new ChildReference(null, new byte[] {0, 0, 1, 0}, DbLsn.makeLsn(235, 400)));
BIN binFromLog = new BIN();
binFromLog.latch();
writeAndRead(buffer, LogEntryType.LOG_BIN, bin, binFromLog);
binFromLog.verify(null);
binFromLog.releaseLatch();
bin.releaseLatch();
/*
* DINs
*/
DIN din =
new DIN(
database,
new byte[] {1, 0, 0, 1},
7,
new byte[] {0, 1, 1, 0},
new ChildReference(null, new byte[] {1, 0, 0, 1}, DbLsn.makeLsn(10, 100)),
5);
din.latch();
din.insertEntry(new ChildReference(null, new byte[] {1, 0, 1, 0}, DbLsn.makeLsn(12, 200)));
din.insertEntry(new ChildReference(null, new byte[] {1, 1, 1, 0}, DbLsn.makeLsn(29, 300)));
din.insertEntry(new ChildReference(null, new byte[] {0, 0, 1, 0}, DbLsn.makeLsn(35, 400)));
/* Write it. */
DIN dinFromLog = new DIN();
dinFromLog.latch();
writeAndRead(buffer, LogEntryType.LOG_DIN, din, dinFromLog);
din.releaseLatch();
dinFromLog.releaseLatch();
/*
* DBINs
*/
DBIN dbin = new DBIN(database, new byte[] {3, 2, 1}, 8, new byte[] {1, 2, 3}, 5);
dbin.latch();
dbin.insertEntry(new ChildReference(null, new byte[] {1, 0, 1, 0}, DbLsn.makeLsn(212, 200)));
dbin.insertEntry(new ChildReference(null, new byte[] {1, 1, 1, 0}, DbLsn.makeLsn(229, 300)));
dbin.insertEntry(new ChildReference(null, new byte[] {0, 0, 1, 0}, DbLsn.makeLsn(235, 400)));
DBIN dbinFromLog = new DBIN();
dbinFromLog.latch();
writeAndRead(buffer, LogEntryType.LOG_DBIN, dbin, dbinFromLog);
dbinFromLog.verify(null);
dbin.releaseLatch();
dbinFromLog.releaseLatch();
/*
* Root
*/
DbTree dbTree = new DbTree(env);
DbTree dbTreeFromLog = new DbTree();
writeAndRead(buffer, LogEntryType.LOG_ROOT, dbTree, dbTreeFromLog);
/*
* MapLN
*/
MapLN mapLn = new MapLN(database);
MapLN mapLnFromLog = new MapLN();
writeAndRead(buffer, LogEntryType.LOG_MAPLN, mapLn, mapLnFromLog);
/*
* UserTxn
*/
/*
* Disabled for now because these txns don't compare equal,
* because one has a name of "main" and the other has a name of
* null because it was read from the log.
Txn txn = new Txn(env, new TransactionConfig());
Txn txnFromLog = new Txn();
writeAndRead(buffer, LogEntryType.TXN_COMMIT, txn, txnFromLog);
txn.commit();
*/
/*
* TxnCommit
*/
TxnCommit commit = new TxnCommit(111, DbLsn.makeLsn(10, 10));
TxnCommit commitFromLog = new TxnCommit();
writeAndRead(buffer, LogEntryType.LOG_TXN_COMMIT, commit, commitFromLog);
/*
* TxnAbort
*/
TxnAbort abort = new TxnAbort(111, DbLsn.makeLsn(11, 11));
TxnAbort abortFromLog = new TxnAbort();
writeAndRead(buffer, LogEntryType.LOG_TXN_ABORT, abort, abortFromLog);
/*
* TxnPrepare
*/
byte[] gid = new byte[64];
byte[] bqual = new byte[64];
TxnPrepare prepare = new TxnPrepare(111, new LogUtils.XidImpl(1, gid, bqual));
TxnPrepare prepareFromLog = new TxnPrepare();
writeAndRead(buffer, LogEntryType.LOG_TXN_PREPARE, prepare, prepareFromLog);
prepare = new TxnPrepare(111, new LogUtils.XidImpl(1, null, bqual));
prepareFromLog = new TxnPrepare();
writeAndRead(buffer, LogEntryType.LOG_TXN_PREPARE, prepare, prepareFromLog);
prepare = new TxnPrepare(111, new LogUtils.XidImpl(1, gid, null));
prepareFromLog = new TxnPrepare();
writeAndRead(buffer, LogEntryType.LOG_TXN_PREPARE, prepare, prepareFromLog);
/*
* IN delete info
*/
INDeleteInfo info = new INDeleteInfo(77, new byte[1], new DatabaseId(100));
INDeleteInfo infoFromLog = new INDeleteInfo();
writeAndRead(buffer, LogEntryType.LOG_IN_DELETE_INFO, info, infoFromLog);
/*
* Checkpoint start
*/
CheckpointStart start = new CheckpointStart(177, "test");
CheckpointStart startFromLog = new CheckpointStart();
writeAndRead(buffer, LogEntryType.LOG_CKPT_START, start, startFromLog);
/*
* Checkpoint end
*/
CheckpointEnd end =
new CheckpointEnd(
"test",
DbLsn.makeLsn(20, 55),
env.getRootLsn(),
env.getTxnManager().getFirstActiveLsn(),
Node.getLastId(),
env.getDbMapTree().getLastDbId(),
env.getTxnManager().getLastTxnId(),
177);
CheckpointEnd endFromLog = new CheckpointEnd();
writeAndRead(buffer, LogEntryType.LOG_CKPT_END, end, endFromLog);
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}