/** RaceDet: get recursion count, assuming lock is held by current thread */
@Uninterruptible
@NoNullCheck
@Inline
public static final int getRecursionCountLocked(Object o, Offset lockOffset) {
if (VM.VerifyAssertions) {
VM._assert(holdsLock(o, lockOffset, RVMThread.getCurrentThread()));
}
Word bits = Magic.getWordAtOffset(o, lockOffset);
int count;
if (bits.and(TL_STAT_MASK).EQ(TL_STAT_FAT)) {
// if locked, then it is locked with a fat lock
Lock l = Lock.getLock(getLockIndex(bits));
l.mutex.lock();
count = l.getRecursionCount();
l.mutex.unlock();
} else {
if (VM.VerifyAssertions) {
VM._assert(
bits.and(TL_STAT_MASK).EQ(TL_STAT_BIASABLE) || bits.and(TL_STAT_MASK).EQ(TL_STAT_THIN));
}
count = getRecCount(bits);
}
if (VM.VerifyAssertions) {
VM._assert(count > 0);
}
return count;
}
@Inline
@Uninterruptible
public static int getRecCount(Word lockWord) {
if (VM.VerifyAssertions) VM._assert(getLockOwner(lockWord) != 0);
if (lockWord.and(TL_STAT_MASK).EQ(TL_STAT_BIASABLE)) {
return lockWord.and(TL_LOCK_COUNT_MASK).rshl(TL_LOCK_COUNT_SHIFT).toInt();
} else {
return lockWord.and(TL_LOCK_COUNT_MASK).rshl(TL_LOCK_COUNT_SHIFT).toInt() + 1;
}
}
/**
* Atomically attempt to set the mark bit of an object. Return true if successful, false if the
* mark bit was already set.
*
* @param object The object whose mark bit is to be written
* @param value The value to which the mark bit will be set
*/
@Inline
private boolean testAndMark(ObjectReference object, Word value) {
Word oldValue, markBit;
do {
oldValue = VM.objectModel.prepareAvailableBits(object);
markBit = oldValue.and(inNurseryGC ? LOS_BIT_MASK : MARK_BIT);
if (markBit.EQ(value)) return false;
} while (!VM.objectModel.attemptAvailableBits(
object, oldValue, oldValue.and(LOS_BIT_MASK.not()).or(value)));
return true;
}
@Inline
@Uninterruptible
public static int getLockOwner(Word lockWord) {
if (VM.VerifyAssertions) VM._assert(!isFat(lockWord));
if (lockWord.and(TL_STAT_MASK).EQ(TL_STAT_BIASABLE)) {
if (lockWord.and(TL_LOCK_COUNT_MASK).isZero()) {
return 0;
} else {
return lockWord.and(TL_THREAD_ID_MASK).toInt();
}
} else {
return lockWord.and(TL_THREAD_ID_MASK).toInt();
}
}
/** atomically write the given value in the mark bit. */
private static void atomicWriteMarkBit(ObjectReference object, Word value) {
while (true) {
Word oldValue = ObjectModel.prepareAvailableBits(object);
Word newValue = oldValue.and(GC_MARK_BIT_MASK.not()).or(value);
if (ObjectModel.attemptAvailableBits(object, oldValue, newValue)) break;
}
}
@Inline
@Uninterruptible
public static boolean attemptToMarkDeflated(Object o, Offset lockOffset, Word oldLockWord) {
// we allow concurrent modification of the lock word when it's thin or fat.
Word changed = oldLockWord.and(TL_UNLOCK_MASK).or(TL_STAT_THIN);
if (VM.VerifyAssertions) VM._assert(getLockOwner(changed) == 0);
return Synchronization.tryCompareAndSwap(o, lockOffset, oldLockWord, changed);
}
/**
* Used to mark boot image objects during a parallel scan of objects during GC Returns true if
* marking was done.
*/
private static boolean testAndMark(ObjectReference object, Word value) throws InlinePragma {
Word oldValue;
do {
oldValue = ObjectModel.prepareAvailableBits(object);
Word markBit = oldValue.and(GC_MARK_BIT_MASK);
if (markBit.EQ(value)) return false;
} while (!ObjectModel.attemptAvailableBits(object, oldValue, oldValue.xor(GC_MARK_BIT_MASK)));
return true;
}
/**
* Return the lock index for a given lock word. Assert valid index ranges, that the fat lock bit
* is set, and that the lock entry exists.
*
* @param lockWord The lock word whose lock index is being established
* @return the lock index corresponding to the lock workd.
*/
@Inline
@Uninterruptible
public static int getLockIndex(Word lockWord) {
int index = lockWord.and(TL_LOCK_ID_MASK).rshl(TL_LOCK_ID_SHIFT).toInt();
if (VM.VerifyAssertions) {
if (!(index > 0 && index < Lock.numLocks())) {
VM.sysWrite("Lock index out of range! Word: ");
VM.sysWrite(lockWord);
VM.sysWrite(" index: ");
VM.sysWrite(index);
VM.sysWrite(" locks: ");
VM.sysWrite(Lock.numLocks());
VM.sysWriteln();
}
VM._assert(index > 0 && index < Lock.numLocks()); // index is in range
VM._assert(lockWord.and(TL_STAT_MASK).EQ(TL_STAT_FAT)); // fat lock bit is set
}
return index;
}
/**
* Perform any required initialization of the GC portion of the header.
*
* @param object the object ref to the storage to be initialized
* @param alloc is this initialization occuring due to (initial) allocation (true) or due to
* copying (false)?
*/
@Inline
public void initializeHeader(ObjectReference object, boolean alloc) {
Word oldValue = VM.objectModel.readAvailableBitsWord(object);
Word newValue = oldValue.and(LOS_BIT_MASK.not()).or(markState);
if (alloc) newValue = newValue.or(NURSERY_BIT);
if (Plan.NEEDS_LOG_BIT_IN_HEADER) newValue = newValue.or(Plan.UNLOGGED_BIT);
VM.objectModel.writeAvailableBitsWord(object, newValue);
Address cell = VM.objectModel.objectStartRef(object);
treadmill.addToTreadmill(Treadmill.midPayloadToNode(cell), alloc);
}
@Inline
@NoNullCheck
@Unpreemptible
public static void inlineUnlockHelper(Object o, Offset lockOffset) {
Word old = Magic.prepareWord(o, lockOffset); // FIXME: bad for PPC?
Word id = old.and(TL_THREAD_ID_MASK.or(TL_STAT_MASK));
Word tid = Word.fromIntSignExtend(RVMThread.getCurrentThread().getLockingId());
if (id.EQ(tid)) {
if (!old.and(TL_LOCK_COUNT_MASK).isZero()) {
setDedicatedU16(o, lockOffset, old.minus(TL_LOCK_COUNT_UNIT));
return;
}
} else if (old.xor(tid).rshl(TL_LOCK_COUNT_SHIFT).EQ(TL_STAT_THIN.rshl(TL_LOCK_COUNT_SHIFT))) {
Magic.sync();
if (Magic.attemptWord(o, lockOffset, old, old.and(TL_UNLOCK_MASK).or(TL_STAT_THIN))) {
return;
}
}
unlock(o, lockOffset);
}
@Uninterruptible
@NoNullCheck
public static boolean holdsLock(Object o, Offset lockOffset, RVMThread thread) {
for (int cnt = 0; ; ++cnt) {
int tid = thread.getLockingId();
Word bits = Magic.getWordAtOffset(o, lockOffset);
if (bits.and(TL_STAT_MASK).EQ(TL_STAT_BIASABLE)) {
// if locked, then it is locked with a thin lock
return bits.and(TL_THREAD_ID_MASK).toInt() == tid && !bits.and(TL_LOCK_COUNT_MASK).isZero();
} else if (bits.and(TL_STAT_MASK).EQ(TL_STAT_THIN)) {
return bits.and(TL_THREAD_ID_MASK).toInt() == tid;
} else {
if (VM.VerifyAssertions) VM._assert(bits.and(TL_STAT_MASK).EQ(TL_STAT_FAT));
// if locked, then it is locked with a fat lock
Lock l = Lock.getLock(getLockIndex(bits));
if (l != null) {
l.mutex.lock();
boolean result = (l.getOwnerId() == tid && l.getLockedObject() == o);
l.mutex.unlock();
return result;
}
}
RVMThread.yield();
}
}
@Inline
@NoNullCheck
@Unpreemptible
public static void inlineLockHelper(Object o, Offset lockOffset) {
Word old = Magic.prepareWord(o, lockOffset); // FIXME: bad for PPC?
Word id = old.and(TL_THREAD_ID_MASK.or(TL_STAT_MASK));
Word tid = Word.fromIntSignExtend(RVMThread.getCurrentThread().getLockingId());
if (id.EQ(tid)) {
Word changed = old.plus(TL_LOCK_COUNT_UNIT);
if (!changed.and(TL_LOCK_COUNT_MASK).isZero()) {
setDedicatedU16(o, lockOffset, changed);
return;
}
} else if (id.EQ(TL_STAT_THIN)) {
// lock is thin and not held by anyone
if (Magic.attemptWord(o, lockOffset, old, old.or(tid))) {
Magic.isync();
return;
}
}
lock(o, lockOffset);
}
@Inline
@Unpreemptible
public static boolean attemptToMarkInflated(
Object o, Offset lockOffset, Word oldLockWord, int lockId, int cnt) {
if (VM.VerifyAssertions) VM._assert(oldLockWord.and(TL_STAT_MASK).NE(TL_STAT_FAT));
if (false) VM.sysWriteln("attemptToMarkInflated with oldLockWord = ", oldLockWord);
// what this needs to do:
// 1) if the lock is thin, it's just a CAS
// 2) if the lock is unbiased, CAS in the inflation
// 3) if the lock is biased in our favor, store the lock without CAS
// 4) if the lock is biased but to someone else, enter the pair handshake
// to unbias it and install the inflated lock
Word changed =
TL_STAT_FAT
.or(Word.fromIntZeroExtend(lockId).lsh(TL_LOCK_ID_SHIFT))
.or(oldLockWord.and(TL_UNLOCK_MASK));
if (false && oldLockWord.and(TL_STAT_MASK).EQ(TL_STAT_THIN))
VM.sysWriteln(
"obj = ",
Magic.objectAsAddress(o),
", old = ",
oldLockWord,
", owner = ",
getLockOwner(oldLockWord),
", rec = ",
getLockOwner(oldLockWord) == 0 ? 0 : getRecCount(oldLockWord),
", changed = ",
changed,
", lockId = ",
lockId);
if (false) VM.sysWriteln("changed = ", changed);
if (oldLockWord.and(TL_STAT_MASK).EQ(TL_STAT_THIN)) {
if (false) VM.sysWriteln("it's thin, inflating the easy way.");
return Synchronization.tryCompareAndSwap(o, lockOffset, oldLockWord, changed);
} else {
return casFromBiased(o, lockOffset, oldLockWord, changed, cnt);
}
}
@NoInline
@Unpreemptible
public static boolean casFromBiased(
Object o, Offset lockOffset, Word oldLockWord, Word changed, int cnt) {
RVMThread me = RVMThread.getCurrentThread();
Word id = oldLockWord.and(TL_THREAD_ID_MASK);
if (id.isZero()) {
if (false) VM.sysWriteln("id is zero - easy case.");
return Synchronization.tryCompareAndSwap(o, lockOffset, oldLockWord, changed);
} else {
if (false) VM.sysWriteln("id = ", id);
int slot = id.toInt() >> TL_THREAD_ID_SHIFT;
if (false) VM.sysWriteln("slot = ", slot);
RVMThread owner = RVMThread.threadBySlot[slot];
if (owner == me /* I own it, so I can unbias it trivially. This occurs
when we are inflating due to, for example, wait() */
|| owner == null /* the thread that owned it is dead, so it's safe to
unbias. */) {
// note that we use a CAS here, but it's only needed in the case
// that owner==null, since in that case some other thread may also
// be unbiasing.
return Synchronization.tryCompareAndSwap(o, lockOffset, oldLockWord, changed);
} else {
boolean result = false;
// NB. this may stop a thread other than the one that had the bias,
// if that thread died and some other thread took its slot. that's
// why we do a CAS below. it's only needed if some other thread
// had seen the owner be null (which may happen if we came here after
// a new thread took the slot while someone else came here when the
// slot was still null). if it was the case that everyone else had
// seen a non-null owner, then the pair handshake would serve as
// sufficient synchronization (the id would identify the set of threads
// that shared that id's communicationLock). oddly, that means that
// this whole thing could be "simplified" to acquire the
// communicationLock even if the owner was null. but that would be
// goofy.
if (false) VM.sysWriteln("entering pair handshake");
owner.beginPairHandshake();
if (false) VM.sysWriteln("done with that");
Word newLockWord = Magic.getWordAtOffset(o, lockOffset);
result = Synchronization.tryCompareAndSwap(o, lockOffset, oldLockWord, changed);
owner.endPairHandshake();
if (false) VM.sysWriteln("that worked.");
return result;
}
}
}
@Inline
@Uninterruptible
private static Word biasBitsToThinBits(Word bits) {
int lockOwner = getLockOwner(bits);
Word changed = bits.and(TL_UNLOCK_MASK).or(TL_STAT_THIN);
if (lockOwner != 0) {
int recCount = getRecCount(bits);
changed =
changed
.or(Word.fromIntZeroExtend(lockOwner))
.or(Word.fromIntZeroExtend(recCount - 1).lsh(TL_LOCK_COUNT_SHIFT));
}
return changed;
}
@NoInline
@NoNullCheck
@Unpreemptible
public static void unlock(Object o, Offset lockOffset) {
Word threadId = Word.fromIntZeroExtend(RVMThread.getCurrentThread().getLockingId());
for (int cnt = 0; ; cnt++) {
Word old = Magic.getWordAtOffset(o, lockOffset);
Word stat = old.and(TL_STAT_MASK);
if (stat.EQ(TL_STAT_BIASABLE)) {
Word id = old.and(TL_THREAD_ID_MASK);
if (id.EQ(threadId)) {
if (old.and(TL_LOCK_COUNT_MASK).isZero()) {
RVMThread.raiseIllegalMonitorStateException(
"biased unlocking: we own this object but the count is already zero", o);
}
setDedicatedU16(o, lockOffset, old.minus(TL_LOCK_COUNT_UNIT));
return;
} else {
RVMThread.raiseIllegalMonitorStateException(
"biased unlocking: we don't own this object", o);
}
} else if (stat.EQ(TL_STAT_THIN)) {
Magic.sync();
Word id = old.and(TL_THREAD_ID_MASK);
if (id.EQ(threadId)) {
Word changed;
if (old.and(TL_LOCK_COUNT_MASK).isZero()) {
changed = old.and(TL_UNLOCK_MASK).or(TL_STAT_THIN);
} else {
changed = old.minus(TL_LOCK_COUNT_UNIT);
}
if (Synchronization.tryCompareAndSwap(o, lockOffset, old, changed)) {
return;
}
} else {
if (false) {
VM.sysWriteln("threadId = ", threadId);
VM.sysWriteln("id = ", id);
}
RVMThread.raiseIllegalMonitorStateException(
"thin unlocking: we don't own this object", o);
}
} else {
if (VM.VerifyAssertions) VM._assert(stat.EQ(TL_STAT_FAT));
// fat unlock
Lock.getLock(getLockIndex(old)).unlockHeavy(o);
return;
}
}
}
private static final void assertAligned(Word value) {
Assert._assert(value.and(Word.fromIntSignExtend(BYTES_IN_INT - 1)).isZero());
}
@NoInline
@NoNullCheck
@Unpreemptible
public static void lock(Object o, Offset lockOffset) {
if (STATS) fastLocks++;
Word threadId = Word.fromIntZeroExtend(RVMThread.getCurrentThread().getLockingId());
for (int cnt = 0; ; cnt++) {
Word old = Magic.getWordAtOffset(o, lockOffset);
Word stat = old.and(TL_STAT_MASK);
boolean tryToInflate = false;
if (stat.EQ(TL_STAT_BIASABLE)) {
Word id = old.and(TL_THREAD_ID_MASK);
if (id.isZero()) {
if (ENABLE_BIASED_LOCKING) {
// lock is unbiased, bias it in our favor and grab it
if (Synchronization.tryCompareAndSwap(
o, lockOffset, old, old.or(threadId).plus(TL_LOCK_COUNT_UNIT))) {
Magic.isync();
return;
}
} else {
// lock is unbiased but biasing is NOT allowed, so turn it into
// a thin lock
if (Synchronization.tryCompareAndSwap(
o, lockOffset, old, old.or(threadId).or(TL_STAT_THIN))) {
Magic.isync();
return;
}
}
} else if (id.EQ(threadId)) {
// lock is biased in our favor
Word changed = old.plus(TL_LOCK_COUNT_UNIT);
if (!changed.and(TL_LOCK_COUNT_MASK).isZero()) {
setDedicatedU16(o, lockOffset, changed);
return;
} else {
tryToInflate = true;
}
} else {
if (casFromBiased(o, lockOffset, old, biasBitsToThinBits(old), cnt)) {
continue; // don't spin, since it's thin now
}
}
} else if (stat.EQ(TL_STAT_THIN)) {
Word id = old.and(TL_THREAD_ID_MASK);
if (id.isZero()) {
if (Synchronization.tryCompareAndSwap(o, lockOffset, old, old.or(threadId))) {
Magic.isync();
return;
}
} else if (id.EQ(threadId)) {
Word changed = old.plus(TL_LOCK_COUNT_UNIT);
if (changed.and(TL_LOCK_COUNT_MASK).isZero()) {
tryToInflate = true;
} else if (Synchronization.tryCompareAndSwap(o, lockOffset, old, changed)) {
Magic.isync();
return;
}
} else if (cnt > retryLimit) {
tryToInflate = true;
}
} else {
if (VM.VerifyAssertions) VM._assert(stat.EQ(TL_STAT_FAT));
// lock is fat. contend on it.
if (Lock.getLock(getLockIndex(old)).lockHeavy(o)) {
return;
}
}
if (tryToInflate) {
if (STATS) slowLocks++;
// the lock is not fat, is owned by someone else, or else the count wrapped.
// attempt to inflate it (this may fail, in which case we'll just harmlessly
// loop around) and lock it (may also fail, if we get the wrong lock). if it
// succeeds, we're done.
// NB: this calls into our attemptToMarkInflated() method, which will do the
// Right Thing if the lock is biased to someone else.
if (inflateAndLock(o, lockOffset)) {
return;
}
} else {
RVMThread.yield();
}
}
}
/** write the given value in the mark bit. */
private static void writeMarkBit(ObjectReference object, Word value) {
Word oldValue = ObjectModel.readAvailableBitsWord(object);
Word newValue = oldValue.and(GC_MARK_BIT_MASK.not()).or(value);
ObjectModel.writeAvailableBitsWord(object, newValue);
}
@Inline
@Uninterruptible
public static boolean isFat(Word lockWord) {
return lockWord.and(TL_STAT_MASK).EQ(TL_STAT_FAT);
}