/**
* Copy numbytes from src to dst. Assumption either the ranges are non overlapping, or src >= dst
* + 4. Also, src and dst are 4 byte aligned and numBytes is a multiple of 4.
*
* @param dst the destination addr
* @param src the source addr
* @param numBytes the number of bytes top copy
*/
@Inline
public static void aligned32Copy(Address dst, Address src, Offset numBytes) {
if (USE_NATIVE && numBytes.sGT(Offset.fromIntSignExtend(NATIVE_THRESHOLD))) {
memcopy(dst, src, numBytes.toWord().toExtent());
} else {
if (VM.BuildFor64Addr) {
Word wordMask = Word.one().lsh(LOG_BYTES_IN_ADDRESS).minus(Word.one());
Word srcAlignment = src.toWord().and(wordMask);
if (srcAlignment.EQ(dst.toWord().and(wordMask))) {
Offset i = Offset.zero();
if (srcAlignment.EQ(Word.fromIntZeroExtend(BYTES_IN_INT))) {
dst.store(src.loadInt(i), i);
i = i.plus(BYTES_IN_INT);
}
Word endAlignment =
srcAlignment.plus(numBytes).and(Word.fromIntSignExtend(BYTES_IN_ADDRESS - 1));
numBytes = numBytes.minus(endAlignment.toOffset());
for (; i.sLT(numBytes); i = i.plus(BYTES_IN_ADDRESS)) {
dst.store(src.loadWord(i), i);
}
if (!endAlignment.isZero()) {
dst.store(src.loadInt(i), i);
}
return;
}
}
// normal case: 32 bit or (64 bit not aligned)
for (Offset i = Offset.zero(); i.sLT(numBytes); i = i.plus(BYTES_IN_INT)) {
dst.store(src.loadInt(i), i);
}
}
}
@Inline
@Uninterruptible
private static Word biasBitsToThinBits(Word bits) {
int lockOwner = getLockOwner(bits);
Word changed = bits.and(TL_UNLOCK_MASK).or(TL_STAT_THIN);
if (lockOwner != 0) {
int recCount = getRecCount(bits);
changed =
changed
.or(Word.fromIntZeroExtend(lockOwner))
.or(Word.fromIntZeroExtend(recCount - 1).lsh(TL_LOCK_COUNT_SHIFT));
}
return changed;
}
/**
* Copy copyBytes from src to dst. Assumption: either the ranges are non overlapping, or {@code
* src >= dst + 4}. Also, src and dst are 4 byte aligned and numBytes is a multiple of 4.
*
* @param dst the destination addr
* @param src the source addr
* @param copyBytes the number of bytes top copy
*/
public static void aligned32Copy(Address dst, Address src, int copyBytes) {
if (VM.VerifyAssertions) {
VM._assert(copyBytes >= 0);
VM._assert((copyBytes & (BYTES_IN_INT - 1)) == 0);
VM._assert(src.toWord().and(Word.fromIntZeroExtend(BYTES_IN_INT - 1)).isZero());
VM._assert(dst.toWord().and(Word.fromIntZeroExtend(BYTES_IN_INT - 1)).isZero());
VM._assert(src.plus(copyBytes).LE(dst) || src.GE(dst.plus(BYTES_IN_INT)));
}
if (USE_NATIVE && copyBytes > NATIVE_THRESHOLD) {
memcopy(dst, src, copyBytes);
} else {
Offset numBytes = Offset.fromIntSignExtend(copyBytes);
if (BYTES_IN_COPY == 8 && copyBytes != 0) {
Word wordMask = Word.fromIntZeroExtend(BYTES_IN_COPY - 1);
Word srcAlignment = src.toWord().and(wordMask);
if (srcAlignment.EQ(dst.toWord().and(wordMask))) {
Offset i = Offset.zero();
if (srcAlignment.EQ(Word.fromIntZeroExtend(BYTES_IN_INT))) {
copy4Bytes(dst.plus(i), src.plus(i));
i = i.plus(BYTES_IN_INT);
}
Word endAlignment = srcAlignment.plus(numBytes).and(wordMask);
numBytes = numBytes.minus(endAlignment.toOffset());
for (; i.sLT(numBytes); i = i.plus(BYTES_IN_COPY)) {
copy8Bytes(dst.plus(i), src.plus(i));
}
if (!endAlignment.isZero()) {
copy4Bytes(dst.plus(i), src.plus(i));
}
return;
}
}
// normal case: 32 bit or (64 bit not aligned)
for (Offset i = Offset.zero(); i.sLT(numBytes); i = i.plus(BYTES_IN_INT)) {
copy4Bytes(dst.plus(i), src.plus(i));
}
}
}
@NoInline
@NoNullCheck
@Unpreemptible
public static void unlock(Object o, Offset lockOffset) {
Word threadId = Word.fromIntZeroExtend(RVMThread.getCurrentThread().getLockingId());
for (int cnt = 0; ; cnt++) {
Word old = Magic.getWordAtOffset(o, lockOffset);
Word stat = old.and(TL_STAT_MASK);
if (stat.EQ(TL_STAT_BIASABLE)) {
Word id = old.and(TL_THREAD_ID_MASK);
if (id.EQ(threadId)) {
if (old.and(TL_LOCK_COUNT_MASK).isZero()) {
RVMThread.raiseIllegalMonitorStateException(
"biased unlocking: we own this object but the count is already zero", o);
}
setDedicatedU16(o, lockOffset, old.minus(TL_LOCK_COUNT_UNIT));
return;
} else {
RVMThread.raiseIllegalMonitorStateException(
"biased unlocking: we don't own this object", o);
}
} else if (stat.EQ(TL_STAT_THIN)) {
Magic.sync();
Word id = old.and(TL_THREAD_ID_MASK);
if (id.EQ(threadId)) {
Word changed;
if (old.and(TL_LOCK_COUNT_MASK).isZero()) {
changed = old.and(TL_UNLOCK_MASK).or(TL_STAT_THIN);
} else {
changed = old.minus(TL_LOCK_COUNT_UNIT);
}
if (Synchronization.tryCompareAndSwap(o, lockOffset, old, changed)) {
return;
}
} else {
if (false) {
VM.sysWriteln("threadId = ", threadId);
VM.sysWriteln("id = ", id);
}
RVMThread.raiseIllegalMonitorStateException(
"thin unlocking: we don't own this object", o);
}
} else {
if (VM.VerifyAssertions) VM._assert(stat.EQ(TL_STAT_FAT));
// fat unlock
Lock.getLock(getLockIndex(old)).unlockHeavy(o);
return;
}
}
}
@Inline
@Unpreemptible
public static boolean attemptToMarkInflated(
Object o, Offset lockOffset, Word oldLockWord, int lockId, int cnt) {
if (VM.VerifyAssertions) VM._assert(oldLockWord.and(TL_STAT_MASK).NE(TL_STAT_FAT));
if (false) VM.sysWriteln("attemptToMarkInflated with oldLockWord = ", oldLockWord);
// what this needs to do:
// 1) if the lock is thin, it's just a CAS
// 2) if the lock is unbiased, CAS in the inflation
// 3) if the lock is biased in our favor, store the lock without CAS
// 4) if the lock is biased but to someone else, enter the pair handshake
// to unbias it and install the inflated lock
Word changed =
TL_STAT_FAT
.or(Word.fromIntZeroExtend(lockId).lsh(TL_LOCK_ID_SHIFT))
.or(oldLockWord.and(TL_UNLOCK_MASK));
if (false && oldLockWord.and(TL_STAT_MASK).EQ(TL_STAT_THIN))
VM.sysWriteln(
"obj = ",
Magic.objectAsAddress(o),
", old = ",
oldLockWord,
", owner = ",
getLockOwner(oldLockWord),
", rec = ",
getLockOwner(oldLockWord) == 0 ? 0 : getRecCount(oldLockWord),
", changed = ",
changed,
", lockId = ",
lockId);
if (false) VM.sysWriteln("changed = ", changed);
if (oldLockWord.and(TL_STAT_MASK).EQ(TL_STAT_THIN)) {
if (false) VM.sysWriteln("it's thin, inflating the easy way.");
return Synchronization.tryCompareAndSwap(o, lockOffset, oldLockWord, changed);
} else {
return casFromBiased(o, lockOffset, oldLockWord, changed, cnt);
}
}
/** Each instance of this class corresponds to one explicitly managed large object space. */
@Uninterruptible
public final class LargeObjectSpace extends BaseLargeObjectSpace {
/**
* **************************************************************************
*
* <p>Class variables
*/
public static final int LOCAL_GC_BITS_REQUIRED = 2;
public static final int GLOBAL_GC_BITS_REQUIRED = 0;
private static final Word MARK_BIT = Word.one(); // ...01
private static final Word NURSERY_BIT = Word.fromIntZeroExtend(2); // ...10
private static final Word LOS_BIT_MASK = Word.fromIntZeroExtend(3); // ...11
/**
* **************************************************************************
*
* <p>Instance variables
*/
private Word markState;
private boolean inNurseryGC;
private final Treadmill treadmill;
/**
* **************************************************************************
*
* <p>Initialization
*/
/**
* The caller specifies the region of virtual memory to be used for this space. If this region
* conflicts with an existing space, then the constructor will fail.
*
* @param name The name of this space (used when printing error messages etc)
* @param pageBudget The number of pages this space may consume before consulting the plan
* @param vmRequest An object describing the virtual memory requested.
*/
public LargeObjectSpace(String name, int pageBudget, VMRequest vmRequest) {
super(name, pageBudget, vmRequest);
treadmill = new Treadmill(LOG_BYTES_IN_PAGE, true);
markState = Word.zero();
}
/**
* **************************************************************************
*
* <p>Collection
*/
/**
* Prepare for a new collection increment. For the mark-sweep collector we must flip the state of
* the mark bit between collections.
*/
public void prepare(boolean fullHeap) {
if (fullHeap) {
if (VM.VERIFY_ASSERTIONS) {
VM.assertions._assert(treadmill.fromSpaceEmpty());
}
markState = MARK_BIT.minus(markState);
}
treadmill.flip(fullHeap);
inNurseryGC = !fullHeap;
}
/**
* A new collection increment has completed. For the mark-sweep collector this means we can
* perform the sweep phase.
*/
public void release(boolean fullHeap) {
// sweep the large objects
sweepLargePages(true); // sweep the nursery
if (VM.VERIFY_ASSERTIONS) VM.assertions._assert(treadmill.nurseryEmpty());
if (fullHeap) sweepLargePages(false); // sweep the mature space
}
/** Sweep through the large pages, releasing all superpages on the "from space" treadmill. */
private void sweepLargePages(boolean sweepNursery) {
while (true) {
Address cell = sweepNursery ? treadmill.popNursery() : treadmill.pop();
if (cell.isZero()) break;
release(getSuperPage(cell));
}
if (VM.VERIFY_ASSERTIONS)
VM.assertions._assert(sweepNursery ? treadmill.nurseryEmpty() : treadmill.fromSpaceEmpty());
}
/**
* Release a group of pages that were allocated together.
*
* @param first The first page in the group of pages that were allocated together.
*/
@Inline
public void release(Address first) {
((FreeListPageResource) pr).releasePages(first);
}
/**
* **************************************************************************
*
* <p>Object processing and tracing
*/
/**
* Trace a reference to an object under a mark sweep collection policy. If the object header is
* not already marked, mark the object in either the bitmap or by moving it off the treadmill, and
* enqueue the object for subsequent processing. The object is marked as (an atomic) side-effect
* of checking whether already marked.
*
* @param trace The trace being conducted.
* @param object The object to be traced.
* @return The object (there is no object forwarding in this collector, so we always return the
* same object: this could be a void method but for compliance to a more general interface).
*/
@Inline
public ObjectReference traceObject(TransitiveClosure trace, ObjectReference object) {
boolean nurseryObject = isInNursery(object);
if (!inNurseryGC || nurseryObject) {
if (testAndMark(object, markState)) {
internalMarkObject(object, nurseryObject);
trace.processNode(object);
}
}
return object;
}
/**
* @param object The object in question
* @return True if this object is known to be live (i.e. it is marked)
*/
@Inline
public boolean isLive(ObjectReference object) {
return testMarkBit(object, markState);
}
/**
* An object has been marked (identifiged as live). Large objects are added to the to-space
* treadmill, while all other objects will have a mark bit set in the superpage header.
*
* @param object The object which has been marked.
*/
@Inline
private void internalMarkObject(ObjectReference object, boolean nurseryObject) {
Address cell = VM.objectModel.objectStartRef(object);
Address node = Treadmill.midPayloadToNode(cell);
treadmill.copy(node, nurseryObject);
}
/**
* **************************************************************************
*
* <p>Header manipulation
*/
/**
* Perform any required initialization of the GC portion of the header.
*
* @param object the object ref to the storage to be initialized
* @param alloc is this initialization occuring due to (initial) allocation (true) or due to
* copying (false)?
*/
@Inline
public void initializeHeader(ObjectReference object, boolean alloc) {
Word oldValue = VM.objectModel.readAvailableBitsWord(object);
Word newValue = oldValue.and(LOS_BIT_MASK.not()).or(markState);
if (alloc) newValue = newValue.or(NURSERY_BIT);
if (Plan.NEEDS_LOG_BIT_IN_HEADER) newValue = newValue.or(Plan.UNLOGGED_BIT);
VM.objectModel.writeAvailableBitsWord(object, newValue);
Address cell = VM.objectModel.objectStartRef(object);
treadmill.addToTreadmill(Treadmill.midPayloadToNode(cell), alloc);
}
/**
* Atomically attempt to set the mark bit of an object. Return true if successful, false if the
* mark bit was already set.
*
* @param object The object whose mark bit is to be written
* @param value The value to which the mark bit will be set
*/
@Inline
private boolean testAndMark(ObjectReference object, Word value) {
Word oldValue, markBit;
do {
oldValue = VM.objectModel.prepareAvailableBits(object);
markBit = oldValue.and(inNurseryGC ? LOS_BIT_MASK : MARK_BIT);
if (markBit.EQ(value)) return false;
} while (!VM.objectModel.attemptAvailableBits(
object, oldValue, oldValue.and(LOS_BIT_MASK.not()).or(value)));
return true;
}
/**
* Return true if the mark bit for an object has the given value.
*
* @param object The object whose mark bit is to be tested
* @param value The value against which the mark bit will be tested
* @return True if the mark bit for the object has the given value.
*/
@Inline
private boolean testMarkBit(ObjectReference object, Word value) {
return VM.objectModel.readAvailableBitsWord(object).and(MARK_BIT).EQ(value);
}
/**
* Return true if the object is in the logical nursery
*
* @param object The object whose status is to be tested
* @return True if the object is in the logical nursery
*/
@Inline
private boolean isInNursery(ObjectReference object) {
return VM.objectModel.readAvailableBitsWord(object).and(NURSERY_BIT).EQ(NURSERY_BIT);
}
/**
* Return the size of the per-superpage header required by this system. In this case it is just
* the underlying superpage header size.
*
* @return The size of the per-superpage header required by this system.
*/
@Inline
protected int superPageHeaderSize() {
return Treadmill.headerSize();
}
/**
* Return the size of the per-cell header for cells of a given class size.
*
* @return The size of the per-cell header for cells of a given class size.
*/
@Inline
protected int cellHeaderSize() {
return 0;
}
/**
* This is the treadmill used by the large object space.
*
* <p>Note that it depends on the specific local in use whether this is being used.
*
* @return The treadmill associated with this large object space.
*/
public Treadmill getTreadmill() {
return this.treadmill;
}
}
@NoInline
@NoNullCheck
@Unpreemptible
public static void lock(Object o, Offset lockOffset) {
if (STATS) fastLocks++;
Word threadId = Word.fromIntZeroExtend(RVMThread.getCurrentThread().getLockingId());
for (int cnt = 0; ; cnt++) {
Word old = Magic.getWordAtOffset(o, lockOffset);
Word stat = old.and(TL_STAT_MASK);
boolean tryToInflate = false;
if (stat.EQ(TL_STAT_BIASABLE)) {
Word id = old.and(TL_THREAD_ID_MASK);
if (id.isZero()) {
if (ENABLE_BIASED_LOCKING) {
// lock is unbiased, bias it in our favor and grab it
if (Synchronization.tryCompareAndSwap(
o, lockOffset, old, old.or(threadId).plus(TL_LOCK_COUNT_UNIT))) {
Magic.isync();
return;
}
} else {
// lock is unbiased but biasing is NOT allowed, so turn it into
// a thin lock
if (Synchronization.tryCompareAndSwap(
o, lockOffset, old, old.or(threadId).or(TL_STAT_THIN))) {
Magic.isync();
return;
}
}
} else if (id.EQ(threadId)) {
// lock is biased in our favor
Word changed = old.plus(TL_LOCK_COUNT_UNIT);
if (!changed.and(TL_LOCK_COUNT_MASK).isZero()) {
setDedicatedU16(o, lockOffset, changed);
return;
} else {
tryToInflate = true;
}
} else {
if (casFromBiased(o, lockOffset, old, biasBitsToThinBits(old), cnt)) {
continue; // don't spin, since it's thin now
}
}
} else if (stat.EQ(TL_STAT_THIN)) {
Word id = old.and(TL_THREAD_ID_MASK);
if (id.isZero()) {
if (Synchronization.tryCompareAndSwap(o, lockOffset, old, old.or(threadId))) {
Magic.isync();
return;
}
} else if (id.EQ(threadId)) {
Word changed = old.plus(TL_LOCK_COUNT_UNIT);
if (changed.and(TL_LOCK_COUNT_MASK).isZero()) {
tryToInflate = true;
} else if (Synchronization.tryCompareAndSwap(o, lockOffset, old, changed)) {
Magic.isync();
return;
}
} else if (cnt > retryLimit) {
tryToInflate = true;
}
} else {
if (VM.VerifyAssertions) VM._assert(stat.EQ(TL_STAT_FAT));
// lock is fat. contend on it.
if (Lock.getLock(getLockIndex(old)).lockHeavy(o)) {
return;
}
}
if (tryToInflate) {
if (STATS) slowLocks++;
// the lock is not fat, is owned by someone else, or else the count wrapped.
// attempt to inflate it (this may fail, in which case we'll just harmlessly
// loop around) and lock it (may also fail, if we get the wrong lock). if it
// succeeds, we're done.
// NB: this calls into our attemptToMarkInflated() method, which will do the
// Right Thing if the lock is biased to someone else.
if (inflateAndLock(o, lockOffset)) {
return;
}
} else {
RVMThread.yield();
}
}
}
public static boolean isPageAligned(Address addr) {
Word pagesizeMask = Word.fromIntZeroExtend(getPagesize() - 1);
return addr.toWord().and(pagesizeMask).isZero();
}
public static boolean isPageMultiple(Offset val) {
Word pagesizeMask = Word.fromIntZeroExtend(getPagesize() - 1);
return val.toWord().and(pagesizeMask).isZero();
}
/**
* Low level copy of <code>copyBytes</code> bytes from <code>src[srcPos]</code> to <code>
* dst[dstPos]</code>.
*
* <p>Assumption: <code>src != dst || (srcPos >= dstPos)</code> and element size is 2 bytes.
*
* @param dstPtr The destination start address
* @param srcPtr The source start address
* @param copyBytes The number of bytes to be copied
*/
public static void aligned16Copy(Address dstPtr, Address srcPtr, int copyBytes) {
if (USE_NATIVE && copyBytes > NATIVE_THRESHOLD) {
memcopy(dstPtr, srcPtr, copyBytes);
} else {
if (copyBytes >= BYTES_IN_COPY
&& (srcPtr.toWord().and(Word.fromIntZeroExtend(BYTES_IN_COPY - 1))
== (dstPtr.toWord().and(Word.fromIntZeroExtend(BYTES_IN_COPY - 1))))) {
// relative alignment is the same
Address endPtr = srcPtr.plus(copyBytes);
Address wordEndPtr =
endPtr.toWord().and(Word.fromIntZeroExtend(BYTES_IN_COPY - 1).not()).toAddress();
if (BYTES_IN_COPY == 8) {
if (srcPtr.toWord().and(Word.fromIntZeroExtend(2)).NE(Word.zero())) {
copy2Bytes(dstPtr, srcPtr);
srcPtr = srcPtr.plus(2);
dstPtr = dstPtr.plus(2);
}
if (srcPtr.toWord().and(Word.fromIntZeroExtend(4)).NE(Word.zero())) {
copy4Bytes(dstPtr, srcPtr);
srcPtr = srcPtr.plus(4);
dstPtr = dstPtr.plus(4);
}
} else {
if (srcPtr.toWord().and(Word.fromIntZeroExtend(2)).NE(Word.zero())) {
copy2Bytes(dstPtr, srcPtr);
srcPtr = srcPtr.plus(2);
dstPtr = dstPtr.plus(2);
}
}
while (srcPtr.LT(wordEndPtr)) {
if (BYTES_IN_COPY == 8) {
copy8Bytes(dstPtr, srcPtr);
} else {
copy4Bytes(dstPtr, srcPtr);
}
srcPtr = srcPtr.plus(BYTES_IN_COPY);
dstPtr = dstPtr.plus(BYTES_IN_COPY);
}
// if(VM.VerifyAssertions) VM._assert(wordEndPtr.EQ(srcPtr));
if (BYTES_IN_COPY == 8) {
if (endPtr.toWord().and(Word.fromIntZeroExtend(4)).NE(Word.zero())) {
copy4Bytes(dstPtr, srcPtr);
srcPtr = srcPtr.plus(4);
dstPtr = dstPtr.plus(4);
}
if (endPtr.toWord().and(Word.fromIntZeroExtend(2)).NE(Word.zero())) {
copy2Bytes(dstPtr, srcPtr);
}
} else {
if (endPtr.toWord().and(Word.fromIntZeroExtend(2)).NE(Word.zero())) {
copy2Bytes(dstPtr, srcPtr);
}
}
} else {
Address endPtr = srcPtr.plus(copyBytes);
while (srcPtr.LT(endPtr)) {
copy2Bytes(dstPtr, srcPtr);
srcPtr = srcPtr.plus(2);
dstPtr = dstPtr.plus(2);
}
}
}
}