/** * Low level copy of <code>len</code> elements from <code>src[srcPos]</code> to <code>dst[dstPos] * </code>. * * <p>Assumption <code>src != dst || (srcPos >= dstPos)</code> and element size is 8 bytes. * * @param src the source array * @param srcIdx index in the source array to begin copy * @param dst the destination array * @param dstIdx index in the destination array to being copy * @param len number of array elements to copy */ @Inline public static void arraycopy64Bit(Object src, int srcIdx, Object dst, int dstIdx, int len) { Address srcPtr = VM_Magic.objectAsAddress(src).plus(srcIdx << LOG_BYTES_IN_DOUBLE); Address dstPtr = VM_Magic.objectAsAddress(dst).plus(dstIdx << LOG_BYTES_IN_DOUBLE); int copyBytes = len << LOG_BYTES_IN_DOUBLE; if (USE_NATIVE && len > (NATIVE_THRESHOLD >> LOG_BYTES_IN_DOUBLE)) { memcopy(dstPtr, srcPtr, copyBytes); } else { // The elements of long[] and double[] are always doubleword aligned // therefore we can do 64 bit load/stores without worrying about alignment. Address endPtr = srcPtr.plus(copyBytes); while (srcPtr.LT(endPtr)) { // We generate abysmal code on IA32 if we try to use the FP registers, // so use the gprs instead even though it results in more instructions. if (VM.BuildForIA32) { dstPtr.store(srcPtr.loadInt()); dstPtr.store(srcPtr.loadInt(Offset.fromIntSignExtend(4)), Offset.fromIntSignExtend(4)); } else { dstPtr.store(srcPtr.loadDouble()); } srcPtr = srcPtr.plus(8); dstPtr = dstPtr.plus(8); } } }
/** * Copy numbytes from src to dst. Assumption either the ranges are non overlapping, or src >= dst * + 4. Also, src and dst are 4 byte aligned and numBytes is a multiple of 4. * * @param dst the destination addr * @param src the source addr * @param numBytes the number of bytes top copy */ @Inline public static void aligned32Copy(Address dst, Address src, Offset numBytes) { if (USE_NATIVE && numBytes.sGT(Offset.fromIntSignExtend(NATIVE_THRESHOLD))) { memcopy(dst, src, numBytes.toWord().toExtent()); } else { if (VM.BuildFor64Addr) { Word wordMask = Word.one().lsh(LOG_BYTES_IN_ADDRESS).minus(Word.one()); Word srcAlignment = src.toWord().and(wordMask); if (srcAlignment.EQ(dst.toWord().and(wordMask))) { Offset i = Offset.zero(); if (srcAlignment.EQ(Word.fromIntZeroExtend(BYTES_IN_INT))) { dst.store(src.loadInt(i), i); i = i.plus(BYTES_IN_INT); } Word endAlignment = srcAlignment.plus(numBytes).and(Word.fromIntSignExtend(BYTES_IN_ADDRESS - 1)); numBytes = numBytes.minus(endAlignment.toOffset()); for (; i.sLT(numBytes); i = i.plus(BYTES_IN_ADDRESS)) { dst.store(src.loadWord(i), i); } if (!endAlignment.isZero()) { dst.store(src.loadInt(i), i); } return; } } // normal case: 32 bit or (64 bit not aligned) for (Offset i = Offset.zero(); i.sLT(numBytes); i = i.plus(BYTES_IN_INT)) { dst.store(src.loadInt(i), i); } } }
@Inline private static void copy8Bytes(Address dstPtr, Address srcPtr) { if (BYTES_IN_COPY == 8) { if (VM.BuildForIA32) { dstPtr.store(srcPtr.loadLong()); } else { dstPtr.store(srcPtr.loadDouble()); } } else { copy4Bytes(dstPtr, srcPtr); copy4Bytes(dstPtr.plus(4), srcPtr.plus(4)); } }
/** * Store a value into a reference field of an object. * * @param object The object to store the field of. * @param index The field index. * @param value The value to store. */ public void storeReferenceField(ObjectReference object, int index, ObjectReference value) { int limit = ObjectModel.getRefs(object); if (Trace.isEnabled(Item.STORE) || ObjectModel.isWatched(object)) { Trace.printf( Item.STORE, "[%s].object[%d/%d] = %s", ObjectModel.getString(object), index, limit, value.toString()); } check(!object.isNull(), "Object can not be null"); check(index >= 0, "Index must be non-negative"); check(index < limit, "Index " + index + " out of bounds " + limit); Address referenceSlot = ObjectModel.getRefSlot(object, index); if (ActivePlan.constraints.needsWriteBarrier()) { context.writeBarrier(object, referenceSlot, value, null, null, Plan.AASTORE_WRITE_BARRIER); if (gcEveryWB) { gc(); } } else { referenceSlot.store(value); } }
/** * Copy <code>numbytes</code> from <code>src</code> to <code>dst</code>. Assumption either the * ranges are non overlapping, or <code>src >= dst + BYTES_IN_ADDRESS</code>. * * @param dst The destination addr * @param src The source addr * @param numBytes The number of bytes to copy */ private static void internalAlignedWordCopy(Address dst, Address src, int numBytes) { Address end = src.plus(numBytes); while (src.LT(end)) { dst.store(src.loadWord()); src = src.plus(BYTES_IN_ADDRESS); dst = dst.plus(BYTES_IN_ADDRESS); } }
/** * Set a region of memory * * @param start The start of the region to be zeroed (must be 4-byte aligned) * @param bytes The number of bytes to be zeroed (must be 4-byte aligned) * @param value The value to which the integers in the region should be set */ public static void set(Address start, int bytes, int value) throws InlinePragma { if (Assert.VERIFY_ASSERTIONS) { assertAligned(start); assertAligned(bytes); } Address end = start.add(bytes); for (Address addr = start; addr.LT(end); addr = addr.add(BYTES_IN_INT)) addr.store(value); }
/** * Zero a small region of memory * * @param start The start of the region to be zeroed (must be 4-byte aligned) * @param bytes The number of bytes to be zeroed (must be 4-byte aligned) */ public static void zeroSmall(Address start, Extent bytes) throws InlinePragma { if (Assert.VERIFY_ASSERTIONS) { assertAligned(start); assertAligned(bytes); } Address end = start.add(bytes); for (Address addr = start; addr.LT(end); addr = addr.add(BYTES_IN_INT)) addr.store(0); }
/** * Given a slot (ie the address of an ObjectReference), ensure that the referent will not move for * the rest of the GC. This is achieved by calling the precopyObject method. * * @param slot The slot to check */ @Inline public final void processPrecopyEdge(Address slot) { ObjectReference child = slot.loadObjectReference(); if (!child.isNull()) { child = precopyObject(child); slot.store(child); } }
/** * Store a value into the data field of an object. * * @param object The object to store the field of. * @param index The field index. * @param value The value to store. */ public void storeDataField(ObjectReference object, int index, int value) { int limit = ObjectModel.getDataCount(object); check(!object.isNull(), "Object can not be null"); check(index >= 0, "Index must be non-negative"); check(index < limit, "Index " + index + " out of bounds " + limit); Address ref = ObjectModel.getDataSlot(object, index); ref.store(value); Trace.trace(Item.STORE, "%s.[%d] = %d", object.toString(), index, value); }
/** * Trace a reference during GC. This involves determining which collection policy applies and * calling the appropriate <code>trace</code> method. * * @param target The object the interior edge points within. * @param slot The location of the interior edge. * @param root True if this is a root edge. */ public final void processInteriorEdge(ObjectReference target, Address slot, boolean root) { Address interiorRef = slot.loadAddress(); Offset offset = interiorRef.diff(target.toAddress()); ObjectReference newTarget = traceObject(target, root); if (VM.VERIFY_ASSERTIONS) { if (offset.sLT(Offset.zero()) || offset.sGT(Offset.fromIntSignExtend(1 << 24))) { // There is probably no object this large Log.writeln("ERROR: Suspiciously large delta to interior pointer"); Log.write(" object base = "); Log.writeln(target); Log.write(" interior reference = "); Log.writeln(interiorRef); Log.write(" delta = "); Log.writeln(offset); VM.assertions._assert(false); } } slot.store(newTarget.toAddress().plus(offset)); }
/** * Low level copy of <code>len</code> elements from <code>src[srcPos]</code> to <code>dst[dstPos] * </code>. * * <p>Assumption: <code>src != dst || (srcPos >= dstPos)</code> and element size is 4 bytes. * * @param src the source array * @param srcIdx index in the source array to begin copy * @param dst the destination array * @param dstIdx index in the destination array to being copy * @param len number of array elements to copy */ @Inline public static void arraycopy32Bit(Object src, int srcIdx, Object dst, int dstIdx, int len) { Address srcPtr = VM_Magic.objectAsAddress(src).plus(srcIdx << LOG_BYTES_IN_INT); Address dstPtr = VM_Magic.objectAsAddress(dst).plus(dstIdx << LOG_BYTES_IN_INT); int copyBytes = len << LOG_BYTES_IN_INT; if (USE_NATIVE && len > (NATIVE_THRESHOLD >> LOG_BYTES_IN_INT)) { memcopy(dstPtr, srcPtr, copyBytes); } else { // The elements of int[] and float[] are always 32 bit aligned // therefore we can do 32 bit load/stores without worrying about alignment. // TODO: do measurements to determine if on PPC it is a good idea to check // for compatible doubleword alignment and handle that case via the FPRs in 64 bit // chunks. // Unclear if this will be a big enough win to justify checking because for big copies // we are going into memcopy anyways and that will be faster than anything we do here. Address endPtr = srcPtr.plus(copyBytes); while (srcPtr.LT(endPtr)) { dstPtr.store(srcPtr.loadInt()); srcPtr = srcPtr.plus(4); dstPtr = dstPtr.plus(4); } } }
/** * Low level copy of len elements from src[srcPos] to dst[dstPos]. * * <p>Assumption src != dst || (srcPos >= dstPos + 2). * * @param src the source array * @param srcPos index in the source array to begin copy * @param dst the destination array * @param dstPos index in the destination array to being copy * @param len number of array elements to copy */ @Inline public static void arraycopy16Bit(Object src, int srcPos, Object dst, int dstPos, int len) { if (USE_NATIVE && len > (NATIVE_THRESHOLD >> LOG_BYTES_IN_SHORT)) { memcopy( VM_Magic.objectAsAddress(dst).plus(dstPos << LOG_BYTES_IN_SHORT), VM_Magic.objectAsAddress(src).plus(srcPos << LOG_BYTES_IN_SHORT), len << LOG_BYTES_IN_SHORT); } else { if (len >= (BYTES_IN_ADDRESS >>> LOG_BYTES_IN_SHORT) && (srcPos & ((BYTES_IN_ADDRESS - 1) >>> LOG_BYTES_IN_SHORT)) == (dstPos & ((BYTES_IN_ADDRESS - 1) >>> LOG_BYTES_IN_SHORT))) { // relative alignment is the same int byteStart = srcPos << LOG_BYTES_IN_SHORT; int wordStart = alignUp(byteStart, BYTES_IN_ADDRESS); int wordEnd = alignDown(byteStart + (len << LOG_BYTES_IN_SHORT), BYTES_IN_ADDRESS); int byteEnd = byteStart + (len << LOG_BYTES_IN_SHORT); int startDiff = wordStart - byteStart; int endDiff = byteEnd - wordEnd; int wordLen = wordEnd - wordStart; Address srcPtr = VM_Magic.objectAsAddress(src).plus((srcPos << LOG_BYTES_IN_SHORT) + startDiff); Address dstPtr = VM_Magic.objectAsAddress(dst).plus((dstPos << LOG_BYTES_IN_SHORT) + startDiff); if (VM.BuildFor64Addr) { switch (startDiff) { case 6: dstPtr.store( srcPtr.loadChar(Offset.fromIntSignExtend(-6)), Offset.fromIntSignExtend(-6)); case 4: dstPtr.store( srcPtr.loadChar(Offset.fromIntSignExtend(-4)), Offset.fromIntSignExtend(-4)); case 2: dstPtr.store( srcPtr.loadChar(Offset.fromIntSignExtend(-2)), Offset.fromIntSignExtend(-2)); } } else { if (startDiff == 2) { dstPtr.store( srcPtr.loadChar(Offset.fromIntSignExtend(-2)), Offset.fromIntSignExtend(-2)); } } Address endPtr = srcPtr.plus(wordLen); while (srcPtr.LT(endPtr)) { dstPtr.store(srcPtr.loadWord()); srcPtr = srcPtr.plus(BYTES_IN_ADDRESS); dstPtr = dstPtr.plus(BYTES_IN_ADDRESS); } if (VM.BuildFor64Addr) { switch (endDiff) { case 6: dstPtr.store( srcPtr.loadChar(Offset.fromIntSignExtend(4)), Offset.fromIntSignExtend(4)); case 4: dstPtr.store( srcPtr.loadChar(Offset.fromIntSignExtend(2)), Offset.fromIntSignExtend(2)); case 2: dstPtr.store(srcPtr.loadChar()); } } else { if (endDiff == 2) { dstPtr.store(srcPtr.loadChar()); } } } else { Address srcPtr = VM_Magic.objectAsAddress(src).plus(srcPos << LOG_BYTES_IN_CHAR); Address dstPtr = VM_Magic.objectAsAddress(dst).plus(dstPos << LOG_BYTES_IN_CHAR); Address endPtr = srcPtr.plus(len << LOG_BYTES_IN_CHAR); while (srcPtr.LT(endPtr)) { dstPtr.store(srcPtr.loadChar()); srcPtr = srcPtr.plus(2); dstPtr = dstPtr.plus(2); } } } }
/** * Store an object reference * * @param slot The location of the reference * @param value The value to store */ @Inline public void storeObjectReference(Address slot, ObjectReference value) { slot.store(value); }
/** * Low level copy of <code>copyBytes</code> bytes from <code>src[srcPos]</code> to <code> * dst[dstPos]</code>. * * <p>Assumption <code>src != dst || (srcPos >= dstPos)</code> and element size is 4 bytes. * * @param dstPtr The destination start address * @param srcPtr The source start address * @param copyBytes The number of bytes to be copied */ public static void aligned8Copy(Address dstPtr, Address srcPtr, int copyBytes) { if (USE_NATIVE && copyBytes > NATIVE_THRESHOLD) { memcopy(dstPtr, srcPtr, copyBytes); } else { if (copyBytes >= BYTES_IN_COPY && (srcPtr.toWord().and(Word.fromIntZeroExtend(BYTES_IN_COPY - 1)) == (dstPtr.toWord().and(Word.fromIntZeroExtend(BYTES_IN_COPY - 1))))) { // relative alignment is the same Address endPtr = srcPtr.plus(copyBytes); Address wordEndPtr = endPtr.toWord().and(Word.fromIntZeroExtend(BYTES_IN_COPY - 1).not()).toAddress(); if (BYTES_IN_COPY == 8) { if (srcPtr.toWord().and(Word.fromIntZeroExtend(1)).NE(Word.zero())) { copy1Bytes(dstPtr, srcPtr); srcPtr = srcPtr.plus(1); dstPtr = dstPtr.plus(1); } if (srcPtr.toWord().and(Word.fromIntZeroExtend(2)).NE(Word.zero())) { copy2Bytes(dstPtr, srcPtr); srcPtr = srcPtr.plus(2); dstPtr = dstPtr.plus(2); } if (srcPtr.toWord().and(Word.fromIntZeroExtend(4)).NE(Word.zero())) { copy4Bytes(dstPtr, srcPtr); srcPtr = srcPtr.plus(4); dstPtr = dstPtr.plus(4); } } else { if (srcPtr.toWord().and(Word.fromIntZeroExtend(1)).NE(Word.zero())) { copy1Bytes(dstPtr, srcPtr); srcPtr = srcPtr.plus(1); dstPtr = dstPtr.plus(1); } if (srcPtr.toWord().and(Word.fromIntZeroExtend(2)).NE(Word.zero())) { copy2Bytes(dstPtr, srcPtr); srcPtr = srcPtr.plus(2); dstPtr = dstPtr.plus(2); } } while (srcPtr.LT(wordEndPtr)) { if (BYTES_IN_COPY == 8) { copy8Bytes(dstPtr, srcPtr); } else { copy4Bytes(dstPtr, srcPtr); } srcPtr = srcPtr.plus(BYTES_IN_COPY); dstPtr = dstPtr.plus(BYTES_IN_COPY); } // if(VM.VerifyAssertions) VM._assert(wordEndPtr.EQ(srcPtr)); if (BYTES_IN_COPY == 8) { if (endPtr.toWord().and(Word.fromIntZeroExtend(4)).NE(Word.zero())) { copy4Bytes(dstPtr, srcPtr); srcPtr = srcPtr.plus(4); dstPtr = dstPtr.plus(4); } if (endPtr.toWord().and(Word.fromIntZeroExtend(2)).NE(Word.zero())) { copy2Bytes(dstPtr, srcPtr); srcPtr = srcPtr.plus(2); dstPtr = dstPtr.plus(2); } if (endPtr.toWord().and(Word.fromIntZeroExtend(1)).NE(Word.zero())) { copy1Bytes(dstPtr, srcPtr); } } else { if (endPtr.toWord().and(Word.fromIntZeroExtend(2)).NE(Word.zero())) { copy2Bytes(dstPtr, srcPtr); srcPtr = srcPtr.plus(2); dstPtr = dstPtr.plus(2); } if (endPtr.toWord().and(Word.fromIntZeroExtend(1)).NE(Word.zero())) { copy1Bytes(dstPtr, srcPtr); } } } else { Address endPtr = srcPtr.plus(copyBytes); while (srcPtr.LT(endPtr)) { dstPtr.store(srcPtr.loadByte()); srcPtr = srcPtr.plus(1); dstPtr = dstPtr.plus(1); } } } }
/** * Trace a reference during GC. This involves determining which collection policy applies and * calling the appropriate <code>trace</code> method. * * @param source The source of the reference. * @param slot The location containing the object reference to be traced. The object reference is * <i>NOT</i> an interior pointer. * @param root True if <code>objLoc</code> is within a root. */ @Inline public final void processEdge(ObjectReference source, Address slot) { ObjectReference object = slot.loadObjectReference(); ObjectReference newObject = traceObject(object, false); slot.store(newObject); }
@Inline private static void copy1Bytes(Address dstPtr, Address srcPtr) { dstPtr.store(srcPtr.loadByte()); }
/** * Low level copy of len elements from src[srcPos] to dst[dstPos]. * * <p>Assumptions: <code> src != dst || (scrPos >= dstPos + 4) </code> and src and dst are 8Bit * arrays. * * @param src the source array * @param srcPos index in the source array to begin copy * @param dst the destination array * @param dstPos index in the destination array to being copy * @param len number of array elements to copy */ @Inline public static void arraycopy8Bit(Object src, int srcPos, Object dst, int dstPos, int len) { if (USE_NATIVE && len > NATIVE_THRESHOLD) { memcopy( VM_Magic.objectAsAddress(dst).plus(dstPos), VM_Magic.objectAsAddress(src).plus(srcPos), len); } else { if (len >= BYTES_IN_ADDRESS && (srcPos & (BYTES_IN_ADDRESS - 1)) == (dstPos & (BYTES_IN_ADDRESS - 1))) { // relative alignment is the same int byteStart = srcPos; int wordStart = alignUp(srcPos, BYTES_IN_ADDRESS); int wordEnd = alignDown(srcPos + len, BYTES_IN_ADDRESS); int byteEnd = srcPos + len; int startDiff = wordStart - byteStart; int endDiff = byteEnd - wordEnd; int wordLen = wordEnd - wordStart; Address srcPtr = VM_Magic.objectAsAddress(src).plus(srcPos + startDiff); Address dstPtr = VM_Magic.objectAsAddress(dst).plus(dstPos + startDiff); if (VM.BuildFor64Addr) { switch (startDiff) { case 7: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-7)), Offset.fromIntSignExtend(-7)); case 6: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-6)), Offset.fromIntSignExtend(-6)); case 5: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-5)), Offset.fromIntSignExtend(-5)); case 4: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-4)), Offset.fromIntSignExtend(-4)); case 3: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-3)), Offset.fromIntSignExtend(-3)); case 2: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-2)), Offset.fromIntSignExtend(-2)); case 1: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-1)), Offset.fromIntSignExtend(-1)); } } else { switch (startDiff) { case 3: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-3)), Offset.fromIntSignExtend(-3)); case 2: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-2)), Offset.fromIntSignExtend(-2)); case 1: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(-1)), Offset.fromIntSignExtend(-1)); } } Address endPtr = srcPtr.plus(wordLen); while (srcPtr.LT(endPtr)) { dstPtr.store(srcPtr.loadWord()); srcPtr = srcPtr.plus(BYTES_IN_ADDRESS); dstPtr = dstPtr.plus(BYTES_IN_ADDRESS); } if (VM.BuildFor64Addr) { switch (endDiff) { case 7: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(6)), Offset.fromIntSignExtend(6)); case 6: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(5)), Offset.fromIntSignExtend(5)); case 5: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(4)), Offset.fromIntSignExtend(4)); case 4: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(3)), Offset.fromIntSignExtend(3)); case 3: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(2)), Offset.fromIntSignExtend(2)); case 2: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(1)), Offset.fromIntSignExtend(1)); case 1: dstPtr.store(srcPtr.loadByte()); } } else { switch (endDiff) { case 3: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(2)), Offset.fromIntSignExtend(2)); case 2: dstPtr.store( srcPtr.loadByte(Offset.fromIntSignExtend(1)), Offset.fromIntSignExtend(1)); case 1: dstPtr.store(srcPtr.loadByte()); } } } else { Address srcPtr = VM_Magic.objectAsAddress(src).plus(srcPos); Address dstPtr = VM_Magic.objectAsAddress(dst).plus(dstPos); Address endPtr = srcPtr.plus(len); while (srcPtr.LT(endPtr)) { dstPtr.store(srcPtr.loadByte()); srcPtr = srcPtr.plus(1); dstPtr = dstPtr.plus(1); } } } }
/** * Trace a reference during GC. This involves determining which collection policy applies and * calling the appropriate <code>trace</code> method. * * @param slot The location containing the object reference to be traced. The object reference is * <i>NOT</i> an interior pointer. * @param root True if <code>objLoc</code> is within a root. */ @Inline public final void processRootEdge(Address slot) { ObjectReference object = slot.loadObjectReference(); ObjectReference newObject = traceObject(object, true); slot.store(newObject); }