/** * Given a memory page and offset within that page, encode this address into a 64-bit long. This * address will remain valid as long as the corresponding page has not been freed. * * @param page a data page allocated by {@link TaskMemoryManager#allocate(long)}. * @param offsetInPage an offset in this page which incorporates the base offset. In other words, * this should be the value that you would pass as the base offset into an UNSAFE call (e.g. * page.baseOffset() + something). * @return an encoded page address. */ public long encodePageNumberAndOffset(MemoryBlock page, long offsetInPage) { if (!inHeap) { // In off-heap mode, an offset is an absolute address that may require a full 64 bits to // encode. Due to our page size limitation, though, we can convert this into an offset that's // relative to the page's base offset; this relative offset will fit in 51 bits. offsetInPage -= page.getBaseOffset(); } return encodePageNumberAndOffset(page.pageNumber, offsetInPage); }
/** * Clean up all allocated memory and pages. Returns the number of bytes freed. A non-zero return * value can be used to detect memory leaks. */ public long cleanUpAllAllocatedMemory() { long freedBytes = 0; for (MemoryBlock page : pageTable) { if (page != null) { freedBytes += page.size(); freePage(page); } } final Iterator<MemoryBlock> iter = allocatedNonPageMemory.iterator(); while (iter.hasNext()) { final MemoryBlock memory = iter.next(); freedBytes += memory.size(); // We don't call free() here because that calls Set.remove, which would lead to a // ConcurrentModificationException here. executorMemoryManager.free(memory); iter.remove(); } return freedBytes; }
/** Free a block of memory allocated via {@link TaskMemoryManager#allocatePage(long)}. */ public void freePage(MemoryBlock page) { assert (page.pageNumber != -1) : "Called freePage() on memory that wasn't allocated with allocatePage()"; executorMemoryManager.free(page); synchronized (this) { allocatedPages.clear(page.pageNumber); } pageTable[page.pageNumber] = null; if (logger.isTraceEnabled()) { logger.trace("Freed page number {} ({} bytes)", page.pageNumber, page.size()); } }
/** * Allocate a block of memory that will be tracked in the MemoryManager's page table; this is * intended for allocating large blocks of memory that will be shared between operators. */ public MemoryBlock allocatePage(long size) { if (size > MAXIMUM_PAGE_SIZE_BYTES) { throw new IllegalArgumentException( "Cannot allocate a page with more than " + MAXIMUM_PAGE_SIZE_BYTES + " bytes"); } final int pageNumber; synchronized (this) { pageNumber = allocatedPages.nextClearBit(0); if (pageNumber >= PAGE_TABLE_SIZE) { throw new IllegalStateException( "Have already allocated a maximum of " + PAGE_TABLE_SIZE + " pages"); } allocatedPages.set(pageNumber); } final MemoryBlock page = executorMemoryManager.allocate(size); page.pageNumber = pageNumber; pageTable[pageNumber] = page; if (logger.isTraceEnabled()) { logger.trace("Allocate page number {} ({} bytes)", pageNumber, size); } return page; }