/** Free a block of memory allocated via {@link TaskMemoryManager#allocatePage(long)}. */
public void freePage(MemoryBlock page) {
assert (page.pageNumber != -1)
: "Called freePage() on memory that wasn't allocated with allocatePage()";
executorMemoryManager.free(page);
synchronized (this) {
allocatedPages.clear(page.pageNumber);
}
pageTable[page.pageNumber] = null;
if (logger.isTraceEnabled()) {
logger.trace("Freed page number {} ({} bytes)", page.pageNumber, page.size());
}
}
/**
* Clean up all allocated memory and pages. Returns the number of bytes freed. A non-zero return
* value can be used to detect memory leaks.
*/
public long cleanUpAllAllocatedMemory() {
long freedBytes = 0;
for (MemoryBlock page : pageTable) {
if (page != null) {
freedBytes += page.size();
freePage(page);
}
}
final Iterator<MemoryBlock> iter = allocatedNonPageMemory.iterator();
while (iter.hasNext()) {
final MemoryBlock memory = iter.next();
freedBytes += memory.size();
// We don't call free() here because that calls Set.remove, which would lead to a
// ConcurrentModificationException here.
executorMemoryManager.free(memory);
iter.remove();
}
return freedBytes;
}
/**
* Allocate a block of memory that will be tracked in the MemoryManager's page table; this is
* intended for allocating large blocks of memory that will be shared between operators.
*/
public MemoryBlock allocatePage(long size) {
if (size > MAXIMUM_PAGE_SIZE_BYTES) {
throw new IllegalArgumentException(
"Cannot allocate a page with more than " + MAXIMUM_PAGE_SIZE_BYTES + " bytes");
}
final int pageNumber;
synchronized (this) {
pageNumber = allocatedPages.nextClearBit(0);
if (pageNumber >= PAGE_TABLE_SIZE) {
throw new IllegalStateException(
"Have already allocated a maximum of " + PAGE_TABLE_SIZE + " pages");
}
allocatedPages.set(pageNumber);
}
final MemoryBlock page = executorMemoryManager.allocate(size);
page.pageNumber = pageNumber;
pageTable[pageNumber] = page;
if (logger.isTraceEnabled()) {
logger.trace("Allocate page number {} ({} bytes)", pageNumber, size);
}
return page;
}
/** Free memory allocated by {@link TaskMemoryManager#allocate(long)}. */
public void free(MemoryBlock memory) {
assert (memory.pageNumber == -1) : "Should call freePage() for pages, not free()";
executorMemoryManager.free(memory);
final boolean wasAlreadyRemoved = !allocatedNonPageMemory.remove(memory);
assert (!wasAlreadyRemoved) : "Called free() on memory that was already freed!";
}
/**
* Allocates a contiguous block of memory. Note that the allocated memory is not guaranteed to be
* zeroed out (call `zero()` on the result if this is necessary). This method is intended to be
* used for allocating operators' internal data structures. For data pages that you want to
* exchange between operators, consider using {@link TaskMemoryManager#allocatePage(long)}, since
* that will enable intra-memory pointers (see {@link
* TaskMemoryManager#encodePageNumberAndOffset(MemoryBlock, long)} and this class's top-level
* Javadoc for more details).
*/
public MemoryBlock allocate(long size) throws OutOfMemoryError {
final MemoryBlock memory = executorMemoryManager.allocate(size);
allocatedNonPageMemory.add(memory);
return memory;
}