Example #1
0
 /** Free a block of memory allocated via {@link TaskMemoryManager#allocatePage(long)}. */
 public void freePage(MemoryBlock page) {
   assert (page.pageNumber != -1)
       : "Called freePage() on memory that wasn't allocated with allocatePage()";
   executorMemoryManager.free(page);
   synchronized (this) {
     allocatedPages.clear(page.pageNumber);
   }
   pageTable[page.pageNumber] = null;
   if (logger.isTraceEnabled()) {
     logger.trace("Freed page number {} ({} bytes)", page.pageNumber, page.size());
   }
 }
Example #2
0
 /**
  * Clean up all allocated memory and pages. Returns the number of bytes freed. A non-zero return
  * value can be used to detect memory leaks.
  */
 public long cleanUpAllAllocatedMemory() {
   long freedBytes = 0;
   for (MemoryBlock page : pageTable) {
     if (page != null) {
       freedBytes += page.size();
       freePage(page);
     }
   }
   final Iterator<MemoryBlock> iter = allocatedNonPageMemory.iterator();
   while (iter.hasNext()) {
     final MemoryBlock memory = iter.next();
     freedBytes += memory.size();
     // We don't call free() here because that calls Set.remove, which would lead to a
     // ConcurrentModificationException here.
     executorMemoryManager.free(memory);
     iter.remove();
   }
   return freedBytes;
 }
Example #3
0
  /**
   * Allocate a block of memory that will be tracked in the MemoryManager's page table; this is
   * intended for allocating large blocks of memory that will be shared between operators.
   */
  public MemoryBlock allocatePage(long size) {
    if (size > MAXIMUM_PAGE_SIZE_BYTES) {
      throw new IllegalArgumentException(
          "Cannot allocate a page with more than " + MAXIMUM_PAGE_SIZE_BYTES + " bytes");
    }

    final int pageNumber;
    synchronized (this) {
      pageNumber = allocatedPages.nextClearBit(0);
      if (pageNumber >= PAGE_TABLE_SIZE) {
        throw new IllegalStateException(
            "Have already allocated a maximum of " + PAGE_TABLE_SIZE + " pages");
      }
      allocatedPages.set(pageNumber);
    }
    final MemoryBlock page = executorMemoryManager.allocate(size);
    page.pageNumber = pageNumber;
    pageTable[pageNumber] = page;
    if (logger.isTraceEnabled()) {
      logger.trace("Allocate page number {} ({} bytes)", pageNumber, size);
    }
    return page;
  }
Example #4
0
 /** Free memory allocated by {@link TaskMemoryManager#allocate(long)}. */
 public void free(MemoryBlock memory) {
   assert (memory.pageNumber == -1) : "Should call freePage() for pages, not free()";
   executorMemoryManager.free(memory);
   final boolean wasAlreadyRemoved = !allocatedNonPageMemory.remove(memory);
   assert (!wasAlreadyRemoved) : "Called free() on memory that was already freed!";
 }
Example #5
0
 /**
  * Allocates a contiguous block of memory. Note that the allocated memory is not guaranteed to be
  * zeroed out (call `zero()` on the result if this is necessary). This method is intended to be
  * used for allocating operators' internal data structures. For data pages that you want to
  * exchange between operators, consider using {@link TaskMemoryManager#allocatePage(long)}, since
  * that will enable intra-memory pointers (see {@link
  * TaskMemoryManager#encodePageNumberAndOffset(MemoryBlock, long)} and this class's top-level
  * Javadoc for more details).
  */
 public MemoryBlock allocate(long size) throws OutOfMemoryError {
   final MemoryBlock memory = executorMemoryManager.allocate(size);
   allocatedNonPageMemory.add(memory);
   return memory;
 }