Exemple #1
0
 /**
  * Given a memory page and offset within that page, encode this address into a 64-bit long. This
  * address will remain valid as long as the corresponding page has not been freed.
  *
  * @param page a data page allocated by {@link TaskMemoryManager#allocate(long)}.
  * @param offsetInPage an offset in this page which incorporates the base offset. In other words,
  *     this should be the value that you would pass as the base offset into an UNSAFE call (e.g.
  *     page.baseOffset() + something).
  * @return an encoded page address.
  */
 public long encodePageNumberAndOffset(MemoryBlock page, long offsetInPage) {
   if (!inHeap) {
     // In off-heap mode, an offset is an absolute address that may require a full 64 bits to
     // encode. Due to our page size limitation, though, we can convert this into an offset that's
     // relative to the page's base offset; this relative offset will fit in 51 bits.
     offsetInPage -= page.getBaseOffset();
   }
   return encodePageNumberAndOffset(page.pageNumber, offsetInPage);
 }
Exemple #2
0
 /**
  * Clean up all allocated memory and pages. Returns the number of bytes freed. A non-zero return
  * value can be used to detect memory leaks.
  */
 public long cleanUpAllAllocatedMemory() {
   long freedBytes = 0;
   for (MemoryBlock page : pageTable) {
     if (page != null) {
       freedBytes += page.size();
       freePage(page);
     }
   }
   final Iterator<MemoryBlock> iter = allocatedNonPageMemory.iterator();
   while (iter.hasNext()) {
     final MemoryBlock memory = iter.next();
     freedBytes += memory.size();
     // We don't call free() here because that calls Set.remove, which would lead to a
     // ConcurrentModificationException here.
     executorMemoryManager.free(memory);
     iter.remove();
   }
   return freedBytes;
 }
Exemple #3
0
 /** Free a block of memory allocated via {@link TaskMemoryManager#allocatePage(long)}. */
 public void freePage(MemoryBlock page) {
   assert (page.pageNumber != -1)
       : "Called freePage() on memory that wasn't allocated with allocatePage()";
   executorMemoryManager.free(page);
   synchronized (this) {
     allocatedPages.clear(page.pageNumber);
   }
   pageTable[page.pageNumber] = null;
   if (logger.isTraceEnabled()) {
     logger.trace("Freed page number {} ({} bytes)", page.pageNumber, page.size());
   }
 }
Exemple #4
0
  /**
   * Allocate a block of memory that will be tracked in the MemoryManager's page table; this is
   * intended for allocating large blocks of memory that will be shared between operators.
   */
  public MemoryBlock allocatePage(long size) {
    if (size > MAXIMUM_PAGE_SIZE_BYTES) {
      throw new IllegalArgumentException(
          "Cannot allocate a page with more than " + MAXIMUM_PAGE_SIZE_BYTES + " bytes");
    }

    final int pageNumber;
    synchronized (this) {
      pageNumber = allocatedPages.nextClearBit(0);
      if (pageNumber >= PAGE_TABLE_SIZE) {
        throw new IllegalStateException(
            "Have already allocated a maximum of " + PAGE_TABLE_SIZE + " pages");
      }
      allocatedPages.set(pageNumber);
    }
    final MemoryBlock page = executorMemoryManager.allocate(size);
    page.pageNumber = pageNumber;
    pageTable[pageNumber] = page;
    if (logger.isTraceEnabled()) {
      logger.trace("Allocate page number {} ({} bytes)", pageNumber, size);
    }
    return page;
  }