/** * Allocates more memory in order to insert an additional record. This will request additional * memory from the {@link ShuffleMemoryManager} and spill if the requested memory can not be * obtained. * * @param requiredSpace the required space in the data page, in bytes, including space for storing * the record size. */ private void allocateSpaceForRecord(int requiredSpace) throws IOException { // TODO: merge these steps to first calculate total memory requirements for this insert, // then try to acquire; no point in acquiring sort buffer only to spill due to no space in the // data page. if (!sorter.hasSpaceForAnotherRecord()) { logger.debug("Attempting to expand sort pointer array"); final long oldPointerArrayMemoryUsage = sorter.getMemoryUsage(); final long memoryToGrowPointerArray = oldPointerArrayMemoryUsage * 2; final long memoryAcquired = shuffleMemoryManager.tryToAcquire(memoryToGrowPointerArray); if (memoryAcquired < memoryToGrowPointerArray) { shuffleMemoryManager.release(memoryAcquired); spill(); } else { sorter.expandPointerArray(); shuffleMemoryManager.release(oldPointerArrayMemoryUsage); } } if (requiredSpace > freeSpaceInCurrentPage) { logger.trace( "Required space {} is less than free space in current page ({})", requiredSpace, freeSpaceInCurrentPage); // TODO: we should track metrics on the amount of space wasted when we roll over to a new page // without using the free space at the end of the current page. We should also do this for // BytesToBytesMap. if (requiredSpace > PAGE_SIZE) { throw new IOException( "Required space " + requiredSpace + " is greater than page size (" + PAGE_SIZE + ")"); } else { final long memoryAcquired = shuffleMemoryManager.tryToAcquire(PAGE_SIZE); if (memoryAcquired < PAGE_SIZE) { shuffleMemoryManager.release(memoryAcquired); spill(); final long memoryAcquiredAfterSpilling = shuffleMemoryManager.tryToAcquire(PAGE_SIZE); if (memoryAcquiredAfterSpilling != PAGE_SIZE) { shuffleMemoryManager.release(memoryAcquiredAfterSpilling); throw new IOException("Unable to acquire " + PAGE_SIZE + " bytes of memory"); } } currentPage = memoryManager.allocatePage(PAGE_SIZE); currentPagePosition = currentPage.getBaseOffset(); freeSpaceInCurrentPage = PAGE_SIZE; allocatedPages.add(currentPage); } } }
/** Allocates new sort data structures. Called when creating the sorter and after each spill. */ private void initializeForWriting() throws IOException { this.writeMetrics = new ShuffleWriteMetrics(); // TODO: move this sizing calculation logic into a static method of sorter: final long memoryRequested = initialSize * 8L * 2; final long memoryAcquired = shuffleMemoryManager.tryToAcquire(memoryRequested); if (memoryAcquired != memoryRequested) { shuffleMemoryManager.release(memoryAcquired); throw new IOException("Could not acquire " + memoryRequested + " bytes of memory"); } this.sorter = new UnsafeInMemorySorter(memoryManager, recordComparator, prefixComparator, initialSize); }