Пример #1
0
  /** Sort and spill the current records in response to memory pressure. */
  @VisibleForTesting
  public void spill() throws IOException {
    logger.info(
        "Thread {} spilling sort data of {} to disk ({} {} so far)",
        Thread.currentThread().getId(),
        Utils.bytesToString(getMemoryUsage()),
        spillWriters.size(),
        spillWriters.size() > 1 ? " times" : " time");

    final UnsafeSorterSpillWriter spillWriter =
        new UnsafeSorterSpillWriter(
            blockManager, fileBufferSizeBytes, writeMetrics, sorter.numRecords());
    spillWriters.add(spillWriter);
    final UnsafeSorterIterator sortedRecords = sorter.getSortedIterator();
    while (sortedRecords.hasNext()) {
      sortedRecords.loadNext();
      final Object baseObject = sortedRecords.getBaseObject();
      final long baseOffset = sortedRecords.getBaseOffset();
      final int recordLength = sortedRecords.getRecordLength();
      spillWriter.write(baseObject, baseOffset, recordLength, sortedRecords.getKeyPrefix());
    }
    spillWriter.close();
    final long sorterMemoryUsage = sorter.getMemoryUsage();
    sorter = null;
    shuffleMemoryManager.release(sorterMemoryUsage);
    final long spillSize = freeMemory();
    taskContext.taskMetrics().incMemoryBytesSpilled(spillSize);
    initializeForWriting();
  }
Пример #2
0
 public UnsafeSorterIterator getSortedIterator() throws IOException {
   final UnsafeSorterIterator inMemoryIterator = sorter.getSortedIterator();
   int numIteratorsToMerge = spillWriters.size() + (inMemoryIterator.hasNext() ? 1 : 0);
   if (spillWriters.isEmpty()) {
     return inMemoryIterator;
   } else {
     final UnsafeSorterSpillMerger spillMerger =
         new UnsafeSorterSpillMerger(recordComparator, prefixComparator, numIteratorsToMerge);
     for (UnsafeSorterSpillWriter spillWriter : spillWriters) {
       spillMerger.addSpill(spillWriter.getReader(blockManager));
     }
     spillWriters.clear();
     if (inMemoryIterator.hasNext()) {
       spillMerger.addSpill(inMemoryIterator);
     }
     return spillMerger.getSortedIterator();
   }
 }