@Override public Option<MapStatus> stop(boolean success) { try { // Update task metrics from accumulators (null in UnsafeShuffleWriterSuite) Map<String, Accumulator<Object>> internalAccumulators = taskContext.internalMetricsToAccumulators(); if (internalAccumulators != null) { internalAccumulators .apply(InternalAccumulator.PEAK_EXECUTION_MEMORY()) .add(getPeakMemoryUsedBytes()); } if (stopping) { return Option.apply(null); } else { stopping = true; if (success) { if (mapStatus == null) { throw new IllegalStateException("Cannot call stop(true) without having called write()"); } return Option.apply(mapStatus); } else { // The map task failed, so delete our output data. shuffleBlockResolver.removeDataByMap(shuffleId, mapId); return Option.apply(null); } } } finally { if (sorter != null) { // If sorter is non-null, then this implies that we called stop() in response to an error, // so we need to clean up memory and spill files created by the sorter sorter.cleanupResources(); } } }
@Override public void write(scala.collection.Iterator<Product2<K, V>> records) throws IOException { // Keep track of success so we know if we encountered an exception // We do this rather than a standard try/catch/re-throw to handle // generic throwables. boolean success = false; try { while (records.hasNext()) { insertRecordIntoSorter(records.next()); } closeAndWriteOutput(); success = true; } finally { if (sorter != null) { try { sorter.cleanupResources(); } catch (Exception e) { // Only throw this error if we won't be masking another // error. if (success) { throw e; } else { logger.error( "In addition to a failure during writing, we failed during " + "cleanup.", e); } } } } }