@Test(timeout = 10000)
 public void testReservationAccounting() throws IOException {
   Configuration conf = new TezConfiguration(defaultConf);
   FileSystem localFs = FileSystem.getLocal(conf);
   InputContext inputContext = createMockInputContext(UUID.randomUUID().toString());
   MergeManager mergeManager =
       new MergeManager(
           conf,
           localFs,
           null,
           inputContext,
           null,
           null,
           null,
           null,
           mock(ExceptionReporter.class),
           2000000,
           null,
           false,
           -1);
   mergeManager.configureAndStart();
   assertEquals(0, mergeManager.getUsedMemory());
   assertEquals(0, mergeManager.getCommitMemory());
   MapOutput mapOutput = mergeManager.reserve(null, 1, 1, 0);
   assertEquals(1, mergeManager.getUsedMemory());
   assertEquals(0, mergeManager.getCommitMemory());
   mapOutput.abort();
   assertEquals(0, mergeManager.getUsedMemory());
   assertEquals(0, mergeManager.getCommitMemory());
   mapOutput = mergeManager.reserve(null, 2, 2, 0);
   mergeManager.closeInMemoryFile(mapOutput);
   assertEquals(2, mergeManager.getUsedMemory());
   assertEquals(2, mergeManager.getCommitMemory());
   mergeManager.releaseCommittedMemory(2);
   assertEquals(0, mergeManager.getUsedMemory());
   assertEquals(0, mergeManager.getCommitMemory());
 }
  private InputAttemptIdentifier[] copyMapOutput(
      MapHost host, DataInputStream input, Set<InputAttemptIdentifier> remaining) {
    MapOutput mapOutput = null;
    InputAttemptIdentifier srcAttemptId = null;
    long decompressedLength = -1;
    long compressedLength = -1;

    try {
      long startTime = System.currentTimeMillis();
      int forReduce = -1;
      // Read the shuffle header
      try {
        ShuffleHeader header = new ShuffleHeader();
        header.readFields(input);
        srcAttemptId = scheduler.getIdentifierForFetchedOutput(header.mapId, header.forReduce);
        compressedLength = header.compressedLength;
        decompressedLength = header.uncompressedLength;
        forReduce = header.forReduce;
      } catch (IllegalArgumentException e) {
        badIdErrs.increment(1);
        LOG.warn("Invalid map id ", e);
        // Don't know which one was bad, so consider all of them as bad
        return remaining.toArray(new InputAttemptIdentifier[remaining.size()]);
      }

      // Do some basic sanity verification
      if (!verifySanity(compressedLength, decompressedLength, forReduce, remaining, srcAttemptId)) {
        return new InputAttemptIdentifier[] {srcAttemptId};
      }

      if (LOG.isDebugEnabled()) {
        LOG.debug(
            "header: "
                + srcAttemptId
                + ", len: "
                + compressedLength
                + ", decomp len: "
                + decompressedLength);
      }

      // Get the location for the map output - either in-memory or on-disk
      mapOutput = merger.reserve(srcAttemptId, decompressedLength, id);

      // Check if we can shuffle *now* ...
      if (mapOutput.getType() == Type.WAIT) {
        LOG.info("fetcher#" + id + " - MergerManager returned Status.WAIT ...");
        // Not an error but wait to process data.
        return EMPTY_ATTEMPT_ID_ARRAY;
      }

      // Go!
      LOG.info(
          "fetcher#"
              + id
              + " about to shuffle output of map "
              + mapOutput.getAttemptIdentifier()
              + " decomp: "
              + decompressedLength
              + " len: "
              + compressedLength
              + " to "
              + mapOutput.getType());
      if (mapOutput.getType() == Type.MEMORY) {
        shuffleToMemory(host, mapOutput, input, (int) decompressedLength, (int) compressedLength);
      } else {
        shuffleToDisk(host, mapOutput, input, compressedLength);
      }

      // Inform the shuffle scheduler
      long endTime = System.currentTimeMillis();
      scheduler.copySucceeded(srcAttemptId, host, compressedLength, endTime - startTime, mapOutput);
      // Note successful shuffle
      remaining.remove(srcAttemptId);
      metrics.successFetch();
      return null;
    } catch (IOException ioe) {
      ioErrs.increment(1);
      if (srcAttemptId == null || mapOutput == null) {
        LOG.info(
            "fetcher#"
                + id
                + " failed to read map header"
                + srcAttemptId
                + " decomp: "
                + decompressedLength
                + ", "
                + compressedLength,
            ioe);
        if (srcAttemptId == null) {
          return remaining.toArray(new InputAttemptIdentifier[remaining.size()]);
        } else {
          return new InputAttemptIdentifier[] {srcAttemptId};
        }
      }

      LOG.warn("Failed to shuffle output of " + srcAttemptId + " from " + host.getHostName(), ioe);

      // Inform the shuffle-scheduler
      mapOutput.abort();
      metrics.failedFetch();
      return new InputAttemptIdentifier[] {srcAttemptId};
    }
  }
  @Test(timeout = 60000l)
  public void testIntermediateMemoryMerge() throws Throwable {
    Configuration conf = new TezConfiguration(defaultConf);
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false);
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, IntWritable.class.getName());
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, IntWritable.class.getName());
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, true);
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 3);

    Path localDir = new Path(workDir, "local");
    Path srcDir = new Path(workDir, "srcData");
    localFs.mkdirs(localDir);
    localFs.mkdirs(srcDir);

    conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDir.toString());

    FileSystem localFs = FileSystem.getLocal(conf);
    LocalDirAllocator localDirAllocator =
        new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS);
    InputContext inputContext = createMockInputContext(UUID.randomUUID().toString());

    ExceptionReporter exceptionReporter = mock(ExceptionReporter.class);

    MergeManager mergeManager =
        new MergeManager(
            conf,
            localFs,
            localDirAllocator,
            inputContext,
            null,
            null,
            null,
            null,
            exceptionReporter,
            2000000,
            null,
            false,
            -1);
    mergeManager.configureAndStart();

    assertEquals(0, mergeManager.getUsedMemory());
    assertEquals(0, mergeManager.getCommitMemory());

    /**
     * Test #1 - Have 4 segments where all of them can fit into memory. - After 3 segment commits,
     * it would trigger mem-to-mem merge. - All of them can be merged in memory.
     */
    byte[] data1 = generateDataBySize(conf, 10);
    byte[] data2 = generateDataBySize(conf, 20);
    byte[] data3 = generateDataBySize(conf, 200);
    byte[] data4 = generateDataBySize(conf, 20000);

    MapOutput mo1 =
        mergeManager.reserve(new InputAttemptIdentifier(0, 0), data1.length, data1.length, 0);
    MapOutput mo2 =
        mergeManager.reserve(new InputAttemptIdentifier(1, 0), data2.length, data2.length, 0);
    MapOutput mo3 =
        mergeManager.reserve(new InputAttemptIdentifier(2, 0), data3.length, data3.length, 0);
    MapOutput mo4 =
        mergeManager.reserve(new InputAttemptIdentifier(3, 0), data4.length, data4.length, 0);

    assertEquals(MapOutput.Type.MEMORY, mo1.getType());
    assertEquals(MapOutput.Type.MEMORY, mo2.getType());
    assertEquals(MapOutput.Type.MEMORY, mo3.getType());
    assertEquals(MapOutput.Type.MEMORY, mo4.getType());
    assertEquals(0, mergeManager.getCommitMemory());

    // size should be ~20230.
    assertEquals(
        data1.length + data2.length + data3.length + data4.length, mergeManager.getUsedMemory());

    System.arraycopy(data1, 0, mo1.getMemory(), 0, data1.length);
    System.arraycopy(data2, 0, mo2.getMemory(), 0, data2.length);
    System.arraycopy(data3, 0, mo3.getMemory(), 0, data3.length);
    System.arraycopy(data4, 0, mo4.getMemory(), 0, data4.length);

    // Committing 3 segments should trigger mem-to-mem merge
    mo1.commit();
    mo2.commit();
    mo3.commit();
    mo4.commit();

    // Wait for mem-to-mem to complete
    mergeManager.waitForMemToMemMerge();

    assertEquals(1, mergeManager.inMemoryMergedMapOutputs.size());
    assertEquals(1, mergeManager.inMemoryMapOutputs.size());

    mergeManager.close(true);

    /**
     * Test #2 - Have 4 segments where all of them can fit into memory, but one of them would be big
     * enough that it can not be fit in memory during mem-to-mem merging.
     *
     * <p>- After 3 segment commits, it would trigger mem-to-mem merge. - Smaller segments which can
     * be fit in additional memory allocated gets merged.
     */
    mergeManager =
        new MergeManager(
            conf,
            localFs,
            localDirAllocator,
            inputContext,
            null,
            null,
            null,
            null,
            exceptionReporter,
            2000000,
            null,
            false,
            -1);
    mergeManager.configureAndStart();

    // Single shuffle limit is 25% of 2000000
    data1 = generateDataBySize(conf, 10);
    data2 = generateDataBySize(conf, 400000);
    data3 = generateDataBySize(conf, 400000);
    data4 = generateDataBySize(conf, 400000);

    mo1 = mergeManager.reserve(new InputAttemptIdentifier(0, 0), data1.length, data1.length, 0);
    mo2 = mergeManager.reserve(new InputAttemptIdentifier(1, 0), data2.length, data2.length, 0);
    mo3 = mergeManager.reserve(new InputAttemptIdentifier(2, 0), data3.length, data3.length, 0);
    mo4 = mergeManager.reserve(new InputAttemptIdentifier(3, 0), data4.length, data4.length, 0);

    assertEquals(MapOutput.Type.MEMORY, mo1.getType());
    assertEquals(MapOutput.Type.MEMORY, mo2.getType());
    assertEquals(MapOutput.Type.MEMORY, mo3.getType());
    assertEquals(MapOutput.Type.MEMORY, mo4.getType());
    assertEquals(0, mergeManager.getCommitMemory());

    assertEquals(
        data1.length + data2.length + data3.length + data4.length, mergeManager.getUsedMemory());

    System.arraycopy(data1, 0, mo1.getMemory(), 0, data1.length);
    System.arraycopy(data2, 0, mo2.getMemory(), 0, data2.length);
    System.arraycopy(data3, 0, mo3.getMemory(), 0, data3.length);
    System.arraycopy(data4, 0, mo4.getMemory(), 0, data4.length);

    // Committing 3 segments should trigger mem-to-mem merge
    mo1.commit();
    mo2.commit();
    mo3.commit();
    mo4.commit();

    // Wait for mem-to-mem to complete
    mergeManager.waitForMemToMemMerge();

    /**
     * Already all segments are in memory which is around 120000. It would not be able to allocate
     * more than 800000 for mem-to-mem. So it would pick up only 2 small segments which can be
     * accomodated within 800000.
     */
    assertEquals(1, mergeManager.inMemoryMergedMapOutputs.size());
    assertEquals(2, mergeManager.inMemoryMapOutputs.size());

    mergeManager.close(true);

    /**
     * Test #3 - Set number of segments for merging to 4. - Have 4 in-memory segments of size 400000
     * each - Committing 4 segments would trigger mem-to-mem - But none of them can be merged as
     * there is no enough head room for merging in memory.
     */
    mergeManager =
        new MergeManager(
            conf,
            localFs,
            localDirAllocator,
            inputContext,
            null,
            null,
            null,
            null,
            exceptionReporter,
            2000000,
            null,
            false,
            -1);
    mergeManager.configureAndStart();

    // Single shuffle limit is 25% of 2000000
    data1 = generateDataBySize(conf, 400000);
    data2 = generateDataBySize(conf, 400000);
    data3 = generateDataBySize(conf, 400000);
    data4 = generateDataBySize(conf, 400000);

    mo1 = mergeManager.reserve(new InputAttemptIdentifier(0, 0), data1.length, data1.length, 0);
    mo2 = mergeManager.reserve(new InputAttemptIdentifier(1, 0), data2.length, data2.length, 0);
    mo3 = mergeManager.reserve(new InputAttemptIdentifier(2, 0), data3.length, data3.length, 0);
    mo4 = mergeManager.reserve(new InputAttemptIdentifier(3, 0), data4.length, data4.length, 0);

    assertEquals(MapOutput.Type.MEMORY, mo1.getType());
    assertEquals(MapOutput.Type.MEMORY, mo2.getType());
    assertEquals(MapOutput.Type.MEMORY, mo3.getType());
    assertEquals(MapOutput.Type.MEMORY, mo4.getType());
    assertEquals(0, mergeManager.getCommitMemory());

    assertEquals(
        data1.length + data2.length + data3.length + data4.length, mergeManager.getUsedMemory());

    System.arraycopy(data1, 0, mo1.getMemory(), 0, data1.length);
    System.arraycopy(data2, 0, mo2.getMemory(), 0, data2.length);
    System.arraycopy(data3, 0, mo3.getMemory(), 0, data3.length);
    System.arraycopy(data4, 0, mo4.getMemory(), 0, data4.length);

    // Committing 3 segments should trigger mem-to-mem merge
    mo1.commit();
    mo2.commit();
    mo3.commit();
    mo4.commit();

    // Wait for mem-to-mem to complete
    mergeManager.waitForMemToMemMerge();

    // None of them can be merged as new mem needed for mem-to-mem can't
    // accomodate any segements
    assertEquals(0, mergeManager.inMemoryMergedMapOutputs.size());
    assertEquals(4, mergeManager.inMemoryMapOutputs.size());

    mergeManager.close(true);

    /**
     * Test #4 - Set number of segments for merging to 4. - Have 4 in-memory segments of size
     * {490000,490000,490000,230000} - Committing 4 segments would trigger mem-to-mem - But only
     * 300000 can fit into memory. This should not be merged as there is no point in merging single
     * segment. It should be added back to the inMemorySegments
     */
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 4);
    mergeManager =
        new MergeManager(
            conf,
            localFs,
            localDirAllocator,
            inputContext,
            null,
            null,
            null,
            null,
            exceptionReporter,
            2000000,
            null,
            false,
            -1);
    mergeManager.configureAndStart();

    // Single shuffle limit is 25% of 2000000
    data1 = generateDataBySize(conf, 490000);
    data2 = generateDataBySize(conf, 490000);
    data3 = generateDataBySize(conf, 490000);
    data4 = generateDataBySize(conf, 230000);

    mo1 = mergeManager.reserve(new InputAttemptIdentifier(0, 0), data1.length, data1.length, 0);
    mo2 = mergeManager.reserve(new InputAttemptIdentifier(1, 0), data2.length, data2.length, 0);
    mo3 = mergeManager.reserve(new InputAttemptIdentifier(2, 0), data3.length, data3.length, 0);
    mo4 = mergeManager.reserve(new InputAttemptIdentifier(3, 0), data4.length, data4.length, 0);

    assertTrue(mergeManager.getUsedMemory() >= (490000 + 490000 + 490000 + 23000));

    assertEquals(MapOutput.Type.MEMORY, mo1.getType());
    assertEquals(MapOutput.Type.MEMORY, mo2.getType());
    assertEquals(MapOutput.Type.MEMORY, mo3.getType());
    assertEquals(MapOutput.Type.MEMORY, mo4.getType());
    assertEquals(0, mergeManager.getCommitMemory());

    assertEquals(
        data1.length + data2.length + data3.length + data4.length, mergeManager.getUsedMemory());

    System.arraycopy(data1, 0, mo1.getMemory(), 0, data1.length);
    System.arraycopy(data2, 0, mo2.getMemory(), 0, data2.length);
    System.arraycopy(data3, 0, mo3.getMemory(), 0, data3.length);
    System.arraycopy(data4, 0, mo4.getMemory(), 0, data4.length);

    // Committing 4 segments should trigger mem-to-mem merge
    mo1.commit();
    mo2.commit();
    mo3.commit();
    mo4.commit();

    // 4 segments were there originally in inMemoryMapOutput.
    int numberOfMapOutputs = 4;

    // Wait for mem-to-mem to complete. Since only 1 segment (230000) can fit
    // into memory, it should return early
    mergeManager.waitForMemToMemMerge();

    // Check if inMemorySegment has got the MapOutput back for merging later
    assertEquals(numberOfMapOutputs, mergeManager.inMemoryMapOutputs.size());

    mergeManager.close(true);

    /**
     * Test #5 - Same to #4, but calling mergeManager.close(false) and confirm that final merge
     * doesn't occur.
     */
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 4);
    mergeManager =
        new MergeManager(
            conf,
            localFs,
            localDirAllocator,
            inputContext,
            null,
            null,
            null,
            null,
            exceptionReporter,
            2000000,
            null,
            false,
            -1);
    mergeManager.configureAndStart();

    // Single shuffle limit is 25% of 2000000
    data1 = generateDataBySize(conf, 490000);
    data2 = generateDataBySize(conf, 490000);
    data3 = generateDataBySize(conf, 490000);
    data4 = generateDataBySize(conf, 230000);

    mo1 = mergeManager.reserve(new InputAttemptIdentifier(0, 0), data1.length, data1.length, 0);
    mo2 = mergeManager.reserve(new InputAttemptIdentifier(1, 0), data2.length, data2.length, 0);
    mo3 = mergeManager.reserve(new InputAttemptIdentifier(2, 0), data3.length, data3.length, 0);
    mo4 = mergeManager.reserve(new InputAttemptIdentifier(3, 0), data4.length, data4.length, 0);

    assertTrue(mergeManager.getUsedMemory() >= (490000 + 490000 + 490000 + 23000));

    assertEquals(MapOutput.Type.MEMORY, mo1.getType());
    assertEquals(MapOutput.Type.MEMORY, mo2.getType());
    assertEquals(MapOutput.Type.MEMORY, mo3.getType());
    assertEquals(MapOutput.Type.MEMORY, mo4.getType());
    assertEquals(0, mergeManager.getCommitMemory());

    assertEquals(
        data1.length + data2.length + data3.length + data4.length, mergeManager.getUsedMemory());

    System.arraycopy(data1, 0, mo1.getMemory(), 0, data1.length);
    System.arraycopy(data2, 0, mo2.getMemory(), 0, data2.length);
    System.arraycopy(data3, 0, mo3.getMemory(), 0, data3.length);
    System.arraycopy(data4, 0, mo4.getMemory(), 0, data4.length);

    // Committing 4 segments should trigger mem-to-mem merge
    mo1.commit();
    mo2.commit();
    mo3.commit();
    mo4.commit();

    // 4 segments were there originally in inMemoryMapOutput.
    numberOfMapOutputs = 4;

    // Wait for mem-to-mem to complete. Since only 1 segment (230000) can fit
    // into memory, it should return early
    mergeManager.waitForMemToMemMerge();

    // Check if inMemorySegment has got the MapOutput back for merging later
    assertEquals(numberOfMapOutputs, mergeManager.inMemoryMapOutputs.size());

    Assert.assertNull(mergeManager.close(false));
    Assert.assertFalse(mergeManager.isMergeComplete());
  }
  @Test(timeout = 20000)
  public void testIntermediateMemoryMergeAccounting() throws Exception {
    Configuration conf = new TezConfiguration(defaultConf);
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false);
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, IntWritable.class.getName());
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, IntWritable.class.getName());
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, true);
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 2);

    Path localDir = new Path(workDir, "local");
    Path srcDir = new Path(workDir, "srcData");
    localFs.mkdirs(localDir);
    localFs.mkdirs(srcDir);

    conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDir.toString());

    FileSystem localFs = FileSystem.getLocal(conf);
    LocalDirAllocator localDirAllocator =
        new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS);
    InputContext inputContext = createMockInputContext(UUID.randomUUID().toString());

    ExceptionReporter exceptionReporter = mock(ExceptionReporter.class);

    MergeManager mergeManager =
        new MergeManager(
            conf,
            localFs,
            localDirAllocator,
            inputContext,
            null,
            null,
            null,
            null,
            exceptionReporter,
            2000000,
            null,
            false,
            -1);
    mergeManager.configureAndStart();

    assertEquals(0, mergeManager.getUsedMemory());
    assertEquals(0, mergeManager.getCommitMemory());

    byte[] data1 = generateData(conf, 10);
    byte[] data2 = generateData(conf, 20);
    MapOutput firstMapOutput = mergeManager.reserve(null, data1.length, data1.length, 0);
    MapOutput secondMapOutput = mergeManager.reserve(null, data2.length, data2.length, 0);
    assertEquals(MapOutput.Type.MEMORY, firstMapOutput.getType());
    assertEquals(MapOutput.Type.MEMORY, secondMapOutput.getType());
    assertEquals(0, mergeManager.getCommitMemory());
    assertEquals(data1.length + data2.length, mergeManager.getUsedMemory());

    System.arraycopy(data1, 0, firstMapOutput.getMemory(), 0, data1.length);
    System.arraycopy(data2, 0, secondMapOutput.getMemory(), 0, data2.length);

    secondMapOutput.commit();
    assertEquals(data2.length, mergeManager.getCommitMemory());
    assertEquals(data1.length + data2.length, mergeManager.getUsedMemory());
    firstMapOutput.commit();

    mergeManager.waitForMemToMemMerge();
    assertEquals(data1.length + data2.length, mergeManager.getCommitMemory());
    assertEquals(data1.length + data2.length, mergeManager.getUsedMemory());
  }
Beispiel #5
0
  private TaskAttemptID[] copyMapOutput(
      MapHost host, DataInputStream input, Set<TaskAttemptID> remaining) {
    MapOutput<K, V> mapOutput = null;
    TaskAttemptID mapId = null;
    long decompressedLength = -1;
    long compressedLength = -1;

    try {
      long startTime = System.currentTimeMillis();
      int forReduce = -1;
      // Read the shuffle header
      try {
        ShuffleHeader header = new ShuffleHeader();
        header.readFields(input);
        mapId = TaskAttemptID.forName(header.mapId);
        compressedLength = header.compressedLength;
        decompressedLength = header.uncompressedLength;
        forReduce = header.forReduce;
      } catch (IllegalArgumentException e) {
        badIdErrs.increment(1);
        LOG.warn("Invalid map id ", e);
        // Don't know which one was bad, so consider all of them as bad
        return remaining.toArray(new TaskAttemptID[remaining.size()]);
      }

      InputStream is = input;
      is = CryptoUtils.wrapIfNecessary(jobConf, is, compressedLength);
      compressedLength -= CryptoUtils.cryptoPadding(jobConf);
      decompressedLength -= CryptoUtils.cryptoPadding(jobConf);

      // Do some basic sanity verification
      if (!verifySanity(compressedLength, decompressedLength, forReduce, remaining, mapId)) {
        return new TaskAttemptID[] {mapId};
      }

      if (LOG.isDebugEnabled()) {
        LOG.debug(
            "header: "
                + mapId
                + ", len: "
                + compressedLength
                + ", decomp len: "
                + decompressedLength);
      }

      // Get the location for the map output - either in-memory or on-disk
      try {
        mapOutput = merger.reserve(mapId, decompressedLength, id);
      } catch (IOException ioe) {
        // kill this reduce attempt
        ioErrs.increment(1);
        scheduler.reportLocalError(ioe);
        return EMPTY_ATTEMPT_ID_ARRAY;
      }

      // Check if we can shuffle *now* ...
      if (mapOutput == null) {
        LOG.info("fetcher#" + id + " - MergeManager returned status WAIT ...");
        // Not an error but wait to process data.
        return EMPTY_ATTEMPT_ID_ARRAY;
      }

      // The codec for lz0,lz4,snappy,bz2,etc. throw java.lang.InternalError
      // on decompression failures. Catching and re-throwing as IOException
      // to allow fetch failure logic to be processed
      try {
        // Go!
        LOG.info(
            "fetcher#"
                + id
                + " about to shuffle output of map "
                + mapOutput.getMapId()
                + " decomp: "
                + decompressedLength
                + " len: "
                + compressedLength
                + " to "
                + mapOutput.getDescription());
        mapOutput.shuffle(host, is, compressedLength, decompressedLength, metrics, reporter);
      } catch (java.lang.InternalError e) {
        LOG.warn("Failed to shuffle for fetcher#" + id, e);
        throw new IOException(e);
      }

      // Inform the shuffle scheduler
      long endTime = System.currentTimeMillis();
      scheduler.copySucceeded(mapId, host, compressedLength, endTime - startTime, mapOutput);
      // Note successful shuffle
      remaining.remove(mapId);
      metrics.successFetch();
      return null;
    } catch (IOException ioe) {
      ioErrs.increment(1);
      if (mapId == null || mapOutput == null) {
        LOG.info(
            "fetcher#"
                + id
                + " failed to read map header"
                + mapId
                + " decomp: "
                + decompressedLength
                + ", "
                + compressedLength,
            ioe);
        if (mapId == null) {
          return remaining.toArray(new TaskAttemptID[remaining.size()]);
        } else {
          return new TaskAttemptID[] {mapId};
        }
      }

      LOG.warn("Failed to shuffle output of " + mapId + " from " + host.getHostName(), ioe);

      // Inform the shuffle-scheduler
      mapOutput.abort();
      metrics.failedFetch();
      return new TaskAttemptID[] {mapId};
    }
  }