private void shuffleToDisk(
MapHost host, MapOutput mapOutput, InputStream input, long compressedLength)
throws IOException {
// Copy data to local-disk
OutputStream output = mapOutput.getDisk();
long bytesLeft = compressedLength;
try {
final int BYTES_TO_READ = 64 * 1024;
byte[] buf = new byte[BYTES_TO_READ];
while (bytesLeft > 0) {
int n = input.read(buf, 0, (int) Math.min(bytesLeft, BYTES_TO_READ));
if (n < 0) {
throw new IOException(
"read past end of stream reading " + mapOutput.getAttemptIdentifier());
}
output.write(buf, 0, n);
bytesLeft -= n;
metrics.inputBytes(n);
}
LOG.info(
"Read "
+ (compressedLength - bytesLeft)
+ " bytes from map-output for "
+ mapOutput.getAttemptIdentifier());
output.close();
} catch (IOException ioe) {
// Close the streams
IOUtils.cleanup(LOG, input, output);
// Re-throw
throw ioe;
}
// Sanity check
if (bytesLeft != 0) {
throw new IOException(
"Incomplete map output received for "
+ mapOutput.getAttemptIdentifier()
+ " from "
+ host.getHostName()
+ " ("
+ bytesLeft
+ " bytes missing of "
+ compressedLength
+ ")");
}
}
private void shuffleToMemory(
MapHost host,
MapOutput mapOutput,
InputStream input,
int decompressedLength,
int compressedLength)
throws IOException {
IFileInputStream checksumIn = new IFileInputStream(input, compressedLength, job);
input = checksumIn;
// Are map-outputs compressed?
if (codec != null) {
decompressor.reset();
input = codec.createInputStream(input, decompressor);
}
// Copy map-output into an in-memory buffer
byte[] shuffleData = mapOutput.getMemory();
try {
IOUtils.readFully(input, shuffleData, 0, shuffleData.length);
metrics.inputBytes(shuffleData.length);
LOG.info(
"Read "
+ shuffleData.length
+ " bytes from map-output for "
+ mapOutput.getAttemptIdentifier());
} catch (IOException ioe) {
// Close the streams
IOUtils.cleanup(LOG, input);
// Re-throw
throw ioe;
}
}