protected ChannelFuture sendMapOutput(
ChannelHandlerContext ctx, Channel ch, String jobId, String mapId, int reduce)
throws IOException {
LocalDirAllocator lDirAlloc = attributes.getLocalDirAllocator();
ShuffleServerMetrics shuffleMetrics = attributes.getShuffleServerMetrics();
TaskTracker tracker = attributes.getTaskTracker();
boolean found = true;
Path indexFileName = null;
Path mapOutputFileName = null;
try {
// Index file
indexFileName =
lDirAlloc.getLocalPathToRead(
TaskTracker.getIntermediateOutputDir(jobId, mapId) + "/file.out.index",
attributes.getJobConf());
// Map-output file
mapOutputFileName =
lDirAlloc.getLocalPathToRead(
TaskTracker.getIntermediateOutputDir(jobId, mapId) + "/file.out",
attributes.getJobConf());
} catch (DiskChecker.DiskErrorException e) {
LOG.error(
"sendMapOutput: Failed to retrieve index or map output "
+ "file, will send ShuffleHeader noting the file can't be found.",
e);
found = false;
}
/**
* Read the index file to get the information about where the map-output for the given reducer
* is available.
*/
IndexRecord info = null;
try {
info = tracker.getIndexInformation(mapId, reduce, indexFileName);
} catch (IOException e) {
LOG.error(
"sendMapOutput: Failed to get the index information, "
+ "will send ShuffleHeader noting that the file can't be found.",
e);
found = false;
info = new IndexRecord(-1, 0, 0);
}
RandomAccessFile spill = null;
if (mapOutputFileName != null) {
File spillfile = new File(mapOutputFileName.toString());
try {
spill = new RandomAccessFile(spillfile, "r");
} catch (FileNotFoundException e) {
LOG.error(
"sendMapOutput: "
+ spillfile
+ " not found, "
+ "will send ShuffleHeader noting that the file can't be found.",
e);
found = false;
}
}
final ShuffleHeader header =
new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce, found);
final DataOutputBuffer dob = new DataOutputBuffer();
header.write(dob);
ChannelFuture writeFuture = ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
// Exit early if we didn't find the spill file.
if (found == false || spill == null) {
attributes
.getTaskTracker()
.mapOutputLost(
TaskAttemptID.forName(mapId),
"sendMapOutput: Couldn't get mapId = " + mapId + ", reduce " + reduce);
return writeFuture;
}
final FileRegion partition =
new DefaultFileRegion(spill.getChannel(), info.startOffset, info.partLength);
writeFuture = ch.write(partition);
writeFuture.addListener(new ChanneFutureListenerMetrics(partition));
shuffleMetrics.outputBytes(info.partLength); // optimistic
LOG.info(
"Sending out "
+ info.partLength
+ " bytes for reduce: "
+ reduce
+ " from map: "
+ mapId
+ " given "
+ info.partLength
+ "/"
+ info.rawLength);
return writeFuture;
}
