/**
* notifyMasterOfReduceTaskCompletion: Get the master's communicator and call reduceTaskCompleted
* on it
*
* @param finished: the reduce task that just completed
*/
private void notifyMasterOfReduceTaskCompletion(ReduceTask finished) {
Registry registry;
String masterNode = communicator.getMasterHostName();
try {
registry = LocateRegistry.getRegistry(masterNode, communicator.getREGISTRY_PORT());
CommunicatorInterface communicator =
(CommunicatorInterface) registry.lookup("communicator_" + masterNode);
communicator.reduceTaskCompleted(finished);
} catch (RemoteException e) {
e.printStackTrace();
} catch (NotBoundException e) {
e.printStackTrace();
}
}
/**
* notifyMasterOfReduceTaskFailure: Get master's communicator and call reduceTaskFailed on it with
* the failed reduceTask and the exception
*
* @param task
* @param e
*/
public void notifyMasterOfReduceTaskFailure(ReduceTask task, Exception e) {
Registry registry;
String masterNode = communicator.getMasterHostName();
try {
registry = LocateRegistry.getRegistry(masterNode, communicator.getREGISTRY_PORT());
CommunicatorInterface communicator =
(CommunicatorInterface) registry.lookup("communicator_" + masterNode);
communicator.reduceTaskFailed(task, e);
} catch (RemoteException e1) {
e1.printStackTrace();
} catch (NotBoundException e1) {
e1.printStackTrace();
}
}
/**
* notifyOfMapperThreadCompletion: We come in here when a mapper task is successfully completed.
* At this point the mapper has generated intermediate files. In this function we run the
* partition function on the keys and decide which reducers we need to send the intermediate files
* to.
*
* <p>The sending of intermediate files as soon as a map task completes is better than sending all
* at the end, especially in situations where the network bandwidth can be a bottleneck.
*
* @param mapRunner
*/
@Override
public synchronized void notifyOfMapperThreadCompletion(MapRunner mapRunner) {
MapTask finished = mapRunner.getMapTask();
JobConfiguration jobConf = finished.getJobConf();
final int chunkID = finished.getChunkID();
File folder = new File(jobConf.getJobDir() + "-intermediate/");
File[] intermediateFiles =
folder.listFiles(
new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(Integer.toString(chunkID));
}
});
/** Iterate through all the intermediate files created for a particular chunkID */
for (File file : intermediateFiles) {
try {
/** The intermediate filename is key-chunkID. Extract the key from the filename */
String[] arr = file.getName().split("-");
String key = "";
for (int i = 0; i < arr.length - 1; i++) {
key += arr[i];
}
// create new partitioner object
Partitioner partitioner = new Partitioner(jobConf.getNumberOfReducers(), key);
/**
* run a partition function which returns a reducer to which this intermediate file should
* be sent to
*/
int dataNodeIndex = partitioner.partition();
/**
* get the list of all the data nodes. Sort them. Use the dataNodeIndex returned by the
* partition function to get the actual reducer node
*/
communicator.acquireDataNodesLock();
List<String> allDataNodes = communicator.getAllDataNodes();
Collections.sort(allDataNodes);
String reducerNode = allDataNodes.get(dataNodeIndex);
/**
* Get the communicator object of the reducer node, Read the intermediate file into the
* memory and call receiveIntermediateFile() on the communicator of the reducer node.
*/
String intermediateFilePath =
jobConf.getJobDir() + "-intermediate/" + key.toString() + "-" + chunkID;
Registry registry;
registry = LocateRegistry.getRegistry(reducerNode, communicator.getREGISTRY_PORT());
CommunicatorInterface communicator =
(CommunicatorInterface) registry.lookup("communicator_" + reducerNode);
communicator.receiveIntermediateFile(
jobConf, Files.readAllBytes(Paths.get(intermediateFilePath)), file.getName());
/**
* At the end of the task completion we send to master a list of reducer nodes to which
* intermediate files were sent to. Hence store this reducer node to the list
*/
finished.addReducer(reducerNode);
} catch (RemoteException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (NotBoundException e) {
e.printStackTrace();
} catch (IndexOutOfBoundsException e) {
System.out.println("This job is about to be terminated");
}
communicator.releaseDataNodesLock();
}
// notify the master that map task has successfully completed
notifyMasterOfMapTaskCompletion(finished);
// remove this task from local data structure
mapLock.lock();
mapTasks.remove(finished);
mapLock.unlock();
}