/** * removeJobTasks: Cleanup mapTasks and reduceTasks for this job * * @param jobID */ private void removeJobTasks(int jobID) { System.out.println( communicator.getLocalHostName() + " proceeding to remove map tasks for job " + jobID); mapLock.lock(); Iterator<MapTask> itr = mapTasks.iterator(); while (itr.hasNext()) { MapTask task = itr.next(); if (task.getJobConf().getJobID() == jobID) { itr.remove(); } } mapLock.unlock(); System.out.println( communicator.getLocalHostName() + " proceeding to remove reduce tasks for job " + jobID); reduceLock.lock(); Iterator<ReduceTask> itr1 = reduceTasks.iterator(); while (itr1.hasNext()) { ReduceTask task = itr1.next(); if (task.getJobConf().getJobID() == jobID) { itr1.remove(); } } reduceLock.unlock(); }
/** * performMap: Add the mapTask to the list of current running mapTasks, Allocate a new MapRunner * and an executor service and start the mapTask * * @param mapTask */ public void performMap(MapTask mapTask) { mapLock.lock(); mapTasks.add(mapTask); mapLock.unlock(); MapRunner runner = new MapRunner(mapTask); // add the DataNode as a listener for this task runner.addListener(this); ExecutorService executor = Executors.newSingleThreadExecutor(); addExecutorToMap(mapTask.getJobConf().getJobID(), executor); executor.execute(runner); }
/** * mapTaskFailureDetected: Retry the failed map task if is has not exceeded it's maximum retry * counts. Otherwise bail out and notify master of failure * * @param task: map task that failed * @param e: Exception that caused the task to fail */ public void mapTaskFailureDetected(MapTask task, Exception e) { if (task.getRetryCount() == 1) { mapLock.lock(); mapTasks.remove(task); mapLock.unlock(); notifyMasterOfMapTaskFailure(task, e); } else { // get the retry count int count = task.getRetryCount(); // increment it count++; // set the incremented retry count task.setRetryCount(count); // try to run it again MapRunner runner = new MapRunner(task); runner.addListener(this); ExecutorService executor = Executors.newSingleThreadExecutor(); addExecutorToMap(task.getJobConf().getJobID(), executor); executor.execute(runner); } }
/** * Performs pre-processing for {@link coflowsim.simulators.Simulator#mergeTasksByRack()} using * {@link #coalesceMappers(int)}, {@link #coalesceReducers(int)}, and {@link * #calcAlphaDeadline(int, double)}. * * @param numRacks number of racks in the simulation * @param machinesPerRack number of machines in each rack * @param deadlineMult deadline duration multiplier */ @SuppressWarnings("unchecked") public void arrangeTasks(int numRacks, int machinesPerRack, double deadlineMult) { if (numMappersInRacks == null) { numMappersInRacks = new int[numRacks]; Arrays.fill(numMappersInRacks, 0); } if (tasksInRacks == null) { tasksInRacks = (Vector<ReduceTask>[]) new Vector[numRacks]; shuffleBytesPerRack = new double[numRacks]; Arrays.fill(shuffleBytesPerRack, 0.0); } for (Task t : tasks) { if (t.taskType == TaskType.MAPPER) { MapTask mt = (MapTask) t; int fromRack = convertMachineToRack(mt.getPlacement(), machinesPerRack); numMappersInRacks[fromRack]++; } if (t.taskType == TaskType.REDUCER) { ReduceTask rt = (ReduceTask) t; int toRack = convertMachineToRack(rt.getPlacement(), machinesPerRack); if (tasksInRacks[toRack] == null) { tasksInRacks[toRack] = new Vector<ReduceTask>(); } addAscending(tasksInRacks[toRack], rt); shuffleBytesPerRack[toRack] += rt.shuffleBytes; } } coalesceMappers(numRacks); coalesceReducers(numRacks); calcAlphaDeadline(numRacks, deadlineMult); }
public void run() { // TODO Auto-generated method stub run = true; System.out.println("Running a map"); while (run) { // Figure out how to read input file // Evaluate it Map<String, List<String>> input = reader.getKeyValuePairs(); if (input == null) { map.setStatus(-1); Message msg = new Message(); msg.setTask(map); msg.setType('f'); coord.conn.sendMessage(msg); return; } OutputCollector<String, String> collect = new OutputCollector<String, String>(); Set<String> keySet = input.keySet(); for (String key : keySet) { List<String> values = input.get(key); for (String value : values) { map.getJob().map(key, value, collect); } } List<Pair> results = collect.getResults(); int length = results.size(); FileOutputStream out; try { System.out.println("Trying to Write to File"); out = new FileOutputStream(new File(map.getOutput().get(0))); BufferedWriter dw = new BufferedWriter(new OutputStreamWriter(out)); for (Pair p : results) { dw.append(p.toString()); dw.newLine(); } dw.newLine(); dw.flush(); dw.close(); out.close(); coord.dataNode.addFileToDFS(map.getOutput().get(0), coord.conn.port, false); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); map.setStatus(-1); Message msg = new Message(); msg.setTask(map); msg.setType('f'); coord.conn.sendMessage(msg); return; } map.setStatus(1); Message msg = new Message(); msg.setTask(map); msg.setType('f'); coord.conn.sendMessage(msg); break; } }
public RunMap(MapTask m, SlaveCoordinator coordinator) { map = m; coord = coordinator; RandomAccessFile read; try { read = new RandomAccessFile((m.getInput()).get(0), "r"); List<RandomAccessFile> readers = new LinkedList<RandomAccessFile>(); readers.add(read); reader = new RecordReader(readers, m.recordlength, true); } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
/** * notifyMasterOfMapTaskCompletion: Get the master's communicator object and call mapTaskCompleted * on the communicator * * @param finished: the map task that completed */ private void notifyMasterOfMapTaskCompletion(MapTask finished) { Registry registry; String masterNode = communicator.getMasterHostName(); JobConfiguration jobConf = finished.getJobConf(); try { registry = LocateRegistry.getRegistry(masterNode, communicator.getREGISTRY_PORT()); CommunicatorInterface communicator = (CommunicatorInterface) registry.lookup("communicator_" + masterNode); communicator.mapTaskCompleted(finished); } catch (RemoteException e) { e.printStackTrace(); } catch (NotBoundException e) { e.printStackTrace(); } }
/** * notifyOfMapperThreadCompletion: We come in here when a mapper task is successfully completed. * At this point the mapper has generated intermediate files. In this function we run the * partition function on the keys and decide which reducers we need to send the intermediate files * to. * * <p>The sending of intermediate files as soon as a map task completes is better than sending all * at the end, especially in situations where the network bandwidth can be a bottleneck. * * @param mapRunner */ @Override public synchronized void notifyOfMapperThreadCompletion(MapRunner mapRunner) { MapTask finished = mapRunner.getMapTask(); JobConfiguration jobConf = finished.getJobConf(); final int chunkID = finished.getChunkID(); File folder = new File(jobConf.getJobDir() + "-intermediate/"); File[] intermediateFiles = folder.listFiles( new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.endsWith(Integer.toString(chunkID)); } }); /** Iterate through all the intermediate files created for a particular chunkID */ for (File file : intermediateFiles) { try { /** The intermediate filename is key-chunkID. Extract the key from the filename */ String[] arr = file.getName().split("-"); String key = ""; for (int i = 0; i < arr.length - 1; i++) { key += arr[i]; } // create new partitioner object Partitioner partitioner = new Partitioner(jobConf.getNumberOfReducers(), key); /** * run a partition function which returns a reducer to which this intermediate file should * be sent to */ int dataNodeIndex = partitioner.partition(); /** * get the list of all the data nodes. Sort them. Use the dataNodeIndex returned by the * partition function to get the actual reducer node */ communicator.acquireDataNodesLock(); List<String> allDataNodes = communicator.getAllDataNodes(); Collections.sort(allDataNodes); String reducerNode = allDataNodes.get(dataNodeIndex); /** * Get the communicator object of the reducer node, Read the intermediate file into the * memory and call receiveIntermediateFile() on the communicator of the reducer node. */ String intermediateFilePath = jobConf.getJobDir() + "-intermediate/" + key.toString() + "-" + chunkID; Registry registry; registry = LocateRegistry.getRegistry(reducerNode, communicator.getREGISTRY_PORT()); CommunicatorInterface communicator = (CommunicatorInterface) registry.lookup("communicator_" + reducerNode); communicator.receiveIntermediateFile( jobConf, Files.readAllBytes(Paths.get(intermediateFilePath)), file.getName()); /** * At the end of the task completion we send to master a list of reducer nodes to which * intermediate files were sent to. Hence store this reducer node to the list */ finished.addReducer(reducerNode); } catch (RemoteException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (NotBoundException e) { e.printStackTrace(); } catch (IndexOutOfBoundsException e) { System.out.println("This job is about to be terminated"); } communicator.releaseDataNodesLock(); } // notify the master that map task has successfully completed notifyMasterOfMapTaskCompletion(finished); // remove this task from local data structure mapLock.lock(); mapTasks.remove(finished); mapLock.unlock(); }