@Override public void updateDemand() { demand = 0; if (isRunnable()) { // For reduces, make sure enough maps are done that reduces can launch if (taskType == TaskType.REDUCE && !job.scheduleReduces()) return; // Add up demand from each TaskInProgress; each TIP can either // - have no attempts running, in which case it demands 1 slot // - have N attempts running, in which case it demands N slots, and may // potentially demand one more slot if it needs to be speculated TaskInProgress[] tips = (taskType == TaskType.MAP ? job.getTasks(TaskType.MAP) : job.getTasks(TaskType.REDUCE)); boolean speculationEnabled = (taskType == TaskType.MAP ? job.hasSpeculativeMaps() : job.hasSpeculativeReduces()); long time = scheduler.getClock().getTime(); for (TaskInProgress tip : tips) { if (!tip.isComplete()) { if (tip.isRunning()) { // Count active tasks and any speculative task we want to launch demand += tip.getActiveTasks().size(); if (speculationEnabled && tip.hasSpeculativeTask(time, job.getStatus().mapProgress())) demand += 1; } else { // Need to launch 1 task demand += 1; } } } } }
private void updateTaskCounts() { for (Map.Entry<JobInProgress, JobInfo> entry : infos.entrySet()) { JobInProgress job = entry.getKey(); JobInfo info = entry.getValue(); if (job.getStatus().getRunState() != JobStatus.RUNNING) continue; // Job is still in PREP state and tasks aren't initialized // Count maps int totalMaps = job.numMapTasks; int finishedMaps = 0; int runningMaps = 0; for (TaskInProgress tip : job.getMapTasks()) { if (tip.isComplete()) { finishedMaps += 1; } else if (tip.isRunning()) { runningMaps += tip.getActiveTasks().size(); } } info.runningMaps = runningMaps; info.neededMaps = (totalMaps - runningMaps - finishedMaps + taskSelector.neededSpeculativeMaps(job)); // Count reduces int totalReduces = job.numReduceTasks; int finishedReduces = 0; int runningReduces = 0; for (TaskInProgress tip : job.getReduceTasks()) { if (tip.isComplete()) { finishedReduces += 1; } else if (tip.isRunning()) { runningReduces += tip.getActiveTasks().size(); } } info.runningReduces = runningReduces; info.neededReduces = (totalReduces - runningReduces - finishedReduces + taskSelector.neededSpeculativeReduces(job)); // If the job was marked as not runnable due to its user or pool having // too many active jobs, set the neededMaps/neededReduces to 0. We still // count runningMaps/runningReduces however so we can give it a deficit. if (!info.runnable) { info.neededMaps = 0; info.neededReduces = 0; } } }