@Override
public void updateDemand() {
demand = 0;
if (isRunnable()) {
// For reduces, make sure enough maps are done that reduces can launch
if (taskType == TaskType.REDUCE && !job.scheduleReduces()) return;
// Add up demand from each TaskInProgress; each TIP can either
// - have no attempts running, in which case it demands 1 slot
// - have N attempts running, in which case it demands N slots, and may
// potentially demand one more slot if it needs to be speculated
TaskInProgress[] tips =
(taskType == TaskType.MAP ? job.getTasks(TaskType.MAP) : job.getTasks(TaskType.REDUCE));
boolean speculationEnabled =
(taskType == TaskType.MAP ? job.hasSpeculativeMaps() : job.hasSpeculativeReduces());
long time = scheduler.getClock().getTime();
for (TaskInProgress tip : tips) {
if (!tip.isComplete()) {
if (tip.isRunning()) {
// Count active tasks and any speculative task we want to launch
demand += tip.getActiveTasks().size();
if (speculationEnabled && tip.hasSpeculativeTask(time, job.getStatus().mapProgress()))
demand += 1;
} else {
// Need to launch 1 task
demand += 1;
}
}
}
}
}
private void updateTaskCounts() {
for (Map.Entry<JobInProgress, JobInfo> entry : infos.entrySet()) {
JobInProgress job = entry.getKey();
JobInfo info = entry.getValue();
if (job.getStatus().getRunState() != JobStatus.RUNNING)
continue; // Job is still in PREP state and tasks aren't initialized
// Count maps
int totalMaps = job.numMapTasks;
int finishedMaps = 0;
int runningMaps = 0;
for (TaskInProgress tip : job.getMapTasks()) {
if (tip.isComplete()) {
finishedMaps += 1;
} else if (tip.isRunning()) {
runningMaps += tip.getActiveTasks().size();
}
}
info.runningMaps = runningMaps;
info.neededMaps =
(totalMaps - runningMaps - finishedMaps + taskSelector.neededSpeculativeMaps(job));
// Count reduces
int totalReduces = job.numReduceTasks;
int finishedReduces = 0;
int runningReduces = 0;
for (TaskInProgress tip : job.getReduceTasks()) {
if (tip.isComplete()) {
finishedReduces += 1;
} else if (tip.isRunning()) {
runningReduces += tip.getActiveTasks().size();
}
}
info.runningReduces = runningReduces;
info.neededReduces =
(totalReduces
- runningReduces
- finishedReduces
+ taskSelector.neededSpeculativeReduces(job));
// If the job was marked as not runnable due to its user or pool having
// too many active jobs, set the neededMaps/neededReduces to 0. We still
// count runningMaps/runningReduces however so we can give it a deficit.
if (!info.runnable) {
info.neededMaps = 0;
info.neededReduces = 0;
}
}
}