예제 #1
0
 @Override
 public void updateDemand() {
   demand = 0;
   if (isRunnable()) {
     // For reduces, make sure enough maps are done that reduces can launch
     if (taskType == TaskType.REDUCE && !job.scheduleReduces()) return;
     // Add up demand from each TaskInProgress; each TIP can either
     // - have no attempts running, in which case it demands 1 slot
     // - have N attempts running, in which case it demands N slots, and may
     //   potentially demand one more slot if it needs to be speculated
     TaskInProgress[] tips =
         (taskType == TaskType.MAP ? job.getTasks(TaskType.MAP) : job.getTasks(TaskType.REDUCE));
     boolean speculationEnabled =
         (taskType == TaskType.MAP ? job.hasSpeculativeMaps() : job.hasSpeculativeReduces());
     long time = scheduler.getClock().getTime();
     for (TaskInProgress tip : tips) {
       if (!tip.isComplete()) {
         if (tip.isRunning()) {
           // Count active tasks and any speculative task we want to launch
           demand += tip.getActiveTasks().size();
           if (speculationEnabled && tip.hasSpeculativeTask(time, job.getStatus().mapProgress()))
             demand += 1;
         } else {
           // Need to launch 1 task
           demand += 1;
         }
       }
     }
   }
 }
예제 #2
0
 private void updateTaskCounts() {
   for (Map.Entry<JobInProgress, JobInfo> entry : infos.entrySet()) {
     JobInProgress job = entry.getKey();
     JobInfo info = entry.getValue();
     if (job.getStatus().getRunState() != JobStatus.RUNNING)
       continue; // Job is still in PREP state and tasks aren't initialized
     // Count maps
     int totalMaps = job.numMapTasks;
     int finishedMaps = 0;
     int runningMaps = 0;
     for (TaskInProgress tip : job.getMapTasks()) {
       if (tip.isComplete()) {
         finishedMaps += 1;
       } else if (tip.isRunning()) {
         runningMaps += tip.getActiveTasks().size();
       }
     }
     info.runningMaps = runningMaps;
     info.neededMaps =
         (totalMaps - runningMaps - finishedMaps + taskSelector.neededSpeculativeMaps(job));
     // Count reduces
     int totalReduces = job.numReduceTasks;
     int finishedReduces = 0;
     int runningReduces = 0;
     for (TaskInProgress tip : job.getReduceTasks()) {
       if (tip.isComplete()) {
         finishedReduces += 1;
       } else if (tip.isRunning()) {
         runningReduces += tip.getActiveTasks().size();
       }
     }
     info.runningReduces = runningReduces;
     info.neededReduces =
         (totalReduces
             - runningReduces
             - finishedReduces
             + taskSelector.neededSpeculativeReduces(job));
     // If the job was marked as not runnable due to its user or pool having
     // too many active jobs, set the neededMaps/neededReduces to 0. We still
     // count runningMaps/runningReduces however so we can give it a deficit.
     if (!info.runnable) {
       info.neededMaps = 0;
       info.neededReduces = 0;
     }
   }
 }