예제 #1
0
 @Override
 public void updateDemand() {
   demand = 0;
   if (isRunnable()) {
     // For reduces, make sure enough maps are done that reduces can launch
     if (taskType == TaskType.REDUCE && !job.scheduleReduces()) return;
     // Add up demand from each TaskInProgress; each TIP can either
     // - have no attempts running, in which case it demands 1 slot
     // - have N attempts running, in which case it demands N slots, and may
     //   potentially demand one more slot if it needs to be speculated
     TaskInProgress[] tips =
         (taskType == TaskType.MAP ? job.getTasks(TaskType.MAP) : job.getTasks(TaskType.REDUCE));
     boolean speculationEnabled =
         (taskType == TaskType.MAP ? job.hasSpeculativeMaps() : job.hasSpeculativeReduces());
     long time = scheduler.getClock().getTime();
     for (TaskInProgress tip : tips) {
       if (!tip.isComplete()) {
         if (tip.isRunning()) {
           // Count active tasks and any speculative task we want to launch
           demand += tip.getActiveTasks().size();
           if (speculationEnabled && tip.hasSpeculativeTask(time, job.getStatus().mapProgress()))
             demand += 1;
         } else {
           // Need to launch 1 task
           demand += 1;
         }
       }
     }
   }
 }
예제 #2
0
 @Override
 public void start() {
   // Register servlet with JobTracker's Jetty server
   if (taskTrackerManager instanceof JobTracker) {
     JobTracker jobTracker = (JobTracker) taskTrackerManager;
     HttpServer infoServer = jobTracker.infoServer;
     infoServer.setAttribute("booking", this);
     infoServer.addServlet("booking", "/booking", BookSchedulerServlet.class);
   }
   super.start();
 }
예제 #3
0
 @Override
 public Task assignTask(TaskTrackerStatus tts, long currentTime, Collection<JobInProgress> visited)
     throws IOException {
   if (isRunnable()) {
     visited.add(job);
     TaskTrackerManager ttm = scheduler.taskTrackerManager;
     ClusterStatus clusterStatus = ttm.getClusterStatus();
     int numTaskTrackers = clusterStatus.getTaskTrackers();
     if (taskType == TaskType.MAP) {
       LocalityLevel localityLevel = scheduler.getAllowedLocalityLevel(job, currentTime);
       scheduler.getEventLog().log("ALLOWED_LOC_LEVEL", job.getJobID(), localityLevel);
       // obtainNewMapTask needs to be passed 1 + the desired locality level
       return job.obtainNewMapTask(
           tts, numTaskTrackers, ttm.getNumberOfUniqueHosts(), localityLevel.toCacheLevelCap());
     } else {
       return job.obtainNewReduceTask(tts, numTaskTrackers, ttm.getNumberOfUniqueHosts());
     }
   } else {
     return null;
   }
 }
예제 #4
0
 public void initialize()
     throws IOException, SAXException, AllocationConfigurationException,
         ParserConfigurationException {
   Configuration conf = scheduler.getConf();
   this.poolNameProperty = conf.get("mapred.fairscheduler.poolnameproperty", "user.name");
   this.allocFile = conf.get("mapred.fairscheduler.allocation.file");
   if (allocFile == null) {
     LOG.warn(
         "No mapred.fairscheduler.allocation.file given in jobconf - "
             + "the fair scheduler will not use any queues.");
   }
   reloadAllocs();
   lastSuccessfulReload = System.currentTimeMillis();
   lastReloadAttempt = System.currentTimeMillis();
   // Create the default pool so that it shows up in the web UI
   getPool(Pool.DEFAULT_POOL_NAME);
 }
예제 #5
0
 @Override
 protected void jobNoLongerRunning(JobInProgress job) {
   super.jobNoLongerRunning(job);
   finishJobStatus.put(job.getJobConf().getJobName(), job.getStatus());
 }
예제 #6
0
 private boolean isRunnable() {
   JobInfo info = scheduler.getJobInfo(job);
   int runState = job.getStatus().getRunState();
   return (info != null && info.runnable && runState == JobStatus.RUNNING);
 }
예제 #7
0
 @Override
 public double getWeight() {
   return scheduler.getJobWeight(job, taskType);
 }