/**
* Update the last recorded status for the given task tracker. It assumes that the taskTrackers
* are locked on entry.
*
* @author Owen O'Malley
* @param trackerName The name of the tracker
* @param status The new status for the task tracker
* @return Was an old status found?
*/
private boolean updateTaskTrackerStatus(String trackerName, TaskTrackerStatus status) {
TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.get(trackerName);
if (oldStatus != null) {
totalMaps -= oldStatus.countMapTasks();
totalReduces -= oldStatus.countReduceTasks();
if (status == null) {
taskTrackers.remove(trackerName);
}
}
if (status != null) {
totalMaps += status.countMapTasks();
totalReduces += status.countReduceTasks();
taskTrackers.put(trackerName, status);
}
return oldStatus != null;
}
@Override
public synchronized List<Task> assignTasks(TaskTracker taskTracker) throws IOException {
TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
final int numTaskTrackers = clusterStatus.getTaskTrackers();
final int clusterMapCapacity = clusterStatus.getMaxMapTasks();
final int clusterReduceCapacity = clusterStatus.getMaxReduceTasks();
Collection<JobInProgress> jobQueue = jobQueueJobInProgressListener.getJobQueue();
//
// Get map + reduce counts for the current tracker.
//
final int trackerMapCapacity = taskTrackerStatus.getMaxMapSlots();
final int trackerReduceCapacity = taskTrackerStatus.getMaxReduceSlots();
final int trackerRunningMaps = taskTrackerStatus.countMapTasks();
final int trackerRunningReduces = taskTrackerStatus.countReduceTasks();
// Assigned tasks
List<Task> assignedTasks = new ArrayList<Task>();
//
// Compute (running + pending) map and reduce task numbers across pool
//
int remainingReduceLoad = 0;
int remainingMapLoad = 0;
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() == JobStatus.RUNNING) {
remainingMapLoad += (job.desiredMaps() - job.finishedMaps());
if (job.scheduleReduces()) {
remainingReduceLoad += (job.desiredReduces() - job.finishedReduces());
}
}
}
}
// Compute the 'load factor' for maps and reduces
double mapLoadFactor = 0.0;
if (clusterMapCapacity > 0) {
mapLoadFactor = (double) remainingMapLoad / clusterMapCapacity;
}
double reduceLoadFactor = 0.0;
if (clusterReduceCapacity > 0) {
reduceLoadFactor = (double) remainingReduceLoad / clusterReduceCapacity;
}
//
// In the below steps, we allocate first map tasks (if appropriate),
// and then reduce tasks if appropriate. We go through all jobs
// in order of job arrival; jobs only get serviced if their
// predecessors are serviced, too.
//
//
// We assign tasks to the current taskTracker if the given machine
// has a workload that's less than the maximum load of that kind of
// task.
// However, if the cluster is close to getting loaded i.e. we don't
// have enough _padding_ for speculative executions etc., we only
// schedule the "highest priority" task i.e. the task from the job
// with the highest priority.
//
final int trackerCurrentMapCapacity =
Math.min((int) Math.ceil(mapLoadFactor * trackerMapCapacity), trackerMapCapacity);
int availableMapSlots = trackerCurrentMapCapacity - trackerRunningMaps;
boolean exceededMapPadding = false;
if (availableMapSlots > 0) {
exceededMapPadding = exceededPadding(true, clusterStatus, trackerMapCapacity);
}
int numLocalMaps = 0;
int numNonLocalMaps = 0;
boolean newIterationJob = false;
scheduleMaps:
for (int i = 0; i < availableMapSlots; ++i) {
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
if (job.getJobConf().isIterative()) {
String iterativeAppID = job.getJobConf().getIterativeAlgorithmID();
if (iterativeAppID.equals("none")) {
throw new IOException("please specify the iteration ID!");
}
String jointype = job.getJobConf().get("mapred.iterative.jointype");
// prepare the iterationid map and jobtask map
if (!this.tracker_mtask_map.containsKey(iterativeAppID)) {
// a new iterative algorithm
Map<String, LinkedList<Integer>> new_tracker_task_map =
new HashMap<String, LinkedList<Integer>>();
this.tracker_mtask_map.put(iterativeAppID, new_tracker_task_map);
Map<String, LinkedList<Integer>> new_tracker_rtask_map =
new HashMap<String, LinkedList<Integer>>();
this.tracker_rtask_map.put(iterativeAppID, new_tracker_rtask_map);
// record the first job of the series of jobs in the iterations
this.first_job_map.put(iterativeAppID, job.getJobID());
// record the list of jobs for a iteration
HashSet<JobID> jobs = new HashSet<JobID>();
jobs.add(job.getJobID());
this.iteration_jobs_map.put(iterativeAppID, jobs);
}
// this is the first job of the series of jobs
if (this.first_job_map.get(iterativeAppID).equals(job.getJobID())
&& job.getJobConf().isIterative()) {
LOG.info(job.getJobID() + " is the first iteration job");
newIterationJob = true;
}
// this is one of the following jobs, and prepare a assignment list for the assignment
if (!newIterationJob) {
LOG.info(job.getJobID() + " is not the first iteration job");
this.iteration_jobs_map.get(iterativeAppID).add(job.getJobID());
if (this.mtask_assign_map.get(job.getJobID()) == null) {
// prepare the map task assignment list
LOG.info("for job " + job.getJobID() + "'s assignment:");
Map<String, LinkedList<Integer>> map_task_assign =
new HashMap<String, LinkedList<Integer>>();
for (Map.Entry<String, LinkedList<Integer>> entry :
this.tracker_mtask_map.get(iterativeAppID).entrySet()) {
String tracker = entry.getKey();
LinkedList<Integer> taskids = entry.getValue();
LinkedList<Integer> copytaskids = new LinkedList<Integer>();
LOG.info("assign on tracker " + tracker);
for (int taskid : taskids) {
copytaskids.add(taskid);
LOG.info("task id " + taskid);
}
map_task_assign.put(tracker, copytaskids);
}
this.mtask_assign_map.put(job.getJobID(), map_task_assign);
// if one2one copy the map assign to reduce assign, the are with the same mapping
if (jointype.equals("one2one")) {
// prepare the reduce task assignment list
Map<String, LinkedList<Integer>> reduce_task_assign =
new HashMap<String, LinkedList<Integer>>();
for (Map.Entry<String, LinkedList<Integer>> entry :
this.tracker_mtask_map.get(iterativeAppID).entrySet()) {
String tracker = entry.getKey();
LinkedList<Integer> taskids = entry.getValue();
LinkedList<Integer> copytaskids = new LinkedList<Integer>();
for (int taskid : taskids) {
copytaskids.add(taskid);
}
reduce_task_assign.put(tracker, copytaskids);
}
this.tracker_rtask_map.put(iterativeAppID, reduce_task_assign);
}
// prepare the reduce task assignment list for all cases
Map<String, LinkedList<Integer>> reduce_task_assign =
new HashMap<String, LinkedList<Integer>>();
for (Map.Entry<String, LinkedList<Integer>> entry :
this.tracker_rtask_map.get(iterativeAppID).entrySet()) {
String tracker = entry.getKey();
LinkedList<Integer> taskids = entry.getValue();
LinkedList<Integer> copytaskids = new LinkedList<Integer>();
for (int taskid : taskids) {
copytaskids.add(taskid);
}
reduce_task_assign.put(tracker, copytaskids);
}
this.rtask_assign_map.put(job.getJobID(), reduce_task_assign);
}
}
Task t = null;
// the first iteration or following iteration
// if the first iteration: assign taskid by default (exception for the one2mul case,
// where we assign staring from 0,...,n)
// else if the following iterations: assign taskid based on the first iteration
// assignment
if (newIterationJob) {
/**
* the one2mul case should be carefully taken care, we want to assgin map0,map1,map2
* and reduce0 to a tracker, and assign map3,map4,map5 and reduce1 to another tracker
*/
if (jointype.equals("one2mul")
&& !tracker_rtask_map
.get(iterativeAppID)
.containsKey(taskTracker.getTrackerName())) {
// if contain the tracker, that means we have assigned tasks for this tracker
int scala = job.getJobConf().getInt("mapred.iterative.data.scala", 1);
// int mapsEachTracker = job.getJobConf().getNumMapTasks() / numTaskTrackers;
int reducersEachTracker = job.getJobConf().getNumReduceTasks() / numTaskTrackers;
if (job.getJobConf().getNumReduceTasks() % numTaskTrackers != 0)
throw new IOException(
"job.getJobConf().getNumReduceTasks() % numTaskTrackers != 0");
if (!this.tracker_mtask_map
.get(iterativeAppID)
.containsKey(taskTracker.getTrackerName())) {
LinkedList<Integer> tasklist = new LinkedList<Integer>();
this.tracker_mtask_map
.get(iterativeAppID)
.put(taskTracker.getTrackerName(), tasklist);
}
if (!this.tracker_rtask_map
.get(iterativeAppID)
.containsKey(taskTracker.getTrackerName())) {
LinkedList<Integer> tasklist = new LinkedList<Integer>();
this.tracker_rtask_map
.get(iterativeAppID)
.put(taskTracker.getTrackerName(), tasklist);
}
// for debugging
String debugout1 = "maps: ";
String debugout2 = "reduces: ";
int reduceOffsetId =
(tracker_rtask_map.get(iterativeAppID).size() - 1)
* reducersEachTracker; // the start reduce id
for (int count = 0; count < reducersEachTracker; count++) {
int reducepartitionid = reduceOffsetId + count;
debugout2 += reducepartitionid + " ";
tracker_rtask_map
.get(iterativeAppID)
.get(taskTracker.getTrackerName())
.add(reducepartitionid);
for (int count2 = 0; count2 < scala; count2++) {
int mappartitionid = reducepartitionid * scala + count2;
// int mapid = job.splitTaskMap.get(mappartitionid);
debugout1 += mappartitionid + " ";
this.tracker_mtask_map
.get(iterativeAppID)
.get(taskTracker.getTrackerName())
.add(mappartitionid);
}
}
// print out for debug
LOG.info(
"tracker "
+ taskTracker.getTrackerName()
+ " assigned tasks "
+ debugout1
+ " and "
+ debugout2);
// make the assignment list
String tracker = taskTracker.getTrackerName();
LinkedList<Integer> mtaskids =
this.tracker_mtask_map.get(iterativeAppID).get(taskTracker.getTrackerName());
LinkedList<Integer> mcopytaskids = new LinkedList<Integer>();
for (int taskid : mtaskids) {
mcopytaskids.add(taskid);
}
if (!mtask_assign_map.containsKey(job.getJobID())) {
Map<String, LinkedList<Integer>> map_task_assign =
new HashMap<String, LinkedList<Integer>>();
this.mtask_assign_map.put(job.getJobID(), map_task_assign);
}
this.mtask_assign_map.get(job.getJobID()).put(tracker, mcopytaskids);
// prepare the reduce task assignment list
LinkedList<Integer> rtaskids =
this.tracker_rtask_map.get(iterativeAppID).get(taskTracker.getTrackerName());
LinkedList<Integer> rcopytaskids = new LinkedList<Integer>();
for (int taskid : rtaskids) {
rcopytaskids.add(taskid);
}
if (!rtask_assign_map.containsKey(job.getJobID())) {
Map<String, LinkedList<Integer>> reduce_task_assign =
new HashMap<String, LinkedList<Integer>>();
this.rtask_assign_map.put(job.getJobID(), reduce_task_assign);
}
this.rtask_assign_map.get(job.getJobID()).put(tracker, rcopytaskids);
// assign a map task for this tracker
Integer target = null;
try {
target =
this.mtask_assign_map
.get(job.getJobID())
.get(taskTracker.getTrackerName())
.peekFirst();
} catch (Exception e) {
e.printStackTrace();
}
if (target == null) {
// all have been assigned, no more work, maybe it should help others to process
LOG.info(
"all map tasks on tasktracker "
+ taskTracker.getTrackerName()
+ " have been processed");
break;
} else {
t =
job.obtainNewNodeOrRackLocalMapTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts(),
target);
}
} else {
t =
job.obtainNewNodeOrRackLocalMapTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts());
}
} else {
Integer target = null;
try {
target =
this.mtask_assign_map
.get(job.getJobID())
.get(taskTracker.getTrackerName())
.peekFirst();
} catch (Exception e) {
e.printStackTrace();
}
if (target == null) {
// all have been assigned, no more work, maybe it should help others to process
LOG.info(
"all map tasks on tasktracker "
+ taskTracker.getTrackerName()
+ " have been processed");
break;
} else {
t =
job.obtainNewNodeOrRackLocalMapTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts(),
target);
}
}
if (t != null) {
assignedTasks.add(t);
++numLocalMaps;
// new iteration job and the first task for a tasktracker
// for one2mul case, we don't need to record the assignment, since we already made the
// assignment list beforehand
if (!newIterationJob || jointype.equals("one2mul")) {
// poll, remove
this.mtask_assign_map
.get(job.getJobID())
.get(taskTracker.getTrackerName())
.pollFirst();
LOG.info("assigning task " + t.getTaskID() + " on " + taskTracker.getTrackerName());
} else {
// record the assignment list for map tasks
if (!this.tracker_mtask_map
.get(iterativeAppID)
.containsKey(taskTracker.getTrackerName())) {
LinkedList<Integer> tasklist = new LinkedList<Integer>();
this.tracker_mtask_map
.get(iterativeAppID)
.put(taskTracker.getTrackerName(), tasklist);
}
this.tracker_mtask_map
.get(iterativeAppID)
.get(taskTracker.getTrackerName())
.add(t.getTaskID().getTaskID().getId());
// prepare the reduce assignment, for mapping with reduce
if (jointype.equals("one2one")) {
// prepare the reduce assignment, for mapping with reduce
if (!first_job_reduces_map.containsKey(iterativeAppID)) {
Map<String, LinkedList<Integer>> tracker_reduce_map =
new HashMap<String, LinkedList<Integer>>();
first_job_reduces_map.put(iterativeAppID, tracker_reduce_map);
}
if (!first_job_reduces_map
.get(iterativeAppID)
.containsKey(taskTracker.getTrackerName())) {
LinkedList<Integer> reduces = new LinkedList<Integer>();
first_job_reduces_map
.get(iterativeAppID)
.put(taskTracker.getTrackerName(), reduces);
}
first_job_reduces_map
.get(iterativeAppID)
.get(taskTracker.getTrackerName())
.add(t.getTaskID().getTaskID().getId());
}
LOG.info("assigning task " + t.getTaskID() + " on " + taskTracker.getTrackerName());
}
// Don't assign map tasks to the hilt!
// Leave some free slots in the cluster for future task-failures,
// speculative tasks etc. beyond the highest priority job
if (exceededMapPadding) {
break scheduleMaps;
}
// Try all jobs again for the next Map task
break;
}
LOG.error("New Node Or Rack Local Map Task failed!");
if (newIterationJob) {
// Try to schedule a node-local or rack-local Map task
t =
job.obtainNewNonLocalMapTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts());
} else {
Integer target =
this.mtask_assign_map
.get(job.getJobID())
.get(taskTracker.getTrackerName())
.peekFirst();
if (target == null) {
// all have been assigned, no more work, maybe it should help others to process
LOG.info(
"all map tasks on tasktracker "
+ taskTracker.getTrackerName()
+ " have been processed");
break;
} else {
t =
job.obtainNewNonLocalMapTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts(),
target);
}
}
if (t != null) {
assignedTasks.add(t);
++numNonLocalMaps;
// new iteration job and the first task for a tasktracker
if (newIterationJob) {
if (!this.tracker_mtask_map
.get(iterativeAppID)
.containsKey(taskTracker.getTrackerName())) {
LinkedList<Integer> tasklist = new LinkedList<Integer>();
this.tracker_mtask_map
.get(iterativeAppID)
.put(taskTracker.getTrackerName(), tasklist);
}
this.tracker_mtask_map
.get(iterativeAppID)
.get(taskTracker.getTrackerName())
.add(t.getTaskID().getTaskID().getId());
} else {
// poll, remove
this.mtask_assign_map
.get(job.getJobID())
.get(taskTracker.getTrackerName())
.pollFirst();
}
// We assign at most 1 off-switch or speculative task
// This is to prevent TaskTrackers from stealing local-tasks
// from other TaskTrackers.
break scheduleMaps;
}
} else {
// not an iterative algorithm, normal schedule
Task t = null;
// Try to schedule a node-local or rack-local Map task
t =
job.obtainNewNodeOrRackLocalMapTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
++numLocalMaps;
// Don't assign map tasks to the hilt!
// Leave some free slots in the cluster for future task-failures,
// speculative tasks etc. beyond the highest priority job
if (exceededMapPadding) {
break scheduleMaps;
}
// Try all jobs again for the next Map task
break;
}
// Try to schedule a node-local or rack-local Map task
t =
job.obtainNewNonLocalMapTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
++numNonLocalMaps;
// We assign at most 1 off-switch or speculative task
// This is to prevent TaskTrackers from stealing local-tasks
// from other TaskTrackers.
break scheduleMaps;
}
}
}
}
}
int assignedMaps = assignedTasks.size();
//
// Same thing, but for reduce tasks
// However we _never_ assign more than 1 reduce task per heartbeat
//
/** should maintain the reduce task location for the termination check */
final int trackerCurrentReduceCapacity =
Math.min((int) Math.ceil(reduceLoadFactor * trackerReduceCapacity), trackerReduceCapacity);
final int availableReduceSlots =
Math.min((trackerCurrentReduceCapacity - trackerRunningReduces), 1);
boolean exceededReducePadding = false;
// LOG.info("availableReduceSlots " + availableReduceSlots);
if (availableReduceSlots > 0) {
exceededReducePadding = exceededPadding(false, clusterStatus, trackerReduceCapacity);
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
LOG.info("job " + job.getJobID());
if (job.getStatus().getRunState() != JobStatus.RUNNING || job.numReduceTasks == 0) {
LOG.info("have to continue " + job.getStatus().getRunState());
continue;
}
Task t = null;
if (job.getJobConf().isIterative()) {
String iterativeAppID = job.getJobConf().getIterativeAlgorithmID();
if (iterativeAppID.equals("none")) {
throw new IOException("please specify the iteration ID!");
}
String jointype = job.getJobConf().get("mapred.iterative.jointype");
if (jointype.equals("one2one")) {
// one-to-one or one-to-mul jobs
if (this.first_job_map.get(iterativeAppID).equals(job.getJobID())
&& job.getJobConf().isIterative()) {
LOG.info(job.getJobID() + " is the first iteration job for reduce");
newIterationJob = true;
}
Integer target = null;
if (newIterationJob) {
if (first_job_reduces_map.get(iterativeAppID) == null) {
throw new IOException(
"I think something is wrong since the tasktracker never receive "
+ "a map task with iterativeapp id "
+ iterativeAppID);
}
if (first_job_reduces_map.get(iterativeAppID).get(taskTracker.getTrackerName())
== null) {
throw new IOException(
"I think something is wrong since the tasktracker never receive "
+ "a map task with iterativeapp id "
+ iterativeAppID
+ " from "
+ taskTracker.getTrackerName());
}
target =
this.first_job_reduces_map
.get(iterativeAppID)
.get(taskTracker.getTrackerName())
.pollFirst();
} else {
// the task assignment has already been processed during the map task assignment, so
// never use tracker_rtask_map
target =
this.rtask_assign_map
.get(job.getJobID())
.get(taskTracker.getTrackerName())
.pollFirst();
}
if (target == null) {
// all have been assigned, no more work, maybe it should help others to process
LOG.info(
"all reduce tasks on tasktracker "
+ taskTracker.getTrackerName()
+ " have been processed");
break;
} else {
t =
job.obtainNewReduceTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts(),
target);
}
} else if (jointype.equals("one2mul")) {
Integer target =
this.rtask_assign_map
.get(job.getJobID())
.get(taskTracker.getTrackerName())
.pollFirst();
if (target == null) {
// all have been assigned, no more work, maybe it should help others to process
LOG.info(
"all reduce tasks on tasktracker "
+ taskTracker.getTrackerName()
+ " have been processed");
break;
} else {
t =
job.obtainNewReduceTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts(),
target);
}
} else {
// one-to-all case, assign tasks in the first iteration job, and remember this mapping
// this is the first job of the series of jobs
if (this.first_job_map.get(iterativeAppID).equals(job.getJobID())) {
LOG.info(job.getJobID() + " is the first iteration job for reduce");
newIterationJob = true;
}
/*
//this is one of the following jobs, and prepare a assignment list for the assignment
else{
LOG.info(job.getJobID() + " is not the first iteration job for reduce");
if(this.rtask_assign_map.get(job.getJobID()) == null){
//prepare the map task assignment list
Map<String, LinkedList<Integer>> reduce_task_assign = new HashMap<String, LinkedList<Integer>>();
for(Map.Entry<String, LinkedList<Integer>> entry : this.tracker_rtask_map.get(iterativeAppID).entrySet()){
String tracker = entry.getKey();
LinkedList<Integer> taskids = entry.getValue();
LinkedList<Integer> copytaskids = new LinkedList<Integer>();
for(int taskid : taskids){
copytaskids.add(taskid);
}
reduce_task_assign.put(tracker, copytaskids);
}
this.rtask_assign_map.put(job.getJobID(), reduce_task_assign);
}
}
*/
// the first iteration or following iteration
// if the first iteration: assign taskid by default
// else if the following iterations: assign taskid based on the first iteration
// assignment
if (newIterationJob) {
t =
job.obtainNewReduceTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
if (!this.tracker_rtask_map
.get(iterativeAppID)
.containsKey(taskTracker.getTrackerName())) {
LinkedList<Integer> tasklist = new LinkedList<Integer>();
this.tracker_rtask_map
.get(iterativeAppID)
.put(taskTracker.getTrackerName(), tasklist);
}
this.tracker_rtask_map
.get(iterativeAppID)
.get(taskTracker.getTrackerName())
.add(t.getTaskID().getTaskID().getId());
LOG.info(
"assigning reduce task "
+ t.getTaskID()
+ " on "
+ taskTracker.getTrackerName());
}
} else {
Integer target =
this.rtask_assign_map
.get(job.getJobID())
.get(taskTracker.getTrackerName())
.peekFirst();
if (target == null) {
// all have been assigned, no more work, maybe it should help others to process
LOG.info(
"all map tasks on tasktracker "
+ taskTracker.getTrackerName()
+ " have been processed");
break;
} else {
t =
job.obtainNewReduceTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts(),
target);
}
if (t != null) {
// poll, remove
this.rtask_assign_map
.get(job.getJobID())
.get(taskTracker.getTrackerName())
.pollFirst();
LOG.info(
"assigning reduce task "
+ t.getTaskID()
+ " on "
+ taskTracker.getTrackerName());
}
}
}
} else {
t =
job.obtainNewReduceTask(
taskTrackerStatus,
numTaskTrackers,
taskTrackerManager.getNumberOfUniqueHosts());
}
LOG.info("try to assign new task " + t);
if (t != null) {
assignedTasks.add(t);
break;
}
// Don't assign reduce tasks to the hilt!
// Leave some free slots in the cluster for future task-failures,
// speculative tasks etc. beyond the highest priority job
if (exceededReducePadding) {
break;
}
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Task assignments for "
+ taskTrackerStatus.getTrackerName()
+ " --> "
+ "["
+ mapLoadFactor
+ ", "
+ trackerMapCapacity
+ ", "
+ trackerCurrentMapCapacity
+ ", "
+ trackerRunningMaps
+ "] -> ["
+ (trackerCurrentMapCapacity - trackerRunningMaps)
+ ", "
+ assignedMaps
+ " ("
+ numLocalMaps
+ ", "
+ numNonLocalMaps
+ ")] ["
+ reduceLoadFactor
+ ", "
+ trackerReduceCapacity
+ ", "
+ trackerCurrentReduceCapacity
+ ","
+ trackerRunningReduces
+ "] -> ["
+ (trackerCurrentReduceCapacity - trackerRunningReduces)
+ ", "
+ (assignedTasks.size() - assignedMaps)
+ "]");
}
return assignedTasks;
}
/*
* TODO:
* For Elf: need to change the major schedule logic, scheduling need
* to be *datacenter-aware*
* */
@Override
public synchronized List<Task> assignTasks(TaskTracker taskTracker) throws IOException {
TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
final int numTaskTrackers = clusterStatus.getTaskTrackers();
final int clusterMapCapacity = clusterStatus.getMaxMapTasks();
final int clusterReduceCapacity = clusterStatus.getMaxReduceTasks();
Collection<JobInProgress> jobQueue = jobQueueJobInProgressListener.getJobQueue();
//
// Get map + reduce counts for the current tracker.
//
final int trackerMapCapacity = taskTrackerStatus.getMaxMapSlots();
final int trackerReduceCapacity = taskTrackerStatus.getMaxReduceSlots();
final int trackerRunningMaps = taskTrackerStatus.countMapTasks();
final int trackerRunningReduces = taskTrackerStatus.countReduceTasks();
// Assigned tasks
List<Task> assignedTasks = new ArrayList<Task>();
//
// Compute (running + pending) map and reduce task numbers across pool
//
int remainingReduceLoad = 0;
int remainingMapLoad = 0;
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() == JobStatus.RUNNING) {
remainingMapLoad += (job.desiredMaps() - job.finishedMaps());
if (job.scheduleReduces()) {
remainingReduceLoad += (job.desiredReduces() - job.finishedReduces());
}
}
}
}
// Compute the 'load factor' for maps and reduces
double mapLoadFactor = 0.0;
if (clusterMapCapacity > 0) {
mapLoadFactor = (double) remainingMapLoad / clusterMapCapacity;
}
double reduceLoadFactor = 0.0;
if (clusterReduceCapacity > 0) {
reduceLoadFactor = (double) remainingReduceLoad / clusterReduceCapacity;
}
//
// In the below steps, we allocate first map tasks (if appropriate),
// and then reduce tasks if appropriate. We go through all jobs
// in order of job arrival; jobs only get serviced if their
// predecessors are serviced, too.
//
//
// We assign tasks to the current taskTracker if the given machine
// has a workload that's less than the maximum load of that kind of
// task.
// However, if the cluster is close to getting loaded i.e. we don't
// have enough _padding_ for speculative executions etc., we only
// schedule the "highest priority" task i.e. the task from the job
// with the highest priority.
//
final int trackerCurrentMapCapacity =
Math.min((int) Math.ceil(mapLoadFactor * trackerMapCapacity), trackerMapCapacity);
int availableMapSlots = trackerCurrentMapCapacity - trackerRunningMaps;
boolean exceededMapPadding = false;
if (availableMapSlots > 0) {
exceededMapPadding = exceededPadding(true, clusterStatus, trackerMapCapacity);
}
int numLocalMaps = 0;
int numNonLocalMaps = 0;
scheduleMaps:
// TODO: for Elf
// The main schedule logic here, outer for loop is for every slot, inner loop is for each job
for (int i = 0; i < availableMapSlots; ++i) {
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = null;
// Try to schedule a node-local or rack-local Map task
t =
job.obtainNewLocalMapTask(
taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
++numLocalMaps;
// Don't assign map tasks to the hilt!
// Leave some free slots in the cluster for future task-failures,
// speculative tasks etc. beyond the highest priority job
if (exceededMapPadding) {
break scheduleMaps;
}
// Try all jobs again for the next Map task
// Note: it's FIFO here: next time in the inner for loop the head-of-queue
// will still be chosen
break;
}
// If no locality for this job, try launching non-local
// Try to schedule a node-local or rack-local Map task --> original comments
// FIXME: is the above comment correct? seems should be non-local task
t =
job.obtainNewNonLocalMapTask(
taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
++numNonLocalMaps;
// We assign at most 1 off-switch or speculative task
// This is to prevent TaskTrackers from stealing local-tasks
// from other TaskTrackers.
break scheduleMaps;
}
}
}
}
int assignedMaps = assignedTasks.size();
//
// Same thing, but for reduce tasks
// However we _never_ assign more than 1 reduce task per heartbeat
//
final int trackerCurrentReduceCapacity =
Math.min((int) Math.ceil(reduceLoadFactor * trackerReduceCapacity), trackerReduceCapacity);
final int availableReduceSlots =
Math.min((trackerCurrentReduceCapacity - trackerRunningReduces), 1);
boolean exceededReducePadding = false;
if (availableReduceSlots > 0) {
exceededReducePadding = exceededPadding(false, clusterStatus, trackerReduceCapacity);
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() != JobStatus.RUNNING || job.numReduceTasks == 0) {
continue;
}
Task t =
job.obtainNewReduceTask(
taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
break;
}
// Don't assign reduce tasks to the hilt!
// Leave some free slots in the cluster for future task-failures,
// speculative tasks etc. beyond the highest priority job
if (exceededReducePadding) {
break;
}
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Task assignments for "
+ taskTrackerStatus.getTrackerName()
+ " --> "
+ "["
+ mapLoadFactor
+ ", "
+ trackerMapCapacity
+ ", "
+ trackerCurrentMapCapacity
+ ", "
+ trackerRunningMaps
+ "] -> ["
+ (trackerCurrentMapCapacity - trackerRunningMaps)
+ ", "
+ assignedMaps
+ " ("
+ numLocalMaps
+ ", "
+ numNonLocalMaps
+ ")] ["
+ reduceLoadFactor
+ ", "
+ trackerReduceCapacity
+ ", "
+ trackerCurrentReduceCapacity
+ ","
+ trackerRunningReduces
+ "] -> ["
+ (trackerCurrentReduceCapacity - trackerRunningReduces)
+ ", "
+ (assignedTasks.size() - assignedMaps)
+ "]");
}
return assignedTasks;
}
/**
* A tracker wants to know if there's a Task to run. Returns a task we'd like the TaskTracker to
* execute right now.
*
* <p>Eventually this function should compute load on the various TaskTrackers, and incorporate
* knowledge of DFS file placement. But for right now, it just grabs a single item out of the
* pending task list and hands it back.
*/
public synchronized Task pollForNewTask(String taskTracker) {
//
// Compute average map and reduce task numbers across pool
//
int avgMaps = 0;
int avgReduces = 0;
int numTaskTrackers;
TaskTrackerStatus tts;
synchronized (taskTrackers) {
numTaskTrackers = taskTrackers.size();
tts = (TaskTrackerStatus) taskTrackers.get(taskTracker);
}
if (numTaskTrackers > 0) {
avgMaps = totalMaps / numTaskTrackers;
avgReduces = totalReduces / numTaskTrackers;
}
int totalCapacity = numTaskTrackers * maxCurrentTasks;
//
// Get map + reduce counts for the current tracker.
//
if (tts == null) {
LOG.warning("Unknown task tracker polling; ignoring: " + taskTracker);
return null;
}
int numMaps = tts.countMapTasks();
int numReduces = tts.countReduceTasks();
//
// In the below steps, we allocate first a map task (if appropriate),
// and then a reduce task if appropriate. We go through all jobs
// in order of job arrival; jobs only get serviced if their
// predecessors are serviced, too.
//
//
// We hand a task to the current taskTracker if the given machine
// has a workload that's equal to or less than the averageMaps
// +/- TASK_ALLOC_EPSILON. (That epsilon is in place in case
// there is an odd machine that is failing for some reason but
// has not yet been removed from the pool, making capacity seem
// larger than it really is.)
//
synchronized (jobsByArrival) {
if ((numMaps < maxCurrentTasks) && (numMaps <= (avgMaps + TASK_ALLOC_EPSILON))) {
int totalNeededMaps = 0;
for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
JobInProgress job = (JobInProgress) it.next();
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = job.obtainNewMapTask(taskTracker, tts);
if (t != null) {
return t;
}
//
// Beyond the highest-priority task, reserve a little
// room for failures and speculative executions; don't
// schedule tasks to the hilt.
//
totalNeededMaps += job.desiredMaps();
double padding = 0;
if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
padding = Math.min(maxCurrentTasks, totalNeededMaps * PAD_FRACTION);
}
if (totalNeededMaps + padding >= totalCapacity) {
break;
}
}
}
//
// Same thing, but for reduce tasks
//
if ((numReduces < maxCurrentTasks) && (numReduces <= (avgReduces + TASK_ALLOC_EPSILON))) {
int totalNeededReduces = 0;
for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
JobInProgress job = (JobInProgress) it.next();
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = job.obtainNewReduceTask(taskTracker, tts);
if (t != null) {
return t;
}
//
// Beyond the highest-priority task, reserve a little
// room for failures and speculative executions; don't
// schedule tasks to the hilt.
//
totalNeededReduces += job.desiredReduces();
double padding = 0;
if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
padding = Math.min(maxCurrentTasks, totalNeededReduces * PAD_FRACTION);
}
if (totalNeededReduces + padding >= totalCapacity) {
break;
}
}
}
}
return null;
}