/** Process incoming heartbeat messages from the task trackers. */
public synchronized int emitHeartbeat(TaskTrackerStatus trackerStatus, boolean initialContact) {
String trackerName = trackerStatus.getTrackerName();
trackerStatus.setLastSeen(System.currentTimeMillis());
synchronized (taskTrackers) {
synchronized (trackerExpiryQueue) {
boolean seenBefore = updateTaskTrackerStatus(trackerName, trackerStatus);
if (initialContact) {
// If it's first contact, then clear out any state hanging around
if (seenBefore) {
lostTaskTracker(trackerName);
}
} else {
// If not first contact, there should be some record of the tracker
if (!seenBefore) {
return InterTrackerProtocol.UNKNOWN_TASKTRACKER;
}
}
if (initialContact) {
trackerExpiryQueue.add(trackerStatus);
}
}
}
updateTaskStatuses(trackerStatus);
// LOG.info("Got heartbeat from "+trackerName);
return InterTrackerProtocol.TRACKERS_OK;
}
/**
* Update the last recorded status for the given task tracker. It assumes that the taskTrackers
* are locked on entry.
*
* @author Owen O'Malley
* @param trackerName The name of the tracker
* @param status The new status for the task tracker
* @return Was an old status found?
*/
private boolean updateTaskTrackerStatus(String trackerName, TaskTrackerStatus status) {
TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.get(trackerName);
if (oldStatus != null) {
totalMaps -= oldStatus.countMapTasks();
totalReduces -= oldStatus.countReduceTasks();
if (status == null) {
taskTrackers.remove(trackerName);
}
}
if (status != null) {
totalMaps += status.countMapTasks();
totalReduces += status.countReduceTasks();
taskTrackers.put(trackerName, status);
}
return oldStatus != null;
}
public int compare(Object o1, Object o2) {
TaskTrackerStatus p1 = (TaskTrackerStatus) o1;
TaskTrackerStatus p2 = (TaskTrackerStatus) o2;
if (p1.getLastSeen() < p2.getLastSeen()) {
return -1;
} else if (p1.getLastSeen() > p2.getLastSeen()) {
return 1;
} else {
return (p1.getTrackerName().compareTo(p2.getTrackerName()));
}
}
/**
* Accept and process a new TaskTracker profile. We might have known about the TaskTracker
* previously, or it might be brand-new. All task-tracker structures have already been updated.
* Just process the contained tasks and any jobs that might be affected.
*/
void updateTaskStatuses(TaskTrackerStatus status) {
for (Iterator it = status.taskReports(); it.hasNext(); ) {
TaskStatus report = (TaskStatus) it.next();
TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(report.getTaskId());
if (tip == null) {
LOG.info(
"Serious problem. While updating status, cannot find taskid " + report.getTaskId());
} else {
JobInProgress job = tip.getJob();
job.updateTaskStatus(tip, report);
if (report.getRunState() == TaskStatus.SUCCEEDED) {
job.completedTask(tip, report.getTaskId());
} else if (report.getRunState() == TaskStatus.FAILED) {
// Tell the job to fail the relevant task
job.failedTask(tip, report.getTaskId(), status.getTrackerName());
}
}
}
}
/**
* A TaskTracker wants to know the physical locations of completed, but not yet closed, tasks.
* This exists so the reduce task thread can locate map task outputs.
*/
public synchronized MapOutputLocation[] locateMapOutputs(
String taskId, String[][] mapTasksNeeded) {
ArrayList v = new ArrayList();
for (int i = 0; i < mapTasksNeeded.length; i++) {
for (int j = 0; j < mapTasksNeeded[i].length; j++) {
TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(mapTasksNeeded[i][j]);
if (tip != null && tip.isComplete(mapTasksNeeded[i][j])) {
String trackerId = (String) taskidToTrackerMap.get(mapTasksNeeded[i][j]);
TaskTrackerStatus tracker;
synchronized (taskTrackers) {
tracker = (TaskTrackerStatus) taskTrackers.get(trackerId);
}
v.add(new MapOutputLocation(mapTasksNeeded[i][j], tracker.getHost(), tracker.getPort()));
break;
}
}
}
// randomly shuffle results to load-balance map output requests
Collections.shuffle(v);
return (MapOutputLocation[]) v.toArray(new MapOutputLocation[v.size()]);
}
/**
* The run method lives for the life of the JobTracker, and removes TaskTrackers that have not
* checked in for some time.
*/
public void run() {
while (shouldRun) {
//
// Thread runs periodically to check whether trackers should be expired.
// The sleep interval must be no more than half the maximum expiry time
// for a task tracker.
//
try {
Thread.sleep(TASKTRACKER_EXPIRY_INTERVAL / 3);
} catch (InterruptedException ie) {
}
//
// Loop through all expired items in the queue
//
synchronized (taskTrackers) {
synchronized (trackerExpiryQueue) {
long now = System.currentTimeMillis();
TaskTrackerStatus leastRecent = null;
while ((trackerExpiryQueue.size() > 0)
&& ((leastRecent = (TaskTrackerStatus) trackerExpiryQueue.first()) != null)
&& (now - leastRecent.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL)) {
// Remove profile from head of queue
trackerExpiryQueue.remove(leastRecent);
String trackerName = leastRecent.getTrackerName();
// Figure out if last-seen time should be updated, or if tracker is dead
TaskTrackerStatus newProfile =
(TaskTrackerStatus) taskTrackers.get(leastRecent.getTrackerName());
// Items might leave the taskTracker set through other means; the
// status stored in 'taskTrackers' might be null, which means the
// tracker has already been destroyed.
if (newProfile != null) {
if (now - newProfile.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL) {
// Remove completely
updateTaskTrackerStatus(trackerName, null);
lostTaskTracker(leastRecent.getTrackerName());
} else {
// Update time by inserting latest profile
trackerExpiryQueue.add(newProfile);
}
}
}
}
}
}
}
private void printFailedAttempts(
JspWriter out,
JobTracker tracker,
JobID jobId,
TaskInProgress tip,
TaskStatus.State failState)
throws IOException {
TaskStatus[] statuses = tip.getTaskStatuses();
TaskID tipId = tip.getTIPId();
for (int i = 0; i < statuses.length; ++i) {
TaskStatus.State taskState = statuses[i].getRunState();
if ((failState == null
&& (taskState == TaskStatus.State.FAILED || taskState == TaskStatus.State.KILLED))
|| taskState == failState) {
String taskTrackerName = statuses[i].getTaskTracker();
TaskTrackerStatus taskTracker = tracker.getTaskTrackerStatus(taskTrackerName);
out.print(
"<tr><td>"
+ statuses[i].getTaskID()
+ "</td><td><a href=\"taskdetails.jsp?jobid="
+ jobId
+ "&tipid="
+ tipId
+ "\">"
+ tipId
+ "</a></td>");
if (taskTracker == null) {
out.print("<td>" + taskTrackerName + "</td>");
} else {
out.print(
"<td><a href=\"http://"
+ taskTracker.getHost()
+ ":"
+ taskTracker.getHttpPort()
+ "\">"
+ taskTracker.getHost()
+ "</a></td>");
}
out.print("<td>" + taskState + "</td>");
out.print("<td><pre>");
String[] failures = tracker.getTaskDiagnostics(statuses[i].getTaskID());
if (failures == null) {
out.print(" ");
} else {
for (int j = 0; j < failures.length; j++) {
out.print(failures[j]);
if (j < (failures.length - 1)) {
out.print("\n-------\n");
}
}
}
out.print("</pre></td>");
out.print("<td>");
String taskLogUrl = null;
if (taskTracker != null) {
taskLogUrl =
TaskLogServlet.getTaskLogUrl(
taskTracker.getHost(),
String.valueOf(taskTracker.getHttpPort()),
statuses[i].getTaskID().toString());
}
if (taskLogUrl != null) {
String tailFourKBUrl = taskLogUrl + "&start=-4097";
String tailEightKBUrl = taskLogUrl + "&start=-8193";
String entireLogUrl = taskLogUrl;
out.print("<a href=\"" + tailFourKBUrl + "\">Last 4KB</a><br/>");
out.print("<a href=\"" + tailEightKBUrl + "\">Last 8KB</a><br/>");
out.print("<a href=\"" + entireLogUrl + "\">All</a><br/>");
} else {
out.print("n/a"); // task tracker was lost
}
out.print("</td>");
out.print("</tr>\n");
}
}
}
/**
* A tracker wants to know if there's a Task to run. Returns a task we'd like the TaskTracker to
* execute right now.
*
* <p>Eventually this function should compute load on the various TaskTrackers, and incorporate
* knowledge of DFS file placement. But for right now, it just grabs a single item out of the
* pending task list and hands it back.
*/
public synchronized Task pollForNewTask(String taskTracker) {
//
// Compute average map and reduce task numbers across pool
//
int avgMaps = 0;
int avgReduces = 0;
int numTaskTrackers;
TaskTrackerStatus tts;
synchronized (taskTrackers) {
numTaskTrackers = taskTrackers.size();
tts = (TaskTrackerStatus) taskTrackers.get(taskTracker);
}
if (numTaskTrackers > 0) {
avgMaps = totalMaps / numTaskTrackers;
avgReduces = totalReduces / numTaskTrackers;
}
int totalCapacity = numTaskTrackers * maxCurrentTasks;
//
// Get map + reduce counts for the current tracker.
//
if (tts == null) {
LOG.warning("Unknown task tracker polling; ignoring: " + taskTracker);
return null;
}
int numMaps = tts.countMapTasks();
int numReduces = tts.countReduceTasks();
//
// In the below steps, we allocate first a map task (if appropriate),
// and then a reduce task if appropriate. We go through all jobs
// in order of job arrival; jobs only get serviced if their
// predecessors are serviced, too.
//
//
// We hand a task to the current taskTracker if the given machine
// has a workload that's equal to or less than the averageMaps
// +/- TASK_ALLOC_EPSILON. (That epsilon is in place in case
// there is an odd machine that is failing for some reason but
// has not yet been removed from the pool, making capacity seem
// larger than it really is.)
//
synchronized (jobsByArrival) {
if ((numMaps < maxCurrentTasks) && (numMaps <= (avgMaps + TASK_ALLOC_EPSILON))) {
int totalNeededMaps = 0;
for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
JobInProgress job = (JobInProgress) it.next();
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = job.obtainNewMapTask(taskTracker, tts);
if (t != null) {
return t;
}
//
// Beyond the highest-priority task, reserve a little
// room for failures and speculative executions; don't
// schedule tasks to the hilt.
//
totalNeededMaps += job.desiredMaps();
double padding = 0;
if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
padding = Math.min(maxCurrentTasks, totalNeededMaps * PAD_FRACTION);
}
if (totalNeededMaps + padding >= totalCapacity) {
break;
}
}
}
//
// Same thing, but for reduce tasks
//
if ((numReduces < maxCurrentTasks) && (numReduces <= (avgReduces + TASK_ALLOC_EPSILON))) {
int totalNeededReduces = 0;
for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
JobInProgress job = (JobInProgress) it.next();
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = job.obtainNewReduceTask(taskTracker, tts);
if (t != null) {
return t;
}
//
// Beyond the highest-priority task, reserve a little
// room for failures and speculative executions; don't
// schedule tasks to the hilt.
//
totalNeededReduces += job.desiredReduces();
double padding = 0;
if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
padding = Math.min(maxCurrentTasks, totalNeededReduces * PAD_FRACTION);
}
if (totalNeededReduces + padding >= totalCapacity) {
break;
}
}
}
}
return null;
}