void removeTaskEntry(String taskid) {
// taskid --> tracker
String tracker = (String) taskidToTrackerMap.remove(taskid);
// tracker --> taskid
TreeSet trackerSet = (TreeSet) trackerToTaskMap.get(tracker);
if (trackerSet != null) {
trackerSet.remove(taskid);
}
// taskid --> TIP
taskidToTIPMap.remove(taskid);
}
/**
* We lost the task tracker! All task-tracker structures have already been updated. Just process
* the contained tasks and any jobs that might be affected.
*/
void lostTaskTracker(String trackerName) {
LOG.info("Lost tracker '" + trackerName + "'");
TreeSet lostTasks = (TreeSet) trackerToTaskMap.get(trackerName);
trackerToTaskMap.remove(trackerName);
if (lostTasks != null) {
for (Iterator it = lostTasks.iterator(); it.hasNext(); ) {
String taskId = (String) it.next();
TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(taskId);
// Tell the job to fail the relevant task
JobInProgress job = tip.getJob();
job.failedTask(tip, taskId, trackerName);
}
}
}
/**
* JobTracker.submitJob() kicks off a new job.
*
* <p>Create a 'JobInProgress' object, which contains both JobProfile and JobStatus. Those two
* sub-objects are sometimes shipped outside of the JobTracker. But JobInProgress adds info that's
* useful for the JobTracker alone.
*
* <p>We add the JIP to the jobInitQueue, which is processed asynchronously to handle
* split-computation and build up the right TaskTracker/Block mapping.
*/
public synchronized JobStatus submitJob(String jobFile) throws IOException {
totalSubmissions++;
JobInProgress job = new JobInProgress(jobFile, this, this.conf);
synchronized (jobs) {
synchronized (jobsByArrival) {
synchronized (jobInitQueue) {
jobs.put(job.getProfile().getJobId(), job);
jobsByArrival.add(job);
jobInitQueue.add(job);
jobInitQueue.notifyAll();
}
}
}
return job.getStatus();
}
/**
* Update the last recorded status for the given task tracker. It assumes that the taskTrackers
* are locked on entry.
*
* @author Owen O'Malley
* @param trackerName The name of the tracker
* @param status The new status for the task tracker
* @return Was an old status found?
*/
private boolean updateTaskTrackerStatus(String trackerName, TaskTrackerStatus status) {
TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.get(trackerName);
if (oldStatus != null) {
totalMaps -= oldStatus.countMapTasks();
totalReduces -= oldStatus.countReduceTasks();
if (status == null) {
taskTrackers.remove(trackerName);
}
}
if (status != null) {
totalMaps += status.countMapTasks();
totalReduces += status.countReduceTasks();
taskTrackers.put(trackerName, status);
}
return oldStatus != null;
}
public synchronized JobProfile getJobProfile(String jobid) {
JobInProgress job = (JobInProgress) jobs.get(jobid);
if (job != null) {
return job.getProfile();
} else {
return null;
}
}
public synchronized JobStatus getJobStatus(String jobid) {
JobInProgress job = (JobInProgress) jobs.get(jobid);
if (job != null) {
return job.getStatus();
} else {
return null;
}
}
/**
* A tracker wants to know if any of its Tasks have been closed (because the job completed,
* whether successfully or not)
*/
public synchronized String pollForTaskWithClosedJob(String taskTracker) {
TreeSet taskIds = (TreeSet) trackerToTaskMap.get(taskTracker);
if (taskIds != null) {
for (Iterator it = taskIds.iterator(); it.hasNext(); ) {
String taskId = (String) it.next();
TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(taskId);
if (tip.shouldCloseForClosedJob(taskId)) {
//
// This is how the JobTracker ends a task at the TaskTracker.
// It may be successfully completed, or may be killed in
// mid-execution.
//
return taskId;
}
}
}
return null;
}
public Vector completedJobs() {
Vector v = new Vector();
for (Iterator it = jobs.values().iterator(); it.hasNext(); ) {
JobInProgress jip = (JobInProgress) it.next();
JobStatus status = jip.getStatus();
if (status.getRunState() == JobStatus.SUCCEEDED) {
v.add(jip);
}
}
return v;
}
public Vector runningJobs() {
Vector v = new Vector();
for (Iterator it = jobs.values().iterator(); it.hasNext(); ) {
JobInProgress jip = (JobInProgress) it.next();
JobStatus status = jip.getStatus();
if (status.getRunState() == JobStatus.RUNNING) {
v.add(jip);
}
}
return v;
}
/**
* A TaskTracker wants to know the physical locations of completed, but not yet closed, tasks.
* This exists so the reduce task thread can locate map task outputs.
*/
public synchronized MapOutputLocation[] locateMapOutputs(
String taskId, String[][] mapTasksNeeded) {
ArrayList v = new ArrayList();
for (int i = 0; i < mapTasksNeeded.length; i++) {
for (int j = 0; j < mapTasksNeeded[i].length; j++) {
TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(mapTasksNeeded[i][j]);
if (tip != null && tip.isComplete(mapTasksNeeded[i][j])) {
String trackerId = (String) taskidToTrackerMap.get(mapTasksNeeded[i][j]);
TaskTrackerStatus tracker;
synchronized (taskTrackers) {
tracker = (TaskTrackerStatus) taskTrackers.get(trackerId);
}
v.add(new MapOutputLocation(mapTasksNeeded[i][j], tracker.getHost(), tracker.getPort()));
break;
}
}
}
// randomly shuffle results to load-balance map output requests
Collections.shuffle(v);
return (MapOutputLocation[]) v.toArray(new MapOutputLocation[v.size()]);
}
// Used for debugging in main.
private static void printDataMap(Map map) throws Exception {
TreeMap res = new TreeMap(map);
Iterator key_itr = map.keySet().iterator();
while (key_itr.hasNext()) {
Object key = key_itr.next();
Object val = map.get(key);
if (val instanceof byte[]) {
String as_bytes = ByteFormatter.nicePrint((byte[]) val);
String as_text = new String((byte[]) val, Constants.BYTE_ENCODING);
res.put(key, as_text + " [" + as_bytes + "]");
}
}
Iterator entries = res.entrySet().iterator();
Map.Entry entry;
while (entries.hasNext()) {
entry = (Map.Entry) entries.next();
System.out.print(" ");
System.out.print(entry.getKey());
System.out.print(": ");
System.out.print(entry.getValue());
System.out.println();
}
}
///////////////////////////////////////////////////////
// Maintain lookup tables; called by JobInProgress
// and TaskInProgress
///////////////////////////////////////////////////////
void createTaskEntry(String taskid, String taskTracker, TaskInProgress tip) {
LOG.info(
"Adding task '"
+ taskid
+ "' to tip "
+ tip.getTIPId()
+ ", for tracker '"
+ taskTracker
+ "'");
// taskid --> tracker
taskidToTrackerMap.put(taskid, taskTracker);
// tracker --> taskid
TreeSet taskset = (TreeSet) trackerToTaskMap.get(taskTracker);
if (taskset == null) {
taskset = new TreeSet();
trackerToTaskMap.put(taskTracker, taskset);
}
taskset.add(taskid);
// taskid --> TIP
taskidToTIPMap.put(taskid, tip);
}
public synchronized TaskReport[] getReduceTaskReports(String jobid) {
JobInProgress job = (JobInProgress) jobs.get(jobid);
if (job == null) {
return new TaskReport[0];
} else {
Vector reports = new Vector();
Vector completeReduceTasks = job.reportTasksInProgress(false, true);
for (Iterator it = completeReduceTasks.iterator(); it.hasNext(); ) {
TaskInProgress tip = (TaskInProgress) it.next();
reports.add(tip.generateSingleReport());
}
Vector incompleteReduceTasks = job.reportTasksInProgress(false, false);
for (Iterator it = incompleteReduceTasks.iterator(); it.hasNext(); ) {
TaskInProgress tip = (TaskInProgress) it.next();
reports.add(tip.generateSingleReport());
}
return (TaskReport[]) reports.toArray(new TaskReport[reports.size()]);
}
}
/**
* Accept and process a new TaskTracker profile. We might have known about the TaskTracker
* previously, or it might be brand-new. All task-tracker structures have already been updated.
* Just process the contained tasks and any jobs that might be affected.
*/
void updateTaskStatuses(TaskTrackerStatus status) {
for (Iterator it = status.taskReports(); it.hasNext(); ) {
TaskStatus report = (TaskStatus) it.next();
TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(report.getTaskId());
if (tip == null) {
LOG.info(
"Serious problem. While updating status, cannot find taskid " + report.getTaskId());
} else {
JobInProgress job = tip.getJob();
job.updateTaskStatus(tip, report);
if (report.getRunState() == TaskStatus.SUCCEEDED) {
job.completedTask(tip, report.getTaskId());
} else if (report.getRunState() == TaskStatus.FAILED) {
// Tell the job to fail the relevant task
job.failedTask(tip, report.getTaskId(), status.getTrackerName());
}
}
}
}
///////////////////////////////////////////////////////////////
// JobTracker methods
///////////////////////////////////////////////////////////////
public JobInProgress getJob(String jobid) {
return (JobInProgress) jobs.get(jobid);
}
public synchronized void killJob(String jobid) {
JobInProgress job = (JobInProgress) jobs.get(jobid);
job.kill();
}
public synchronized ClusterStatus getClusterStatus() {
synchronized (taskTrackers) {
return new ClusterStatus(taskTrackers.size(), totalMaps, totalReduces, maxCurrentTasks);
}
}
/**
* Process an HTML get or post.
*
* @exception ServletException From inherited class.
* @exception IOException From inherited class.
*/
public void scanOutXML(
PrintWriter out,
String strDirectory,
String strFilename,
String[] strPlus,
String[] strMinus,
boolean bExcludeParams,
boolean bAnalyzeParams)
throws IOException {
File dir = new File(strDirectory + '/' + strFilename);
if (dir.isDirectory()) return;
try {
FileReader is = new FileReader(strDirectory + '/' + strFilename);
BufferedReader r = new BufferedReader(is);
String string = null;
Hashtable ht = new Hashtable();
Set setstrExtensions = new HashSet();
int iCount = 0;
int iBytes = 0;
while ((string = r.readLine()) != null) {
StringTokenizer st = new StringTokenizer(string, " \"", false);
Data data = new Data();
int iTokenCount = 0;
while (st.hasMoreTokens()) {
iTokenCount++;
string = st.nextToken();
if (iTokenCount == IP) data.m_IP = string;
if (iTokenCount == URL) {
if (bExcludeParams)
if (string.indexOf('?') != -1) string = string.substring(0, string.indexOf('?'));
if (bAnalyzeParams)
if (string.indexOf('?') != -1) string = string.substring(string.indexOf('?') + 1);
data.m_URL = string;
}
if (iTokenCount == PROTOCOL)
if (!string.startsWith("HTTP")) {
data.m_URL += " " + string;
iTokenCount--;
}
if (iTokenCount == BYTES) data.m_iBytes = Integer.parseInt(string);
}
if (!this.filterURL(data.m_URL, strPlus, strMinus, setstrExtensions)) continue;
iCount++;
iBytes += data.m_iBytes;
if (ht.get(data.m_URL) == null) ht.put(data.m_URL, data);
else {
int iThisBytes = data.m_iBytes;
data = (Data) ht.get(data.m_URL);
data.m_iCount++;
data.m_iBytes += iThisBytes;
}
}
Comparator comparator = new Test();
TreeMap tm = new TreeMap(comparator);
Iterator iterator = ht.values().iterator();
while (iterator.hasNext()) {
Data data = (Data) iterator.next();
tm.put(new Integer(data.m_iCount), data);
}
out.println("<file>");
this.printXML(out, "directory", strDirectory);
this.printXML(out, "name", strFilename);
iterator = tm.values().iterator();
while (iterator.hasNext()) {
out.println("<data>");
Data data = (Data) iterator.next();
this.printXML(out, "url", data.m_URL);
this.printXML(out, "count", Integer.toString(data.m_iCount));
out.println("</data>");
}
this.printXML(out, "hits", Integer.toString(iCount));
this.printXML(out, "bytes", Integer.toString(iBytes));
this.printXML(out, "unique", Integer.toString(tm.size()));
iterator = setstrExtensions.iterator();
out.println("<extensions>");
while (iterator.hasNext()) {
this.printXML(out, "extension", (String) iterator.next());
}
out.println("</extensions>");
out.println("</file>");
} catch (FileNotFoundException ex) {
ex.printStackTrace();
} catch (IOException ex) {
ex.printStackTrace();
}
}
/**
* A tracker wants to know if there's a Task to run. Returns a task we'd like the TaskTracker to
* execute right now.
*
* <p>Eventually this function should compute load on the various TaskTrackers, and incorporate
* knowledge of DFS file placement. But for right now, it just grabs a single item out of the
* pending task list and hands it back.
*/
public synchronized Task pollForNewTask(String taskTracker) {
//
// Compute average map and reduce task numbers across pool
//
int avgMaps = 0;
int avgReduces = 0;
int numTaskTrackers;
TaskTrackerStatus tts;
synchronized (taskTrackers) {
numTaskTrackers = taskTrackers.size();
tts = (TaskTrackerStatus) taskTrackers.get(taskTracker);
}
if (numTaskTrackers > 0) {
avgMaps = totalMaps / numTaskTrackers;
avgReduces = totalReduces / numTaskTrackers;
}
int totalCapacity = numTaskTrackers * maxCurrentTasks;
//
// Get map + reduce counts for the current tracker.
//
if (tts == null) {
LOG.warning("Unknown task tracker polling; ignoring: " + taskTracker);
return null;
}
int numMaps = tts.countMapTasks();
int numReduces = tts.countReduceTasks();
//
// In the below steps, we allocate first a map task (if appropriate),
// and then a reduce task if appropriate. We go through all jobs
// in order of job arrival; jobs only get serviced if their
// predecessors are serviced, too.
//
//
// We hand a task to the current taskTracker if the given machine
// has a workload that's equal to or less than the averageMaps
// +/- TASK_ALLOC_EPSILON. (That epsilon is in place in case
// there is an odd machine that is failing for some reason but
// has not yet been removed from the pool, making capacity seem
// larger than it really is.)
//
synchronized (jobsByArrival) {
if ((numMaps < maxCurrentTasks) && (numMaps <= (avgMaps + TASK_ALLOC_EPSILON))) {
int totalNeededMaps = 0;
for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
JobInProgress job = (JobInProgress) it.next();
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = job.obtainNewMapTask(taskTracker, tts);
if (t != null) {
return t;
}
//
// Beyond the highest-priority task, reserve a little
// room for failures and speculative executions; don't
// schedule tasks to the hilt.
//
totalNeededMaps += job.desiredMaps();
double padding = 0;
if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
padding = Math.min(maxCurrentTasks, totalNeededMaps * PAD_FRACTION);
}
if (totalNeededMaps + padding >= totalCapacity) {
break;
}
}
}
//
// Same thing, but for reduce tasks
//
if ((numReduces < maxCurrentTasks) && (numReduces <= (avgReduces + TASK_ALLOC_EPSILON))) {
int totalNeededReduces = 0;
for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
JobInProgress job = (JobInProgress) it.next();
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = job.obtainNewReduceTask(taskTracker, tts);
if (t != null) {
return t;
}
//
// Beyond the highest-priority task, reserve a little
// room for failures and speculative executions; don't
// schedule tasks to the hilt.
//
totalNeededReduces += job.desiredReduces();
double padding = 0;
if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
padding = Math.min(maxCurrentTasks, totalNeededReduces * PAD_FRACTION);
}
if (totalNeededReduces + padding >= totalCapacity) {
break;
}
}
}
}
return null;
}
public Collection taskTrackers() {
synchronized (taskTrackers) {
return taskTrackers.values();
}
}
public TaskTrackerStatus getTaskTracker(String trackerID) {
synchronized (taskTrackers) {
return (TaskTrackerStatus) taskTrackers.get(trackerID);
}
}