/**
* A tracker wants to know if there's a Task to run. Returns a task we'd like the TaskTracker to
* execute right now.
*
* <p>Eventually this function should compute load on the various TaskTrackers, and incorporate
* knowledge of DFS file placement. But for right now, it just grabs a single item out of the
* pending task list and hands it back.
*/
public synchronized Task pollForNewTask(String taskTracker) {
//
// Compute average map and reduce task numbers across pool
//
int avgMaps = 0;
int avgReduces = 0;
int numTaskTrackers;
TaskTrackerStatus tts;
synchronized (taskTrackers) {
numTaskTrackers = taskTrackers.size();
tts = (TaskTrackerStatus) taskTrackers.get(taskTracker);
}
if (numTaskTrackers > 0) {
avgMaps = totalMaps / numTaskTrackers;
avgReduces = totalReduces / numTaskTrackers;
}
int totalCapacity = numTaskTrackers * maxCurrentTasks;
//
// Get map + reduce counts for the current tracker.
//
if (tts == null) {
LOG.warning("Unknown task tracker polling; ignoring: " + taskTracker);
return null;
}
int numMaps = tts.countMapTasks();
int numReduces = tts.countReduceTasks();
//
// In the below steps, we allocate first a map task (if appropriate),
// and then a reduce task if appropriate. We go through all jobs
// in order of job arrival; jobs only get serviced if their
// predecessors are serviced, too.
//
//
// We hand a task to the current taskTracker if the given machine
// has a workload that's equal to or less than the averageMaps
// +/- TASK_ALLOC_EPSILON. (That epsilon is in place in case
// there is an odd machine that is failing for some reason but
// has not yet been removed from the pool, making capacity seem
// larger than it really is.)
//
synchronized (jobsByArrival) {
if ((numMaps < maxCurrentTasks) && (numMaps <= (avgMaps + TASK_ALLOC_EPSILON))) {
int totalNeededMaps = 0;
for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
JobInProgress job = (JobInProgress) it.next();
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = job.obtainNewMapTask(taskTracker, tts);
if (t != null) {
return t;
}
//
// Beyond the highest-priority task, reserve a little
// room for failures and speculative executions; don't
// schedule tasks to the hilt.
//
totalNeededMaps += job.desiredMaps();
double padding = 0;
if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
padding = Math.min(maxCurrentTasks, totalNeededMaps * PAD_FRACTION);
}
if (totalNeededMaps + padding >= totalCapacity) {
break;
}
}
}
//
// Same thing, but for reduce tasks
//
if ((numReduces < maxCurrentTasks) && (numReduces <= (avgReduces + TASK_ALLOC_EPSILON))) {
int totalNeededReduces = 0;
for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
JobInProgress job = (JobInProgress) it.next();
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = job.obtainNewReduceTask(taskTracker, tts);
if (t != null) {
return t;
}
//
// Beyond the highest-priority task, reserve a little
// room for failures and speculative executions; don't
// schedule tasks to the hilt.
//
totalNeededReduces += job.desiredReduces();
double padding = 0;
if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
padding = Math.min(maxCurrentTasks, totalNeededReduces * PAD_FRACTION);
}
if (totalNeededReduces + padding >= totalCapacity) {
break;
}
}
}
}
return null;
}
public synchronized ClusterStatus getClusterStatus() {
synchronized (taskTrackers) {
return new ClusterStatus(taskTrackers.size(), totalMaps, totalReduces, maxCurrentTasks);
}
}
/**
* Process an HTML get or post.
*
* @exception ServletException From inherited class.
* @exception IOException From inherited class.
*/
public void scanOutXML(
PrintWriter out,
String strDirectory,
String strFilename,
String[] strPlus,
String[] strMinus,
boolean bExcludeParams,
boolean bAnalyzeParams)
throws IOException {
File dir = new File(strDirectory + '/' + strFilename);
if (dir.isDirectory()) return;
try {
FileReader is = new FileReader(strDirectory + '/' + strFilename);
BufferedReader r = new BufferedReader(is);
String string = null;
Hashtable ht = new Hashtable();
Set setstrExtensions = new HashSet();
int iCount = 0;
int iBytes = 0;
while ((string = r.readLine()) != null) {
StringTokenizer st = new StringTokenizer(string, " \"", false);
Data data = new Data();
int iTokenCount = 0;
while (st.hasMoreTokens()) {
iTokenCount++;
string = st.nextToken();
if (iTokenCount == IP) data.m_IP = string;
if (iTokenCount == URL) {
if (bExcludeParams)
if (string.indexOf('?') != -1) string = string.substring(0, string.indexOf('?'));
if (bAnalyzeParams)
if (string.indexOf('?') != -1) string = string.substring(string.indexOf('?') + 1);
data.m_URL = string;
}
if (iTokenCount == PROTOCOL)
if (!string.startsWith("HTTP")) {
data.m_URL += " " + string;
iTokenCount--;
}
if (iTokenCount == BYTES) data.m_iBytes = Integer.parseInt(string);
}
if (!this.filterURL(data.m_URL, strPlus, strMinus, setstrExtensions)) continue;
iCount++;
iBytes += data.m_iBytes;
if (ht.get(data.m_URL) == null) ht.put(data.m_URL, data);
else {
int iThisBytes = data.m_iBytes;
data = (Data) ht.get(data.m_URL);
data.m_iCount++;
data.m_iBytes += iThisBytes;
}
}
Comparator comparator = new Test();
TreeMap tm = new TreeMap(comparator);
Iterator iterator = ht.values().iterator();
while (iterator.hasNext()) {
Data data = (Data) iterator.next();
tm.put(new Integer(data.m_iCount), data);
}
out.println("<file>");
this.printXML(out, "directory", strDirectory);
this.printXML(out, "name", strFilename);
iterator = tm.values().iterator();
while (iterator.hasNext()) {
out.println("<data>");
Data data = (Data) iterator.next();
this.printXML(out, "url", data.m_URL);
this.printXML(out, "count", Integer.toString(data.m_iCount));
out.println("</data>");
}
this.printXML(out, "hits", Integer.toString(iCount));
this.printXML(out, "bytes", Integer.toString(iBytes));
this.printXML(out, "unique", Integer.toString(tm.size()));
iterator = setstrExtensions.iterator();
out.println("<extensions>");
while (iterator.hasNext()) {
this.printXML(out, "extension", (String) iterator.next());
}
out.println("</extensions>");
out.println("</file>");
} catch (FileNotFoundException ex) {
ex.printStackTrace();
} catch (IOException ex) {
ex.printStackTrace();
}
}
