public void verifyTaskGeneric(
Task task,
String id,
String state,
String type,
String successfulAttempt,
long startTime,
long finishTime,
long elapsedTime,
float progress) {
TaskId taskid = task.getID();
String tid = MRApps.toString(taskid);
TaskReport report = task.getReport();
WebServicesTestUtils.checkStringMatch("id", tid, id);
WebServicesTestUtils.checkStringMatch("type", task.getType().toString(), type);
WebServicesTestUtils.checkStringMatch("state", report.getTaskState().toString(), state);
// not easily checked without duplicating logic, just make sure its here
assertNotNull("successfulAttempt null", successfulAttempt);
assertEquals("startTime wrong", report.getStartTime(), startTime);
assertEquals("finishTime wrong", report.getFinishTime(), finishTime);
assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f);
}
protected DataStatistics dataStatisticsForTask(TaskId taskID) {
JobId jobID = taskID.getJobId();
Job job = context.getJob(jobID);
if (job == null) {
return null;
}
Task task = job.getTask(taskID);
if (task == null) {
return null;
}
return task.getType() == TaskType.MAP
? mapperStatistics.get(job)
: task.getType() == TaskType.REDUCE ? reducerStatistics.get(job) : null;
}
@Override protected void render(Block html) {
if (app.getJob() == null) {
html.
h2($(TITLE));
return;
}
TaskType type = null;
String symbol = $(TASK_TYPE);
if (!symbol.isEmpty()) {
type = MRApps.taskType(symbol);
}
TBODY<TABLE<Hamlet>> tbody = html.
table("#tasks").
thead().
tr().
th("Task").
th("Progress").
th("State").
th("Start Time").
th("Finish Time").
th("Elapsed Time")._()._().
tbody();
StringBuilder tasksTableData = new StringBuilder("[\n");
for (Task task : app.getJob().getTasks().values()) {
if (type != null && task.getType() != type) {
continue;
}
TaskInfo info = new TaskInfo(task);
String tid = info.getId();
String pct = percent(info.getProgress() / 100);
tasksTableData.append("[\"<a href='").append(url("task", tid))
.append("'>").append(tid).append("</a>\",\"")
//Progress bar
.append("<br title='").append(pct)
.append("'> <div class='").append(C_PROGRESSBAR).append("' title='")
.append(join(pct, '%')).append("'> ").append("<div class='")
.append(C_PROGRESSBAR_VALUE).append("' style='")
.append(join("width:", pct, '%')).append("'> </div> </div>\",\"")
.append(info.getState()).append("\",\"")
.append(info.getStartTime()).append("\",\"")
.append(info.getFinishTime()).append("\",\"")
.append(info.getElapsedTime()).append("\"],\n");
}
//Remove the last comma and close off the array of arrays
if(tasksTableData.charAt(tasksTableData.length() - 2) == ',') {
tasksTableData.delete(tasksTableData.length()-2, tasksTableData.length()-1);
}
tasksTableData.append("]");
html.script().$type("text/javascript").
_("var tasksTableData=" + tasksTableData)._();
tbody._()._();
}
public void verifyHsTask(JSONArray arr, Job job, String type) throws JSONException {
for (Task task : job.getTasks().values()) {
TaskId id = task.getID();
String tid = MRApps.toString(id);
Boolean found = false;
if (type != null && task.getType() == MRApps.taskType(type)) {
for (int i = 0; i < arr.length(); i++) {
JSONObject info = arr.getJSONObject(i);
if (tid.matches(info.getString("id"))) {
found = true;
verifyHsSingleTask(info, task);
}
}
assertTrue("task with id: " + tid + " not in web service output", found);
}
}
}
/**
* Go through a job and update the member variables with counts for information to output in the
* page.
*
* @param job the job to get counts for.
*/
private void countTasksAndAttempts(Job job) {
final Map<TaskId, Task> tasks = job.getTasks();
if (tasks == null) {
return;
}
for (Task task : tasks.values()) {
switch (task.getType()) {
case MAP:
// Task counts
switch (task.getState()) {
case RUNNING:
++this.mapsRunning;
break;
case SCHEDULED:
++this.mapsPending;
break;
default:
break;
}
break;
case REDUCE:
// Task counts
switch (task.getState()) {
case RUNNING:
++this.reducesRunning;
break;
case SCHEDULED:
++this.reducesPending;
break;
default:
break;
}
break;
default:
throw new IllegalStateException("Task type is neither map nor reduce: " + task.getType());
}
// Attempts counts
Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
int newAttempts, running, successful, failed, killed;
for (TaskAttempt attempt : attempts.values()) {
newAttempts = 0;
running = 0;
successful = 0;
failed = 0;
killed = 0;
if (TaskAttemptStateUI.NEW.correspondsTo(attempt.getState())) {
++newAttempts;
} else if (TaskAttemptStateUI.RUNNING.correspondsTo(attempt.getState())) {
++running;
} else if (TaskAttemptStateUI.SUCCESSFUL.correspondsTo(attempt.getState())) {
++successful;
} else if (TaskAttemptStateUI.FAILED.correspondsTo(attempt.getState())) {
++failed;
} else if (TaskAttemptStateUI.KILLED.correspondsTo(attempt.getState())) {
++killed;
}
switch (task.getType()) {
case MAP:
this.newMapAttempts += newAttempts;
this.runningMapAttempts += running;
this.successfulMapAttempts += successful;
this.failedMapAttempts += failed;
this.killedMapAttempts += killed;
break;
case REDUCE:
this.newReduceAttempts += newAttempts;
this.runningReduceAttempts += running;
this.successfulReduceAttempts += successful;
this.failedReduceAttempts += failed;
this.killedReduceAttempts += killed;
break;
default:
throw new IllegalStateException("Task type neither map nor reduce: " + task.getType());
}
}
}
}
/*
* (non-Javadoc)
* @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block)
*/
@Override protected void render(Block html) {
if (app.getJob() == null) {
html.
h2($(TITLE));
return;
}
TaskType type = null;
String symbol = $(TASK_TYPE);
if (!symbol.isEmpty()) {
type = MRApps.taskType(symbol);
}
THEAD<TABLE<Hamlet>> thead;
if(type != null)
thead = html.table("#"+app.getJob().getID()
+ type).$class("dt-tasks").thead();
else
thead = html.table("#tasks").thead();
//Create the spanning row
int attemptColSpan = type == TaskType.REDUCE ? 8 : 3;
thead.tr().
th().$colspan(5).$class("ui-state-default")._("Task")._().
th().$colspan(attemptColSpan).$class("ui-state-default").
_("Successful Attempt")._().
_();
TR<THEAD<TABLE<Hamlet>>> theadRow = thead.
tr().
th("Name").
th("State").
th("Start Time").
th("Finish Time").
th("Elapsed Time").
th("Start Time"); //Attempt
if(type == TaskType.REDUCE) {
theadRow.th("Shuffle Finish Time"); //Attempt
theadRow.th("Merge Finish Time"); //Attempt
}
theadRow.th("Finish Time"); //Attempt
if(type == TaskType.REDUCE) {
theadRow.th("Elapsed Time Shuffle"); //Attempt
theadRow.th("Elapsed Time Merge"); //Attempt
theadRow.th("Elapsed Time Reduce"); //Attempt
}
theadRow.th("Elapsed Time"); //Attempt
TBODY<TABLE<Hamlet>> tbody = theadRow._()._().tbody();
// Write all the data into a JavaScript array of arrays for JQuery
// DataTables to display
StringBuilder tasksTableData = new StringBuilder("[\n");
for (Task task : app.getJob().getTasks().values()) {
if (type != null && task.getType() != type) {
continue;
}
TaskInfo info = new TaskInfo(task);
String tid = info.getId();
long startTime = info.getStartTime();
long finishTime = info.getFinishTime();
long elapsed = info.getElapsedTime();
long attemptStartTime = -1;
long shuffleFinishTime = -1;
long sortFinishTime = -1;
long attemptFinishTime = -1;
long elapsedShuffleTime = -1;
long elapsedSortTime = -1;;
long elapsedReduceTime = -1;
long attemptElapsed = -1;
TaskAttempt successful = info.getSuccessful();
if(successful != null) {
TaskAttemptInfo ta;
if(type == TaskType.REDUCE) {
ReduceTaskAttemptInfo rta = new ReduceTaskAttemptInfo(successful, type);
shuffleFinishTime = rta.getShuffleFinishTime();
sortFinishTime = rta.getMergeFinishTime();
elapsedShuffleTime = rta.getElapsedShuffleTime();
elapsedSortTime = rta.getElapsedMergeTime();
elapsedReduceTime = rta.getElapsedReduceTime();
ta = rta;
} else {
ta = new TaskAttemptInfo(successful, type, false);
}
attemptStartTime = ta.getStartTime();
attemptFinishTime = ta.getFinishTime();
attemptElapsed = ta.getElapsedTime();
}
tasksTableData.append("[\"")
.append("<a href='" + url("task", tid)).append("'>")
.append(tid).append("</a>\",\"")
.append(info.getState()).append("\",\"")
.append(startTime).append("\",\"")
.append(finishTime).append("\",\"")
.append(elapsed).append("\",\"")
.append(attemptStartTime).append("\",\"");
if(type == TaskType.REDUCE) {
tasksTableData.append(shuffleFinishTime).append("\",\"")
.append(sortFinishTime).append("\",\"");
}
tasksTableData.append(attemptFinishTime).append("\",\"");
if(type == TaskType.REDUCE) {
tasksTableData.append(elapsedShuffleTime).append("\",\"")
.append(elapsedSortTime).append("\",\"")
.append(elapsedReduceTime).append("\",\"");
}
tasksTableData.append(attemptElapsed).append("\"],\n");
}
//Remove the last comma and close off the array of arrays
if(tasksTableData.charAt(tasksTableData.length() - 2) == ',') {
tasksTableData.delete(
tasksTableData.length()-2, tasksTableData.length()-1);
}
tasksTableData.append("]");
html.script().$type("text/javascript").
_("var tasksTableData=" + tasksTableData)._();
TR<TFOOT<TABLE<Hamlet>>> footRow = tbody._().tfoot().tr();
footRow.th().input("search_init").$type(InputType.text).$name("task")
.$value("ID")._()._().th().input("search_init").$type(InputType.text)
.$name("state").$value("State")._()._().th().input("search_init")
.$type(InputType.text).$name("start_time").$value("Start Time")._()._()
.th().input("search_init").$type(InputType.text).$name("finish_time")
.$value("Finish Time")._()._().th().input("search_init")
.$type(InputType.text).$name("elapsed_time").$value("Elapsed Time")._()
._().th().input("search_init").$type(InputType.text)
.$name("attempt_start_time").$value("Start Time")._()._();
if(type == TaskType.REDUCE) {
footRow.th().input("search_init").$type(InputType.text)
.$name("shuffle_time").$value("Shuffle Time")._()._();
footRow.th().input("search_init").$type(InputType.text)
.$name("merge_time").$value("Merge Time")._()._();
}
footRow.th().input("search_init").$type(InputType.text)
.$name("attempt_finish").$value("Finish Time")._()._();
if(type == TaskType.REDUCE) {
footRow.th().input("search_init").$type(InputType.text)
.$name("elapsed_shuffle_time").$value("Elapsed Shuffle Time")._()._();
footRow.th().input("search_init").$type(InputType.text)
.$name("elapsed_merge_time").$value("Elapsed Merge Time")._()._();
footRow.th().input("search_init").$type(InputType.text)
.$name("elapsed_reduce_time").$value("Elapsed Reduce Time")._()._();
}
footRow.th().input("search_init").$type(InputType.text)
.$name("attempt_elapsed").$value("Elapsed Time")._()._();
footRow._()._()._();
}