private String getHistoryFile(Configuration conf, String jobId) throws IOException {
String jtAddress = "scheme://" + conf.get("mapred.job.tracker");
String jtHttpAddr = "scheme://" + conf.get("mapred.job.tracker.http.address");
try {
String host = new URI(jtAddress).getHost();
int port = new URI(jtHttpAddr).getPort();
HttpClient client = new HttpClient();
String jobUrl = "http://" + host + ":" + port + "/jobdetails.jsp";
GetMethod get = new GetMethod(jobUrl);
get.setQueryString("jobid=" + jobId);
get.setFollowRedirects(false);
int status = client.executeMethod(get);
String file = null;
if (status == HttpStatus.SC_MOVED_PERMANENTLY || status == HttpStatus.SC_MOVED_TEMPORARILY) {
file = get.getResponseHeader("Location").toString();
file = file.substring(file.lastIndexOf('=') + 1);
file = JobHistory.JobInfo.decodeJobHistoryFileName(file);
} else {
LOG.warn("JobURL {} for id: {} returned {}", jobUrl, jobId, status);
}
return file;
} catch (URISyntaxException e) {
throw new IOException("JT Address: " + jtAddress + ", http Address: " + jtHttpAddr, e);
}
}
@Override
public String getFromHistory(String jobId) throws IOException {
Configuration conf = getConf();
String file = getHistoryFile(conf, jobId);
if (file == null) return null;
JobHistory.JobInfo jobInfo = new JobHistory.JobInfo(jobId);
DefaultJobHistoryParser.parseJobTasks(file, jobInfo, new Path(file).getFileSystem(conf));
LOG.info("History file: {}", file);
LOG.debug("Number of tasks in the history file: {}", jobInfo.getAllTasks().size());
for (JobHistory.Task task : jobInfo.getAllTasks().values()) {
if (task.get(JobHistory.Keys.TASK_TYPE).equals(JobHistory.Values.MAP.name())
&& task.get(JobHistory.Keys.TASK_STATUS).equals(JobHistory.Values.SUCCESS.name())) {
for (JobHistory.TaskAttempt attempt : task.getTaskAttempts().values()) {
if (attempt.get(JobHistory.Keys.TASK_STATUS).equals(JobHistory.Values.SUCCESS.name())) {
return JobHistory.getTaskLogsUrl(attempt);
}
}
}
}
LOG.warn("Unable to find successful map task attempt");
return null;
}
public void test(
String workflowId,
String workflowName,
String workflowNodeName,
Map<String, String[]> adjacencies) {
Configuration conf = new Configuration();
setProperties(conf, workflowId, workflowName, workflowNodeName, adjacencies);
String log =
log(
"JOB",
new String[] {ID, NAME, NODE, ADJ},
new String[] {
conf.get(ID_PROP),
conf.get(NAME_PROP),
conf.get(NODE_PROP),
JobHistory.JobInfo.getWorkflowAdjacencies(conf)
});
ParsedLine line = new ParsedLine(log);
JobID jobid = new JobID("id", 1);
JobSubmittedEvent event =
new JobSubmittedEvent(
jobid,
workflowName,
"",
0l,
"",
null,
"",
line.get(ID),
line.get(NAME),
line.get(NODE),
line.get(ADJ));
WorkflowContext context = MapReduceJobHistoryUpdater.buildWorkflowContext(event);
String resultingWorkflowId = workflowId;
if (workflowId.isEmpty()) resultingWorkflowId = jobid.toString().replace("job_", "mr_");
assertEquals("Didn't recover workflowId", resultingWorkflowId, context.getWorkflowId());
assertEquals("Didn't recover workflowName", workflowName, context.getWorkflowName());
assertEquals(
"Didn't recover workflowNodeName", workflowNodeName, context.getWorkflowEntityName());
Map<String, String[]> resultingAdjacencies = adjacencies;
if (resultingAdjacencies.size() == 0) {
resultingAdjacencies = new HashMap<String, String[]>();
resultingAdjacencies.put(workflowNodeName, new String[] {});
}
assertEquals(
"Got incorrect number of adjacencies",
resultingAdjacencies.size(),
context.getWorkflowDag().getEntries().size());
for (WorkflowDagEntry entry : context.getWorkflowDag().getEntries()) {
String[] sTargets = resultingAdjacencies.get(entry.getSource());
assertNotNull("No original targets for " + entry.getSource(), sTargets);
List<String> dTargets = entry.getTargets();
assertEquals(
"Got incorrect number of targets for " + entry.getSource(),
sTargets.length,
dTargets.size());
for (int i = 0; i < sTargets.length; i++) {
assertEquals("Got incorrect target for " + entry.getSource(), sTargets[i], dTargets.get(i));
}
}
}