public void getTaskDiagnostics(String job) {
try {
String errinfo[];
StringBuilder sb = new StringBuilder();
JobID jobid = JobID.forName(job);
JobStatus status = client.getJobStatus(jobid);
System.out.println(status.getTrackingUrl());
System.out.println(status.getJobFile());
System.out.println(status.getHistoryFile());
System.out.println(status.getQueue());
System.out.println(status.toString());
System.out.println("#### QueueAclsInfo ####");
QueueAclsInfo[] qais = client.getQueueAclsForCurrentUser();
for (int i = 0; i < qais.length; i++) {
QueueAclsInfo qai = qais[i];
System.out.println(qai.getQueueName().toString());
String[] ops = qai.getOperations();
for (int j = 0; j < ops.length; j++) {
System.out.println(ops[j]);
}
}
System.out.println("#### QueueInfo ####");
QueueInfo[] qis = client.getRootQueues();
for (int i = 0; i < qis.length; i++) {
QueueInfo qi = qis[i];
System.out.println(qi.getQueueName());
System.out.println(qi.getSchedulingInfo());
System.out.println(qi.getState());
System.out.println(qi.getProperties());
}
int mapnum = client.getTaskReports(jobid, TaskType.MAP).length;
int rednum = client.getTaskReports(jobid, TaskType.REDUCE).length;
float progress = status.getMapProgress();
sb.append(
String.format(
"m %4d %6s",
mapnum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100)));
sb.append("|");
progress = status.getReduceProgress();
sb.append(
String.format(
"r %4d %6s",
rednum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100)));
System.out.println(sb.toString());
printTaskAttempt(client.getTaskReports(jobid, TaskType.MAP));
printTaskAttempt(client.getTaskReports(jobid, TaskType.REDUCE));
System.out.println(sb.toString());
} catch (Exception e) {
System.out.println(e);
}
}
@Test
public void testJobReportFromHistoryServer() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenReturn(getJobReportResponseFromHistoryServer());
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());
Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());
Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
}
@Test
public void testAMAccessDisabled() throws IOException {
// test only applicable when AM not reachable
if (isAMReachableFromClient) {
return;
}
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenReturn(getJobReportResponseFromHistoryServer());
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getFinishedApplicationReport());
} catch (YarnException e) {
throw new IOException(e);
}
ClientServiceDelegate clientServiceDelegate =
spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("N/A", jobStatus.getJobName());
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
// Should not reach AM even for second and third times too.
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("N/A", jobStatus.getJobName());
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("N/A", jobStatus.getJobName());
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
// The third time around, app is completed, so should go to JHS
JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus1);
Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile());
Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl());
Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f);
Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f);
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
}
@Test
public void testHistoryServerNotConfigured() throws Exception {
// RM doesn't have app report and job History Server is not configured
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(null, getRMDelegate());
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertEquals("N/A", jobStatus.getUsername());
Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState());
// RM has app report and job History Server is not configured
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
ApplicationReport applicationReport = getFinishedApplicationReport();
when(rm.getApplicationReport(jobId.getAppId())).thenReturn(applicationReport);
clientServiceDelegate = getClientServiceDelegate(null, rm);
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername());
Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
}
/**
* Set the priority of the job, defaulting to NORMAL.
*
* @param jp new job priority
*/
public synchronized void setJobPriority(JobPriority jp) {
super.setPriority(org.apache.hadoop.mapreduce.JobPriority.valueOf(jp.name()));
}
protected synchronized void setJobACLs(Map<JobACL, AccessControlList> acls) {
super.setJobACLs(acls);
}
/**
* Used to set the scheduling information associated to a particular Job.
*
* @param schedulingInfo Scheduling information of the job
*/
protected synchronized void setSchedulingInfo(String schedulingInfo) {
super.setSchedulingInfo(schedulingInfo);
}
/**
* Sets the setup progress of this job
*
* @param p The value of setup progress to set to
*/
protected synchronized void setSetupProgress(float p) {
super.setSetupProgress(p);
}
/**
* Set the start time of the job
*
* @param startTime The startTime of the job
*/
protected synchronized void setStartTime(long startTime) {
super.setStartTime(startTime);
}
/** Change the current run state of the job. */
protected synchronized void setRunState(int state) {
super.setState(getEnum(state));
}
/** Set the job retire flag to true. */
protected synchronized void setRetired() {
super.setRetired();
}
/** Set the link to the web-ui for details of the job. */
protected synchronized void setTrackingUrl(String trackingUrl) {
super.setTrackingUrl(trackingUrl);
}
/** Set the job history file url for a completed job */
protected synchronized void setHistoryFile(String historyFile) {
super.setHistoryFile(historyFile);
}
/**
* Set the finish time of the job
*
* @param finishTime The finishTime of the job
*/
protected synchronized void setFinishTime(long finishTime) {
super.setFinishTime(finishTime);
}
/**
* Sets the reduce progress of this Job
*
* @param p The value of reduce progress to set to
*/
protected synchronized void setReduceProgress(float p) {
super.setReduceProgress(p);
}
@Test
public void testReconnectOnAMRestart() throws IOException {
// test not applicable when AM not reachable
// as instantiateAMProxy is not called at all
if (!isAMReachableFromClient) {
return;
}
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
// RM returns AM1 url, null, null and AM2 url on invocations.
// Nulls simulate the time when AM2 is in the process of restarting.
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getRunningApplicationReport(null, 0))
.thenReturn(getRunningApplicationReport(null, 0))
.thenReturn(getRunningApplicationReport("am2", 90));
} catch (YarnException e) {
throw new IOException(e);
}
GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
when(jobReportResponse1.getJobReport())
.thenReturn(
MRBuilderUtils.newJobReport(
jobId,
"jobName-firstGen",
"user",
JobState.RUNNING,
0,
0,
0,
0,
0,
0,
0,
"anything",
null,
false,
""));
// First AM returns a report with jobName firstGen and simulates AM shutdown
// on second invocation.
MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
.thenReturn(jobReportResponse1)
.thenThrow(new RuntimeException("AM is down!"));
GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
when(jobReportResponse2.getJobReport())
.thenReturn(
MRBuilderUtils.newJobReport(
jobId,
"jobName-secondGen",
"user",
JobState.RUNNING,
0,
0,
0,
0,
0,
0,
0,
"anything",
null,
false,
""));
// Second AM generation returns a report with jobName secondGen
MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
.thenReturn(jobReportResponse2);
ClientServiceDelegate clientServiceDelegate =
spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
// First time, connection should be to AM1, then to AM2. Further requests
// should use the same proxy to AM2 and so instantiateProxy shouldn't be
// called.
doReturn(firstGenAMProxy)
.doReturn(secondGenAMProxy)
.when(clientServiceDelegate)
.instantiateAMProxy(any(InetSocketAddress.class));
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class));
}
/** @param userName The username of the job */
protected synchronized void setUsername(String userName) {
super.setUsername(userName);
}
public static JobStatus downgrade(org.apache.hadoop.mapreduce.JobStatus stat) {
JobStatus old =
new JobStatus(
JobID.downgrade(stat.getJobID()),
stat.getSetupProgress(),
stat.getMapProgress(),
stat.getReduceProgress(),
stat.getCleanupProgress(),
stat.getState().getValue(),
JobPriority.valueOf(stat.getPriority().name()),
stat.getUsername(),
stat.getJobName(),
stat.getJobFile(),
stat.getTrackingUrl());
old.setStartTime(stat.getStartTime());
old.setFinishTime(stat.getFinishTime());
old.setSchedulingInfo(stat.getSchedulingInfo());
old.setHistoryFile(stat.getHistoryFile());
return old;
}
