private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries)
throws IOException {
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced1")))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced2")))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced3")));
ClientServiceDelegate clientServiceDelegate =
new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
try {
clientServiceDelegate.getJobStatus(oldJobId);
Assert.fail("It should throw exception after retries");
} catch (IOException e) {
System.out.println("fail to get job status,and e=" + e.toString());
}
verify(rmDelegate, times(noOfRetries)).getApplicationReport(any(ApplicationId.class));
} catch (YarnException e) {
throw new IOException(e);
}
}
@Test
public void testRMDownRestoreForJobStatusBeforeGetAMReport() throws IOException {
Configuration conf = new YarnConfiguration();
conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
.thenReturn(getJobReportResponse());
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced1")))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced2")))
.thenReturn(getFinishedApplicationReport());
ClientServiceDelegate clientServiceDelegate =
new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
verify(rmDelegate, times(3)).getApplicationReport(any(ApplicationId.class));
Assert.assertNotNull(jobStatus);
} catch (YarnException e) {
throw new IOException(e);
}
}
@Test
public void testUnknownAppInRM() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(getJobReportResponse());
ClientServiceDelegate clientServiceDelegate =
getClientServiceDelegate(historyServerProxy, getRMDelegate());
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
}
@Test
public void testAMAccessDisabled() throws IOException {
// test only applicable when AM not reachable
if (isAMReachableFromClient) {
return;
}
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenReturn(getJobReportResponseFromHistoryServer());
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getFinishedApplicationReport());
} catch (YarnException e) {
throw new IOException(e);
}
ClientServiceDelegate clientServiceDelegate =
spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("N/A", jobStatus.getJobName());
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
// Should not reach AM even for second and third times too.
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("N/A", jobStatus.getJobName());
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("N/A", jobStatus.getJobName());
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
// The third time around, app is completed, so should go to JHS
JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus1);
Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile());
Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl());
Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f);
Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f);
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
}
@Test
public void testCountersFromHistoryServer() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getCounters(getCountersRequest()))
.thenReturn(getCountersResponseFromHistoryServer());
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
Counters counters = TypeConverter.toYarn(clientServiceDelegate.getJobCounters(oldJobId));
Assert.assertNotNull(counters);
Assert.assertEquals(
1001, counters.getCounterGroup("dummyCounters").getCounter("dummyCounter").getValue());
}
@Test
public void testJobReportFromHistoryServer() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenReturn(getJobReportResponseFromHistoryServer());
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());
Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());
Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
}
@Test
public void testHistoryServerNotConfigured() throws Exception {
// RM doesn't have app report and job History Server is not configured
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(null, getRMDelegate());
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertEquals("N/A", jobStatus.getUsername());
Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState());
// RM has app report and job History Server is not configured
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
ApplicationReport applicationReport = getFinishedApplicationReport();
when(rm.getApplicationReport(jobId.getAppId())).thenReturn(applicationReport);
clientServiceDelegate = getClientServiceDelegate(null, rm);
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername());
Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
}
@Test
public void testRetriesOnConnectionFailure() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenThrow(new RuntimeException("1"))
.thenThrow(new RuntimeException("2"))
.thenReturn(getJobReportResponse());
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
verify(historyServerProxy, times(3)).getJobReport(any(GetJobReportRequest.class));
}
@Test
public void testRemoteExceptionFromHistoryServer() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenThrow(new IOException("Job ID doesnot Exist"));
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
try {
clientServiceDelegate.getJobStatus(oldJobId);
Assert.fail("Invoke should throw exception after retries.");
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("Job ID doesnot Exist"));
}
}
@Test
public void testRetriesOnAMConnectionFailures() throws Exception {
if (!isAMReachableFromClient) {
return;
}
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
.thenReturn(getRunningApplicationReport("am1", 78));
// throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
// succeed in the 5th call.
final MRClientProtocol amProxy = mock(MRClientProtocol.class);
when(amProxy.getJobReport(any(GetJobReportRequest.class)))
.thenThrow(new RuntimeException("11"))
.thenThrow(new RuntimeException("22"))
.thenThrow(new RuntimeException("33"))
.thenThrow(new RuntimeException("44"))
.thenReturn(getJobReportResponse());
Configuration conf = new YarnConfiguration();
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate =
new ClientServiceDelegate(conf, rm, oldJobId, null) {
@Override
MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr)
throws IOException {
super.instantiateAMProxy(serviceAddr);
return amProxy;
}
};
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
// assert maxClientRetry is not decremented.
Assert.assertEquals(
conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES),
clientServiceDelegate.getMaxClientRetry());
verify(amProxy, times(5)).getJobReport(any(GetJobReportRequest.class));
}
@Test(timeout = 20000)
public void testJobKill() throws Exception {
clientDelegate = mock(ClientServiceDelegate.class);
when(clientDelegate.getJobStatus(any(JobID.class)))
.thenReturn(
new org.apache.hadoop.mapreduce.JobStatus(
jobId, 0f, 0f, 0f, 0f, State.PREP, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp"));
when(clientDelegate.killJob(any(JobID.class))).thenReturn(true);
doAnswer(
new Answer<ClientServiceDelegate>() {
@Override
public ClientServiceDelegate answer(InvocationOnMock invocation) throws Throwable {
return clientDelegate;
}
})
.when(clientCache)
.getClient(any(JobID.class));
yarnRunner.killJob(jobId);
verify(resourceMgrDelegate).killApplication(appId);
when(clientDelegate.getJobStatus(any(JobID.class)))
.thenReturn(
new org.apache.hadoop.mapreduce.JobStatus(
jobId,
0f,
0f,
0f,
0f,
State.RUNNING,
JobPriority.HIGH,
"tmp",
"tmp",
"tmp",
"tmp"));
yarnRunner.killJob(jobId);
verify(clientDelegate).killJob(jobId);
}
@Test
public void testReconnectOnAMRestart() throws IOException {
// test not applicable when AM not reachable
// as instantiateAMProxy is not called at all
if (!isAMReachableFromClient) {
return;
}
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
// RM returns AM1 url, null, null and AM2 url on invocations.
// Nulls simulate the time when AM2 is in the process of restarting.
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getRunningApplicationReport(null, 0))
.thenReturn(getRunningApplicationReport(null, 0))
.thenReturn(getRunningApplicationReport("am2", 90));
} catch (YarnException e) {
throw new IOException(e);
}
GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
when(jobReportResponse1.getJobReport())
.thenReturn(
MRBuilderUtils.newJobReport(
jobId,
"jobName-firstGen",
"user",
JobState.RUNNING,
0,
0,
0,
0,
0,
0,
0,
"anything",
null,
false,
""));
// First AM returns a report with jobName firstGen and simulates AM shutdown
// on second invocation.
MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
.thenReturn(jobReportResponse1)
.thenThrow(new RuntimeException("AM is down!"));
GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
when(jobReportResponse2.getJobReport())
.thenReturn(
MRBuilderUtils.newJobReport(
jobId,
"jobName-secondGen",
"user",
JobState.RUNNING,
0,
0,
0,
0,
0,
0,
0,
"anything",
null,
false,
""));
// Second AM generation returns a report with jobName secondGen
MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
.thenReturn(jobReportResponse2);
ClientServiceDelegate clientServiceDelegate =
spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
// First time, connection should be to AM1, then to AM2. Further requests
// should use the same proxy to AM2 and so instantiateProxy shouldn't be
// called.
doReturn(firstGenAMProxy)
.doReturn(secondGenAMProxy)
.when(clientServiceDelegate)
.instantiateAMProxy(any(InetSocketAddress.class));
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class));
}