@Test public void testAMAccessDisabled() throws IOException { // test only applicable when AM not reachable if (isAMReachableFromClient) { return; } MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(getJobReportRequest())) .thenReturn(getJobReportResponseFromHistoryServer()); ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class); try { when(rmDelegate.getApplicationReport(jobId.getAppId())) .thenReturn(getRunningApplicationReport("am1", 78)) .thenReturn(getRunningApplicationReport("am1", 78)) .thenReturn(getRunningApplicationReport("am1", 78)) .thenReturn(getFinishedApplicationReport()); } catch (YarnException e) { throw new IOException(e); } ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(historyServerProxy, rmDelegate)); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("N/A", jobStatus.getJobName()); verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class)); // Should not reach AM even for second and third times too. jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("N/A", jobStatus.getJobName()); verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class)); jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("N/A", jobStatus.getJobName()); verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class)); // The third time around, app is completed, so should go to JHS JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus1); Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile()); Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl()); Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f); Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f); verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class)); }
private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries) throws IOException { conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient); MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class); try { when(rmDelegate.getApplicationReport(jobId.getAppId())) .thenThrow( new java.lang.reflect.UndeclaredThrowableException( new IOException("Connection refuced1"))) .thenThrow( new java.lang.reflect.UndeclaredThrowableException( new IOException("Connection refuced2"))) .thenThrow( new java.lang.reflect.UndeclaredThrowableException( new IOException("Connection refuced3"))); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy); try { clientServiceDelegate.getJobStatus(oldJobId); Assert.fail("It should throw exception after retries"); } catch (IOException e) { System.out.println("fail to get job status,and e=" + e.toString()); } verify(rmDelegate, times(noOfRetries)).getApplicationReport(any(ApplicationId.class)); } catch (YarnException e) { throw new IOException(e); } }
@Test public void testRMDownRestoreForJobStatusBeforeGetAMReport() throws IOException { Configuration conf = new YarnConfiguration(); conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3); conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient); MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(any(GetJobReportRequest.class))) .thenReturn(getJobReportResponse()); ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class); try { when(rmDelegate.getApplicationReport(jobId.getAppId())) .thenThrow( new java.lang.reflect.UndeclaredThrowableException( new IOException("Connection refuced1"))) .thenThrow( new java.lang.reflect.UndeclaredThrowableException( new IOException("Connection refuced2"))) .thenReturn(getFinishedApplicationReport()); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); verify(rmDelegate, times(3)).getApplicationReport(any(ApplicationId.class)); Assert.assertNotNull(jobStatus); } catch (YarnException e) { throw new IOException(e); } }
@Test public void testHistoryServerNotConfigured() throws Exception { // RM doesn't have app report and job History Server is not configured ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(null, getRMDelegate()); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertEquals("N/A", jobStatus.getUsername()); Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState()); // RM has app report and job History Server is not configured ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); ApplicationReport applicationReport = getFinishedApplicationReport(); when(rm.getApplicationReport(jobId.getAppId())).thenReturn(applicationReport); clientServiceDelegate = getClientServiceDelegate(null, rm); jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername()); Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState()); }
@Test public void testUnknownAppInRM() throws Exception { MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(getJobReportResponse()); ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, getRMDelegate()); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); }
@Test(timeout = 20000) public void testJobKill() throws Exception { clientDelegate = mock(ClientServiceDelegate.class); when(clientDelegate.getJobStatus(any(JobID.class))) .thenReturn( new org.apache.hadoop.mapreduce.JobStatus( jobId, 0f, 0f, 0f, 0f, State.PREP, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp")); when(clientDelegate.killJob(any(JobID.class))).thenReturn(true); doAnswer( new Answer<ClientServiceDelegate>() { @Override public ClientServiceDelegate answer(InvocationOnMock invocation) throws Throwable { return clientDelegate; } }) .when(clientCache) .getClient(any(JobID.class)); yarnRunner.killJob(jobId); verify(resourceMgrDelegate).killApplication(appId); when(clientDelegate.getJobStatus(any(JobID.class))) .thenReturn( new org.apache.hadoop.mapreduce.JobStatus( jobId, 0f, 0f, 0f, 0f, State.RUNNING, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp")); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); }
@Test public void testJobReportFromHistoryServer() throws Exception { MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(getJobReportRequest())) .thenReturn(getJobReportResponseFromHistoryServer()); ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null); ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile()); Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl()); Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f); Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f); }
@Test public void testRetriesOnConnectionFailure() throws Exception { MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(getJobReportRequest())) .thenThrow(new RuntimeException("1")) .thenThrow(new RuntimeException("2")) .thenReturn(getJobReportResponse()); ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null); ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); verify(historyServerProxy, times(3)).getJobReport(any(GetJobReportRequest.class)); }
@Test public void testRemoteExceptionFromHistoryServer() throws Exception { MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(getJobReportRequest())) .thenThrow(new IOException("Job ID doesnot Exist")); ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null); ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm); try { clientServiceDelegate.getJobStatus(oldJobId); Assert.fail("Invoke should throw exception after retries."); } catch (IOException e) { Assert.assertTrue(e.getMessage().contains("Job ID doesnot Exist")); } }
@Test public void testRetriesOnAMConnectionFailures() throws Exception { if (!isAMReachableFromClient) { return; } ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())) .thenReturn(getRunningApplicationReport("am1", 78)); // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and // succeed in the 5th call. final MRClientProtocol amProxy = mock(MRClientProtocol.class); when(amProxy.getJobReport(any(GetJobReportRequest.class))) .thenThrow(new RuntimeException("11")) .thenThrow(new RuntimeException("22")) .thenThrow(new RuntimeException("33")) .thenThrow(new RuntimeException("44")) .thenReturn(getJobReportResponse()); Configuration conf = new YarnConfiguration(); conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rm, oldJobId, null) { @Override MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr) throws IOException { super.instantiateAMProxy(serviceAddr); return amProxy; } }; JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); // assert maxClientRetry is not decremented. Assert.assertEquals( conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES), clientServiceDelegate.getMaxClientRetry()); verify(amProxy, times(5)).getJobReport(any(GetJobReportRequest.class)); }
@Test public void testReconnectOnAMRestart() throws IOException { // test not applicable when AM not reachable // as instantiateAMProxy is not called at all if (!isAMReachableFromClient) { return; } MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); // RM returns AM1 url, null, null and AM2 url on invocations. // Nulls simulate the time when AM2 is in the process of restarting. ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class); try { when(rmDelegate.getApplicationReport(jobId.getAppId())) .thenReturn(getRunningApplicationReport("am1", 78)) .thenReturn(getRunningApplicationReport(null, 0)) .thenReturn(getRunningApplicationReport(null, 0)) .thenReturn(getRunningApplicationReport("am2", 90)); } catch (YarnException e) { throw new IOException(e); } GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class); when(jobReportResponse1.getJobReport()) .thenReturn( MRBuilderUtils.newJobReport( jobId, "jobName-firstGen", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null, false, "")); // First AM returns a report with jobName firstGen and simulates AM shutdown // on second invocation. MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class); when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class))) .thenReturn(jobReportResponse1) .thenThrow(new RuntimeException("AM is down!")); GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class); when(jobReportResponse2.getJobReport()) .thenReturn( MRBuilderUtils.newJobReport( jobId, "jobName-secondGen", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null, false, "")); // Second AM generation returns a report with jobName secondGen MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class); when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class))) .thenReturn(jobReportResponse2); ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(historyServerProxy, rmDelegate)); // First time, connection should be to AM1, then to AM2. Further requests // should use the same proxy to AM2 and so instantiateProxy shouldn't be // called. doReturn(firstGenAMProxy) .doReturn(secondGenAMProxy) .when(clientServiceDelegate) .instantiateAMProxy(any(InetSocketAddress.class)); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("jobName-firstGen", jobStatus.getJobName()); jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("jobName-secondGen", jobStatus.getJobName()); jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("jobName-secondGen", jobStatus.getJobName()); verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class)); }