public void getTaskDiagnostics(String job) { try { String errinfo[]; StringBuilder sb = new StringBuilder(); JobID jobid = JobID.forName(job); JobStatus status = client.getJobStatus(jobid); System.out.println(status.getTrackingUrl()); System.out.println(status.getJobFile()); System.out.println(status.getHistoryFile()); System.out.println(status.getQueue()); System.out.println(status.toString()); System.out.println("#### QueueAclsInfo ####"); QueueAclsInfo[] qais = client.getQueueAclsForCurrentUser(); for (int i = 0; i < qais.length; i++) { QueueAclsInfo qai = qais[i]; System.out.println(qai.getQueueName().toString()); String[] ops = qai.getOperations(); for (int j = 0; j < ops.length; j++) { System.out.println(ops[j]); } } System.out.println("#### QueueInfo ####"); QueueInfo[] qis = client.getRootQueues(); for (int i = 0; i < qis.length; i++) { QueueInfo qi = qis[i]; System.out.println(qi.getQueueName()); System.out.println(qi.getSchedulingInfo()); System.out.println(qi.getState()); System.out.println(qi.getProperties()); } int mapnum = client.getTaskReports(jobid, TaskType.MAP).length; int rednum = client.getTaskReports(jobid, TaskType.REDUCE).length; float progress = status.getMapProgress(); sb.append( String.format( "m %4d %6s", mapnum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100))); sb.append("|"); progress = status.getReduceProgress(); sb.append( String.format( "r %4d %6s", rednum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100))); System.out.println(sb.toString()); printTaskAttempt(client.getTaskReports(jobid, TaskType.MAP)); printTaskAttempt(client.getTaskReports(jobid, TaskType.REDUCE)); System.out.println(sb.toString()); } catch (Exception e) { System.out.println(e); } }
@Test public void testAMAccessDisabled() throws IOException { // test only applicable when AM not reachable if (isAMReachableFromClient) { return; } MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(getJobReportRequest())) .thenReturn(getJobReportResponseFromHistoryServer()); ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class); try { when(rmDelegate.getApplicationReport(jobId.getAppId())) .thenReturn(getRunningApplicationReport("am1", 78)) .thenReturn(getRunningApplicationReport("am1", 78)) .thenReturn(getRunningApplicationReport("am1", 78)) .thenReturn(getFinishedApplicationReport()); } catch (YarnException e) { throw new IOException(e); } ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(historyServerProxy, rmDelegate)); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("N/A", jobStatus.getJobName()); verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class)); // Should not reach AM even for second and third times too. jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("N/A", jobStatus.getJobName()); verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class)); jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("N/A", jobStatus.getJobName()); verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class)); // The third time around, app is completed, so should go to JHS JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus1); Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile()); Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl()); Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f); Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f); verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class)); }
@Test public void testJobReportFromHistoryServer() throws Exception { MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); when(historyServerProxy.getJobReport(getJobReportRequest())) .thenReturn(getJobReportResponseFromHistoryServer()); ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null); ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm); JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile()); Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl()); Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f); Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f); }
public static JobStatus downgrade(org.apache.hadoop.mapreduce.JobStatus stat) { JobStatus old = new JobStatus( JobID.downgrade(stat.getJobID()), stat.getSetupProgress(), stat.getMapProgress(), stat.getReduceProgress(), stat.getCleanupProgress(), stat.getState().getValue(), JobPriority.valueOf(stat.getPriority().name()), stat.getUsername(), stat.getJobName(), stat.getJobFile(), stat.getTrackingUrl()); old.setStartTime(stat.getStartTime()); old.setFinishTime(stat.getFinishTime()); old.setSchedulingInfo(stat.getSchedulingInfo()); old.setHistoryFile(stat.getHistoryFile()); return old; }