예제 #1
0
  public void getTaskDiagnostics(String job) {

    try {
      String errinfo[];
      StringBuilder sb = new StringBuilder();

      JobID jobid = JobID.forName(job);
      JobStatus status = client.getJobStatus(jobid);
      System.out.println(status.getTrackingUrl());
      System.out.println(status.getJobFile());
      System.out.println(status.getHistoryFile());
      System.out.println(status.getQueue());
      System.out.println(status.toString());

      System.out.println("#### QueueAclsInfo ####");
      QueueAclsInfo[] qais = client.getQueueAclsForCurrentUser();
      for (int i = 0; i < qais.length; i++) {
        QueueAclsInfo qai = qais[i];
        System.out.println(qai.getQueueName().toString());
        String[] ops = qai.getOperations();
        for (int j = 0; j < ops.length; j++) {
          System.out.println(ops[j]);
        }
      }

      System.out.println("#### QueueInfo ####");
      QueueInfo[] qis = client.getRootQueues();
      for (int i = 0; i < qis.length; i++) {
        QueueInfo qi = qis[i];
        System.out.println(qi.getQueueName());
        System.out.println(qi.getSchedulingInfo());
        System.out.println(qi.getState());
        System.out.println(qi.getProperties());
      }

      int mapnum = client.getTaskReports(jobid, TaskType.MAP).length;
      int rednum = client.getTaskReports(jobid, TaskType.REDUCE).length;
      float progress = status.getMapProgress();
      sb.append(
          String.format(
              "m %4d %6s",
              mapnum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100)));
      sb.append("|");
      progress = status.getReduceProgress();
      sb.append(
          String.format(
              "r %4d %6s",
              rednum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100)));

      System.out.println(sb.toString());
      printTaskAttempt(client.getTaskReports(jobid, TaskType.MAP));
      printTaskAttempt(client.getTaskReports(jobid, TaskType.REDUCE));
      System.out.println(sb.toString());

    } catch (Exception e) {
      System.out.println(e);
    }
  }
  @Test
  public void testJobReportFromHistoryServer() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenReturn(getJobReportResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());
    Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());
    Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
    Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
  }
  @Test
  public void testAMAccessDisabled() throws IOException {
    // test only applicable when AM not reachable
    if (isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenReturn(getJobReportResponseFromHistoryServer());

    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getFinishedApplicationReport());
    } catch (YarnException e) {
      throw new IOException(e);
    }

    ClientServiceDelegate clientServiceDelegate =
        spy(getClientServiceDelegate(historyServerProxy, rmDelegate));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());

    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));

    // Should not reach AM even for second and third times too.
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));

    // The third time around, app is completed, so should go to JHS
    JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus1);
    Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile());
    Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl());
    Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f);
    Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f);

    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
  }
  @Test
  public void testHistoryServerNotConfigured() throws Exception {
    // RM doesn't have app report and job History Server is not configured
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(null, getRMDelegate());
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals("N/A", jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState());

    // RM has app report and job History Server is not configured
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    ApplicationReport applicationReport = getFinishedApplicationReport();
    when(rm.getApplicationReport(jobId.getAppId())).thenReturn(applicationReport);

    clientServiceDelegate = getClientServiceDelegate(null, rm);
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }
예제 #5
0
 /**
  * Set the priority of the job, defaulting to NORMAL.
  *
  * @param jp new job priority
  */
 public synchronized void setJobPriority(JobPriority jp) {
   super.setPriority(org.apache.hadoop.mapreduce.JobPriority.valueOf(jp.name()));
 }
예제 #6
0
 protected synchronized void setJobACLs(Map<JobACL, AccessControlList> acls) {
   super.setJobACLs(acls);
 }
예제 #7
0
 /**
  * Used to set the scheduling information associated to a particular Job.
  *
  * @param schedulingInfo Scheduling information of the job
  */
 protected synchronized void setSchedulingInfo(String schedulingInfo) {
   super.setSchedulingInfo(schedulingInfo);
 }
예제 #8
0
 /**
  * Sets the setup progress of this job
  *
  * @param p The value of setup progress to set to
  */
 protected synchronized void setSetupProgress(float p) {
   super.setSetupProgress(p);
 }
예제 #9
0
 /**
  * Set the start time of the job
  *
  * @param startTime The startTime of the job
  */
 protected synchronized void setStartTime(long startTime) {
   super.setStartTime(startTime);
 }
예제 #10
0
 /** Change the current run state of the job. */
 protected synchronized void setRunState(int state) {
   super.setState(getEnum(state));
 }
예제 #11
0
 /** Set the job retire flag to true. */
 protected synchronized void setRetired() {
   super.setRetired();
 }
예제 #12
0
 /** Set the link to the web-ui for details of the job. */
 protected synchronized void setTrackingUrl(String trackingUrl) {
   super.setTrackingUrl(trackingUrl);
 }
예제 #13
0
 /** Set the job history file url for a completed job */
 protected synchronized void setHistoryFile(String historyFile) {
   super.setHistoryFile(historyFile);
 }
예제 #14
0
 /**
  * Set the finish time of the job
  *
  * @param finishTime The finishTime of the job
  */
 protected synchronized void setFinishTime(long finishTime) {
   super.setFinishTime(finishTime);
 }
예제 #15
0
 /**
  * Sets the reduce progress of this Job
  *
  * @param p The value of reduce progress to set to
  */
 protected synchronized void setReduceProgress(float p) {
   super.setReduceProgress(p);
 }
  @Test
  public void testReconnectOnAMRestart() throws IOException {
    // test not applicable when AM not reachable
    // as instantiateAMProxy is not called at all
    if (!isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);

    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport(null, 0))
          .thenReturn(getRunningApplicationReport(null, 0))
          .thenReturn(getRunningApplicationReport("am2", 90));
    } catch (YarnException e) {
      throw new IOException(e);
    }

    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport())
        .thenReturn(
            MRBuilderUtils.newJobReport(
                jobId,
                "jobName-firstGen",
                "user",
                JobState.RUNNING,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                "anything",
                null,
                false,
                ""));

    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse1)
        .thenThrow(new RuntimeException("AM is down!"));

    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport())
        .thenReturn(
            MRBuilderUtils.newJobReport(
                jobId,
                "jobName-secondGen",
                "user",
                JobState.RUNNING,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                "anything",
                null,
                false,
                ""));

    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse2);

    ClientServiceDelegate clientServiceDelegate =
        spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests
    // should use the same proxy to AM2 and so instantiateProxy shouldn't be
    // called.
    doReturn(firstGenAMProxy)
        .doReturn(secondGenAMProxy)
        .when(clientServiceDelegate)
        .instantiateAMProxy(any(InetSocketAddress.class));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class));
  }
예제 #17
0
 /** @param userName The username of the job */
 protected synchronized void setUsername(String userName) {
   super.setUsername(userName);
 }
예제 #18
0
 public static JobStatus downgrade(org.apache.hadoop.mapreduce.JobStatus stat) {
   JobStatus old =
       new JobStatus(
           JobID.downgrade(stat.getJobID()),
           stat.getSetupProgress(),
           stat.getMapProgress(),
           stat.getReduceProgress(),
           stat.getCleanupProgress(),
           stat.getState().getValue(),
           JobPriority.valueOf(stat.getPriority().name()),
           stat.getUsername(),
           stat.getJobName(),
           stat.getJobFile(),
           stat.getTrackingUrl());
   old.setStartTime(stat.getStartTime());
   old.setFinishTime(stat.getFinishTime());
   old.setSchedulingInfo(stat.getSchedulingInfo());
   old.setHistoryFile(stat.getHistoryFile());
   return old;
 }