示例#1
0
 /**
  * simple test JobPriorityChangeEvent and JobPriorityChange
  *
  * @throws Exception
  */
 @Test(timeout = 10000)
 public void testJobPriorityChange() throws Exception {
   org.apache.hadoop.mapreduce.JobID jid = new JobID("001", 1);
   JobPriorityChangeEvent test = new JobPriorityChangeEvent(jid, JobPriority.LOW);
   assertEquals(test.getJobId().toString(), jid.toString());
   assertEquals(test.getPriority(), JobPriority.LOW);
 }
示例#2
0
 @Test(timeout = 10000)
 public void testJobQueueChange() throws Exception {
   org.apache.hadoop.mapreduce.JobID jid = new JobID("001", 1);
   JobQueueChangeEvent test = new JobQueueChangeEvent(jid, "newqueue");
   assertEquals(test.getJobId().toString(), jid.toString());
   assertEquals(test.getJobQueueName(), "newqueue");
 }
示例#3
0
 private static void setJobID(Job job, JobID jobID, String namedOutput) {
   JobID newJobID =
       jobID == null || jobID.getJtIdentifier().contains(namedOutput)
           ? jobID
           : new JobID(jobID.getJtIdentifier() + "_" + namedOutput, jobID.getId());
   job.setJobID(newJobID);
 }
 private static StubbedJob createStubbedJob(
     Configuration conf, Dispatcher dispatcher, int numSplits, AppContext appContext) {
   JobID jobID = JobID.forName("job_1234567890000_0001");
   JobId jobId = TypeConverter.toYarn(jobID);
   if (appContext == null) {
     appContext = mock(AppContext.class);
     when(appContext.hasSuccessfullyUnregistered()).thenReturn(true);
   }
   StubbedJob job =
       new StubbedJob(
           jobId,
           ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0),
           conf,
           dispatcher.getEventHandler(),
           true,
           "somebody",
           numSplits,
           appContext);
   dispatcher.register(JobEventType.class, job);
   EventHandler mockHandler = mock(EventHandler.class);
   dispatcher.register(TaskEventType.class, mockHandler);
   dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class, mockHandler);
   dispatcher.register(JobFinishEvent.Type.class, mockHandler);
   return job;
 }
 private boolean testUberDecision(Configuration conf) {
   JobID jobID = JobID.forName("job_1234567890000_0001");
   JobId jobId = TypeConverter.toYarn(jobID);
   MRAppMetrics mrAppMetrics = MRAppMetrics.create();
   JobImpl job =
       new JobImpl(
           jobId,
           ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0),
           conf,
           mock(EventHandler.class),
           null,
           new JobTokenSecretManager(),
           new Credentials(),
           null,
           null,
           mrAppMetrics,
           null,
           true,
           null,
           0,
           null,
           null,
           null,
           null);
   InitTransition initTransition = getInitTransition(2);
   JobEvent mockJobEvent = mock(JobEvent.class);
   initTransition.transition(job, mockJobEvent);
   boolean isUber = job.isUber();
   return isUber;
 }
  @Test
  public void testReportDiagnostics() throws Exception {
    JobID jobID = JobID.forName("job_1234567890000_0001");
    JobId jobId = TypeConverter.toYarn(jobID);
    final String diagMsg = "some diagnostic message";
    final JobDiagnosticsUpdateEvent diagUpdateEvent = new JobDiagnosticsUpdateEvent(jobId, diagMsg);
    MRAppMetrics mrAppMetrics = MRAppMetrics.create();
    AppContext mockContext = mock(AppContext.class);
    when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
    JobImpl job =
        new JobImpl(
            jobId,
            Records.newRecord(ApplicationAttemptId.class),
            new Configuration(),
            mock(EventHandler.class),
            null,
            mock(JobTokenSecretManager.class),
            null,
            new SystemClock(),
            null,
            mrAppMetrics,
            null,
            true,
            null,
            0,
            null,
            mockContext,
            null,
            null);
    job.handle(diagUpdateEvent);
    String diagnostics = job.getReport().getDiagnostics();
    Assert.assertNotNull(diagnostics);
    Assert.assertTrue(diagnostics.contains(diagMsg));

    job =
        new JobImpl(
            jobId,
            Records.newRecord(ApplicationAttemptId.class),
            new Configuration(),
            mock(EventHandler.class),
            null,
            mock(JobTokenSecretManager.class),
            null,
            new SystemClock(),
            null,
            mrAppMetrics,
            null,
            true,
            null,
            0,
            null,
            mockContext,
            null,
            null);
    job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
    job.handle(diagUpdateEvent);
    diagnostics = job.getReport().getDiagnostics();
    Assert.assertNotNull(diagnostics);
    Assert.assertTrue(diagnostics.contains(diagMsg));
  }
示例#7
0
 public static String getJobFile(
     Configuration conf, String user, org.apache.hadoop.mapreduce.JobID jobId) {
   Path jobFile =
       new Path(
           MRApps.getStagingAreaDir(conf, user),
           jobId.toString() + Path.SEPARATOR + MRJobConfig.JOB_CONF_FILE);
   return jobFile.toString();
 }
 @Override
 protected Job createJob(Configuration conf, JobStateInternal forcedState, String diagnostic) {
   JobImpl jobImpl = mock(JobImpl.class);
   when(jobImpl.getInternalState()).thenReturn(this.jobStateInternal);
   JobID jobID = JobID.forName("job_1234567890000_0001");
   JobId jobId = TypeConverter.toYarn(jobID);
   when(jobImpl.getID()).thenReturn(jobId);
   ((AppContext) getContext()).getAllJobs().put(jobImpl.getID(), jobImpl);
   return jobImpl;
 }
示例#9
0
  /**
   * Tests if {@link TraceBuilder} can correctly identify and parse jobhistory filenames. The
   * testcase checks if {@link TraceBuilder} - correctly identifies a jobhistory filename -
   * correctly parses a jobhistory filename to extract out the jobid - correctly identifies a
   * job-configuration filename stored along with the jobhistory files
   */
  @Test
  public void testJobHistoryFilenameParsing() throws IOException {
    final Configuration conf = new Configuration();
    final FileSystem lfs = FileSystem.getLocal(conf);
    String user = "******";
    org.apache.hadoop.mapred.JobID jid = new org.apache.hadoop.mapred.JobID("12345", 1);
    final Path rootInputDir =
        new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);

    // Check if jobhistory filename are detected properly
    Path jhFilename = new Path(jid + "_1234_user_jobname");
    JobID extractedJID = JobID.forName(TraceBuilder.extractJobID(jhFilename.getName()));
    assertEquals("TraceBuilder failed to parse the current JH filename", jid, extractedJID);

    // Check if the conf filename in jobhistory are detected properly
    Path jhConfFilename = new Path(jid + "_conf.xml");
    assertTrue(
        "TraceBuilder failed to parse the current JH conf filename",
        TraceBuilder.isJobConfXml(jhConfFilename.getName(), null));
  }
 /**
  * Get the user log directory for the job jobid.
  *
  * @param jobid the jobid object
  * @return user log directory for the job
  */
 public static File getJobDir(JobID jobid) {
   return getJobDir(jobid.toString());
 }
/** Tests for ClientServiceDelegate.java */
@RunWith(value = Parameterized.class)
public class TestClientServiceDelegate {
  private JobID oldJobId = JobID.forName("job_1315895242400_2");
  private org.apache.hadoop.mapreduce.v2.api.records.JobId jobId = TypeConverter.toYarn(oldJobId);
  private boolean isAMReachableFromClient;

  public TestClientServiceDelegate(boolean isAMReachableFromClient) {
    this.isAMReachableFromClient = isAMReachableFromClient;
  }

  @Parameters
  public static Collection<Object[]> data() {
    Object[][] data = new Object[][] {{true}, {false}};
    return Arrays.asList(data);
  }

  @Test
  public void testUnknownAppInRM() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(getJobReportResponse());
    ClientServiceDelegate clientServiceDelegate =
        getClientServiceDelegate(historyServerProxy, getRMDelegate());

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
  }

  @Test
  public void testRemoteExceptionFromHistoryServer() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenThrow(new IOException("Job ID doesnot Exist"));

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);

    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);

    try {
      clientServiceDelegate.getJobStatus(oldJobId);
      Assert.fail("Invoke should throw exception after retries.");
    } catch (IOException e) {
      Assert.assertTrue(e.getMessage().contains("Job ID doesnot Exist"));
    }
  }

  @Test
  public void testRetriesOnConnectionFailure() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenThrow(new RuntimeException("1"))
        .thenThrow(new RuntimeException("2"))
        .thenReturn(getJobReportResponse());

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);

    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    verify(historyServerProxy, times(3)).getJobReport(any(GetJobReportRequest.class));
  }

  @Test
  public void testRetriesOnAMConnectionFailures() throws Exception {
    if (!isAMReachableFromClient) {
      return;
    }

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
        .thenReturn(getRunningApplicationReport("am1", 78));

    // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
    // succeed in the 5th call.
    final MRClientProtocol amProxy = mock(MRClientProtocol.class);
    when(amProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenThrow(new RuntimeException("11"))
        .thenThrow(new RuntimeException("22"))
        .thenThrow(new RuntimeException("33"))
        .thenThrow(new RuntimeException("44"))
        .thenReturn(getJobReportResponse());
    Configuration conf = new YarnConfiguration();
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    ClientServiceDelegate clientServiceDelegate =
        new ClientServiceDelegate(conf, rm, oldJobId, null) {
          @Override
          MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr)
              throws IOException {
            super.instantiateAMProxy(serviceAddr);
            return amProxy;
          }
        };

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);

    Assert.assertNotNull(jobStatus);
    // assert maxClientRetry is not decremented.
    Assert.assertEquals(
        conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES),
        clientServiceDelegate.getMaxClientRetry());
    verify(amProxy, times(5)).getJobReport(any(GetJobReportRequest.class));
  }

  @Test
  public void testHistoryServerNotConfigured() throws Exception {
    // RM doesn't have app report and job History Server is not configured
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(null, getRMDelegate());
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals("N/A", jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState());

    // RM has app report and job History Server is not configured
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    ApplicationReport applicationReport = getFinishedApplicationReport();
    when(rm.getApplicationReport(jobId.getAppId())).thenReturn(applicationReport);

    clientServiceDelegate = getClientServiceDelegate(null, rm);
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }

  @Test
  public void testJobReportFromHistoryServer() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenReturn(getJobReportResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());
    Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());
    Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
    Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
  }

  @Test
  public void testCountersFromHistoryServer() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getCounters(getCountersRequest()))
        .thenReturn(getCountersResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);

    Counters counters = TypeConverter.toYarn(clientServiceDelegate.getJobCounters(oldJobId));
    Assert.assertNotNull(counters);
    Assert.assertEquals(
        1001, counters.getCounterGroup("dummyCounters").getCounter("dummyCounter").getValue());
  }

  @Test
  public void testReconnectOnAMRestart() throws IOException {
    // test not applicable when AM not reachable
    // as instantiateAMProxy is not called at all
    if (!isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);

    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport(null, 0))
          .thenReturn(getRunningApplicationReport(null, 0))
          .thenReturn(getRunningApplicationReport("am2", 90));
    } catch (YarnException e) {
      throw new IOException(e);
    }

    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport())
        .thenReturn(
            MRBuilderUtils.newJobReport(
                jobId,
                "jobName-firstGen",
                "user",
                JobState.RUNNING,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                "anything",
                null,
                false,
                ""));

    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse1)
        .thenThrow(new RuntimeException("AM is down!"));

    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport())
        .thenReturn(
            MRBuilderUtils.newJobReport(
                jobId,
                "jobName-secondGen",
                "user",
                JobState.RUNNING,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                "anything",
                null,
                false,
                ""));

    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse2);

    ClientServiceDelegate clientServiceDelegate =
        spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests
    // should use the same proxy to AM2 and so instantiateProxy shouldn't be
    // called.
    doReturn(firstGenAMProxy)
        .doReturn(secondGenAMProxy)
        .when(clientServiceDelegate)
        .instantiateAMProxy(any(InetSocketAddress.class));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class));
  }

  @Test
  public void testAMAccessDisabled() throws IOException {
    // test only applicable when AM not reachable
    if (isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenReturn(getJobReportResponseFromHistoryServer());

    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getFinishedApplicationReport());
    } catch (YarnException e) {
      throw new IOException(e);
    }

    ClientServiceDelegate clientServiceDelegate =
        spy(getClientServiceDelegate(historyServerProxy, rmDelegate));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());

    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));

    // Should not reach AM even for second and third times too.
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));

    // The third time around, app is completed, so should go to JHS
    JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus1);
    Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile());
    Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl());
    Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f);
    Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f);

    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
  }

  @Test
  public void testRMDownForJobStatusBeforeGetAMReport() throws IOException {
    Configuration conf = new YarnConfiguration();
    testRMDownForJobStatusBeforeGetAMReport(conf, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES);
  }

  @Test
  public void testRMDownForJobStatusBeforeGetAMReportWithRetryTimes() throws IOException {
    Configuration conf = new YarnConfiguration();
    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 2);
    testRMDownForJobStatusBeforeGetAMReport(
        conf,
        conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES));
  }

  @Test
  public void testRMDownRestoreForJobStatusBeforeGetAMReport() throws IOException {
    Configuration conf = new YarnConfiguration();
    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);

    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenThrow(
              new java.lang.reflect.UndeclaredThrowableException(
                  new IOException("Connection refuced1")))
          .thenThrow(
              new java.lang.reflect.UndeclaredThrowableException(
                  new IOException("Connection refuced2")))
          .thenReturn(getFinishedApplicationReport());
      ClientServiceDelegate clientServiceDelegate =
          new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
      JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
      verify(rmDelegate, times(3)).getApplicationReport(any(ApplicationId.class));
      Assert.assertNotNull(jobStatus);
    } catch (YarnException e) {
      throw new IOException(e);
    }
  }

  private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries)
      throws IOException {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenThrow(
              new java.lang.reflect.UndeclaredThrowableException(
                  new IOException("Connection refuced1")))
          .thenThrow(
              new java.lang.reflect.UndeclaredThrowableException(
                  new IOException("Connection refuced2")))
          .thenThrow(
              new java.lang.reflect.UndeclaredThrowableException(
                  new IOException("Connection refuced3")));
      ClientServiceDelegate clientServiceDelegate =
          new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
      try {
        clientServiceDelegate.getJobStatus(oldJobId);
        Assert.fail("It should throw exception after retries");
      } catch (IOException e) {
        System.out.println("fail to get job status,and e=" + e.toString());
      }
      verify(rmDelegate, times(noOfRetries)).getApplicationReport(any(ApplicationId.class));
    } catch (YarnException e) {
      throw new IOException(e);
    }
  }

  private GetJobReportRequest getJobReportRequest() {
    GetJobReportRequest request = Records.newRecord(GetJobReportRequest.class);
    request.setJobId(jobId);
    return request;
  }

  private GetJobReportResponse getJobReportResponse() {
    GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class);
    JobReport jobReport = Records.newRecord(JobReport.class);
    jobReport.setJobId(jobId);
    jobReport.setJobState(JobState.SUCCEEDED);
    jobReportResponse.setJobReport(jobReport);
    return jobReportResponse;
  }

  private GetCountersRequest getCountersRequest() {
    GetCountersRequest request = Records.newRecord(GetCountersRequest.class);
    request.setJobId(jobId);
    return request;
  }

  private ApplicationReport getFinishedApplicationReport() {
    ApplicationId appId = ApplicationId.newInstance(1234, 5);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
    return ApplicationReport.newInstance(
        appId,
        attemptId,
        "user",
        "queue",
        "appname",
        "host",
        124,
        null,
        YarnApplicationState.FINISHED,
        "diagnostics",
        "url",
        0,
        0,
        FinalApplicationStatus.SUCCEEDED,
        null,
        "N/A",
        0.0f,
        YarnConfiguration.DEFAULT_APPLICATION_TYPE,
        null);
  }

  private ApplicationReport getRunningApplicationReport(String host, int port) {
    ApplicationId appId = ApplicationId.newInstance(1234, 5);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
    return ApplicationReport.newInstance(
        appId,
        attemptId,
        "user",
        "queue",
        "appname",
        host,
        port,
        null,
        YarnApplicationState.RUNNING,
        "diagnostics",
        "url",
        0,
        0,
        FinalApplicationStatus.UNDEFINED,
        null,
        "N/A",
        0.0f,
        YarnConfiguration.DEFAULT_APPLICATION_TYPE,
        null);
  }

  private ResourceMgrDelegate getRMDelegate() throws IOException {
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    try {
      ApplicationId appId = jobId.getAppId();
      when(rm.getApplicationReport(appId))
          .thenThrow(new ApplicationNotFoundException(appId + " not found"));
    } catch (YarnException e) {
      throw new IOException(e);
    }
    return rm;
  }

  private ClientServiceDelegate getClientServiceDelegate(
      MRClientProtocol historyServerProxy, ResourceMgrDelegate rm) {
    Configuration conf = new YarnConfiguration();
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    ClientServiceDelegate clientServiceDelegate =
        new ClientServiceDelegate(conf, rm, oldJobId, historyServerProxy);
    return clientServiceDelegate;
  }

  private GetJobReportResponse getJobReportResponseFromHistoryServer() {
    GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class);
    JobReport jobReport = Records.newRecord(JobReport.class);
    jobReport.setJobId(jobId);
    jobReport.setJobState(JobState.SUCCEEDED);
    jobReport.setMapProgress(1.0f);
    jobReport.setReduceProgress(1.0f);
    jobReport.setJobFile("TestJobFilePath");
    jobReport.setTrackingUrl("http://TestTrackingUrl");
    jobReportResponse.setJobReport(jobReport);
    return jobReportResponse;
  }

  private GetCountersResponse getCountersResponseFromHistoryServer() {
    GetCountersResponse countersResponse = Records.newRecord(GetCountersResponse.class);
    Counter counter = Records.newRecord(Counter.class);
    CounterGroup counterGroup = Records.newRecord(CounterGroup.class);
    Counters counters = Records.newRecord(Counters.class);
    counter.setDisplayName("dummyCounter");
    counter.setName("dummyCounter");
    counter.setValue(1001);
    counterGroup.setName("dummyCounters");
    counterGroup.setDisplayName("dummyCounters");
    counterGroup.setCounter("dummyCounter", counter);
    counters.setCounterGroup("dummyCounters", counterGroup);
    countersResponse.setCounters(counters);
    return countersResponse;
  }
}
示例#12
0
 @Override
 public TaskAttemptID newTaskAttemptID(JobID jobId, boolean isMap, int taskId, int id) {
   return new TaskAttemptID(
       jobId.getJtIdentifier(), jobId.getId(), isMap ? TaskType.MAP : TaskType.REDUCE, taskId, id);
 }
  public void test(
      String workflowId,
      String workflowName,
      String workflowNodeName,
      Map<String, String[]> adjacencies) {
    Configuration conf = new Configuration();
    setProperties(conf, workflowId, workflowName, workflowNodeName, adjacencies);
    String log =
        log(
            "JOB",
            new String[] {ID, NAME, NODE, ADJ},
            new String[] {
              conf.get(ID_PROP),
              conf.get(NAME_PROP),
              conf.get(NODE_PROP),
              JobHistory.JobInfo.getWorkflowAdjacencies(conf)
            });
    ParsedLine line = new ParsedLine(log);
    JobID jobid = new JobID("id", 1);
    JobSubmittedEvent event =
        new JobSubmittedEvent(
            jobid,
            workflowName,
            "",
            0l,
            "",
            null,
            "",
            line.get(ID),
            line.get(NAME),
            line.get(NODE),
            line.get(ADJ));
    WorkflowContext context = MapReduceJobHistoryUpdater.buildWorkflowContext(event);

    String resultingWorkflowId = workflowId;
    if (workflowId.isEmpty()) resultingWorkflowId = jobid.toString().replace("job_", "mr_");
    assertEquals("Didn't recover workflowId", resultingWorkflowId, context.getWorkflowId());
    assertEquals("Didn't recover workflowName", workflowName, context.getWorkflowName());
    assertEquals(
        "Didn't recover workflowNodeName", workflowNodeName, context.getWorkflowEntityName());

    Map<String, String[]> resultingAdjacencies = adjacencies;
    if (resultingAdjacencies.size() == 0) {
      resultingAdjacencies = new HashMap<String, String[]>();
      resultingAdjacencies.put(workflowNodeName, new String[] {});
    }
    assertEquals(
        "Got incorrect number of adjacencies",
        resultingAdjacencies.size(),
        context.getWorkflowDag().getEntries().size());
    for (WorkflowDagEntry entry : context.getWorkflowDag().getEntries()) {
      String[] sTargets = resultingAdjacencies.get(entry.getSource());
      assertNotNull("No original targets for " + entry.getSource(), sTargets);
      List<String> dTargets = entry.getTargets();
      assertEquals(
          "Got incorrect number of targets for " + entry.getSource(),
          sTargets.length,
          dTargets.size());
      for (int i = 0; i < sTargets.length; i++) {
        assertEquals("Got incorrect target for " + entry.getSource(), sTargets[i], dTargets.get(i));
      }
    }
  }
 public static Path getJobPath(JobID jobID, Path workingDirectory) {
   return new Path(workingDirectory, jobID.toString());
 }
示例#15
0
  @Test
  public void testCheckAccess() {
    // Create two unique users
    String user1 = System.getProperty("user.name");
    String user2 = user1 + "1234";
    UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser(user1);
    UserGroupInformation ugi2 = UserGroupInformation.createRemoteUser(user2);

    // Create the job
    JobID jobID = JobID.forName("job_1234567890000_0001");
    JobId jobId = TypeConverter.toYarn(jobID);

    // Setup configuration access only to user1 (owner)
    Configuration conf1 = new Configuration();
    conf1.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
    conf1.set(MRJobConfig.JOB_ACL_VIEW_JOB, "");

    // Verify access
    JobImpl job1 =
        new JobImpl(
            jobId, null, conf1, null, null, null, null, null, null, null, null, true, null, 0, null,
            null, null, null);
    Assert.assertTrue(job1.checkAccess(ugi1, JobACL.VIEW_JOB));
    Assert.assertFalse(job1.checkAccess(ugi2, JobACL.VIEW_JOB));

    // Setup configuration access to the user1 (owner) and user2
    Configuration conf2 = new Configuration();
    conf2.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
    conf2.set(MRJobConfig.JOB_ACL_VIEW_JOB, user2);

    // Verify access
    JobImpl job2 =
        new JobImpl(
            jobId, null, conf2, null, null, null, null, null, null, null, null, true, null, 0, null,
            null, null, null);
    Assert.assertTrue(job2.checkAccess(ugi1, JobACL.VIEW_JOB));
    Assert.assertTrue(job2.checkAccess(ugi2, JobACL.VIEW_JOB));

    // Setup configuration access with security enabled and access to all
    Configuration conf3 = new Configuration();
    conf3.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
    conf3.set(MRJobConfig.JOB_ACL_VIEW_JOB, "*");

    // Verify access
    JobImpl job3 =
        new JobImpl(
            jobId, null, conf3, null, null, null, null, null, null, null, null, true, null, 0, null,
            null, null, null);
    Assert.assertTrue(job3.checkAccess(ugi1, JobACL.VIEW_JOB));
    Assert.assertTrue(job3.checkAccess(ugi2, JobACL.VIEW_JOB));

    // Setup configuration access without security enabled
    Configuration conf4 = new Configuration();
    conf4.setBoolean(MRConfig.MR_ACLS_ENABLED, false);
    conf4.set(MRJobConfig.JOB_ACL_VIEW_JOB, "");

    // Verify access
    JobImpl job4 =
        new JobImpl(
            jobId, null, conf4, null, null, null, null, null, null, null, null, true, null, 0, null,
            null, null, null);
    Assert.assertTrue(job4.checkAccess(ugi1, JobACL.VIEW_JOB));
    Assert.assertTrue(job4.checkAccess(ugi2, JobACL.VIEW_JOB));

    // Setup configuration access without security enabled
    Configuration conf5 = new Configuration();
    conf5.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
    conf5.set(MRJobConfig.JOB_ACL_VIEW_JOB, "");

    // Verify access
    JobImpl job5 =
        new JobImpl(
            jobId, null, conf5, null, null, null, null, null, null, null, null, true, null, 0, null,
            null, null, null);
    Assert.assertTrue(job5.checkAccess(ugi1, null));
    Assert.assertTrue(job5.checkAccess(ugi2, null));
  }