@Test
  public void testRMDownRestoreForJobStatusBeforeGetAMReport() throws IOException {
    Configuration conf = new YarnConfiguration();
    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);

    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenThrow(
              new java.lang.reflect.UndeclaredThrowableException(
                  new IOException("Connection refuced1")))
          .thenThrow(
              new java.lang.reflect.UndeclaredThrowableException(
                  new IOException("Connection refuced2")))
          .thenReturn(getFinishedApplicationReport());
      ClientServiceDelegate clientServiceDelegate =
          new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
      JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
      verify(rmDelegate, times(3)).getApplicationReport(any(ApplicationId.class));
      Assert.assertNotNull(jobStatus);
    } catch (YarnException e) {
      throw new IOException(e);
    }
  }
 private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries)
     throws IOException {
   conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
   conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
   MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
   ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
   try {
     when(rmDelegate.getApplicationReport(jobId.getAppId()))
         .thenThrow(
             new java.lang.reflect.UndeclaredThrowableException(
                 new IOException("Connection refuced1")))
         .thenThrow(
             new java.lang.reflect.UndeclaredThrowableException(
                 new IOException("Connection refuced2")))
         .thenThrow(
             new java.lang.reflect.UndeclaredThrowableException(
                 new IOException("Connection refuced3")));
     ClientServiceDelegate clientServiceDelegate =
         new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
     try {
       clientServiceDelegate.getJobStatus(oldJobId);
       Assert.fail("It should throw exception after retries");
     } catch (IOException e) {
       System.out.println("fail to get job status,and e=" + e.toString());
     }
     verify(rmDelegate, times(noOfRetries)).getApplicationReport(any(ApplicationId.class));
   } catch (YarnException e) {
     throw new IOException(e);
   }
 }
 private ResourceMgrDelegate getRMDelegate() throws IOException {
   ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
   try {
     ApplicationId appId = jobId.getAppId();
     when(rm.getApplicationReport(appId))
         .thenThrow(new ApplicationNotFoundException(appId + " not found"));
   } catch (YarnException e) {
     throw new IOException(e);
   }
   return rm;
 }
  @Test
  public void testAMAccessDisabled() throws IOException {
    // test only applicable when AM not reachable
    if (isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenReturn(getJobReportResponseFromHistoryServer());

    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getFinishedApplicationReport());
    } catch (YarnException e) {
      throw new IOException(e);
    }

    ClientServiceDelegate clientServiceDelegate =
        spy(getClientServiceDelegate(historyServerProxy, rmDelegate));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());

    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));

    // Should not reach AM even for second and third times too.
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));

    // The third time around, app is completed, so should go to JHS
    JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus1);
    Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile());
    Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl());
    Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f);
    Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f);

    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
  }
  @Test
  public void testCountersFromHistoryServer() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getCounters(getCountersRequest()))
        .thenReturn(getCountersResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);

    Counters counters = TypeConverter.toYarn(clientServiceDelegate.getJobCounters(oldJobId));
    Assert.assertNotNull(counters);
    Assert.assertEquals(
        1001, counters.getCounterGroup("dummyCounters").getCounter("dummyCounter").getValue());
  }
  @Test
  public void testJobReportFromHistoryServer() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenReturn(getJobReportResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());
    Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());
    Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
    Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
  }
  @Test
  public void testHistoryServerNotConfigured() throws Exception {
    // RM doesn't have app report and job History Server is not configured
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(null, getRMDelegate());
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals("N/A", jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState());

    // RM has app report and job History Server is not configured
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    ApplicationReport applicationReport = getFinishedApplicationReport();
    when(rm.getApplicationReport(jobId.getAppId())).thenReturn(applicationReport);

    clientServiceDelegate = getClientServiceDelegate(null, rm);
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }
  @Test
  public void testRetriesOnConnectionFailure() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenThrow(new RuntimeException("1"))
        .thenThrow(new RuntimeException("2"))
        .thenReturn(getJobReportResponse());

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);

    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    verify(historyServerProxy, times(3)).getJobReport(any(GetJobReportRequest.class));
  }
  @Test
  public void testRemoteExceptionFromHistoryServer() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenThrow(new IOException("Job ID doesnot Exist"));

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);

    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);

    try {
      clientServiceDelegate.getJobStatus(oldJobId);
      Assert.fail("Invoke should throw exception after retries.");
    } catch (IOException e) {
      Assert.assertTrue(e.getMessage().contains("Job ID doesnot Exist"));
    }
  }
  @Test
  public void testRetriesOnAMConnectionFailures() throws Exception {
    if (!isAMReachableFromClient) {
      return;
    }

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
        .thenReturn(getRunningApplicationReport("am1", 78));

    // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
    // succeed in the 5th call.
    final MRClientProtocol amProxy = mock(MRClientProtocol.class);
    when(amProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenThrow(new RuntimeException("11"))
        .thenThrow(new RuntimeException("22"))
        .thenThrow(new RuntimeException("33"))
        .thenThrow(new RuntimeException("44"))
        .thenReturn(getJobReportResponse());
    Configuration conf = new YarnConfiguration();
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    ClientServiceDelegate clientServiceDelegate =
        new ClientServiceDelegate(conf, rm, oldJobId, null) {
          @Override
          MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr)
              throws IOException {
            super.instantiateAMProxy(serviceAddr);
            return amProxy;
          }
        };

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);

    Assert.assertNotNull(jobStatus);
    // assert maxClientRetry is not decremented.
    Assert.assertEquals(
        conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES),
        clientServiceDelegate.getMaxClientRetry());
    verify(amProxy, times(5)).getJobReport(any(GetJobReportRequest.class));
  }
Ejemplo n.º 11
0
 @Test(timeout = 20000)
 public void testJobSubmissionFailure() throws Exception {
   when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class)))
       .thenReturn(appId);
   ApplicationReport report = mock(ApplicationReport.class);
   when(report.getApplicationId()).thenReturn(appId);
   when(report.getDiagnostics()).thenReturn(failString);
   when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED);
   when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report);
   Credentials credentials = new Credentials();
   File jobxml = new File(testWorkDir, "job.xml");
   OutputStream out = new FileOutputStream(jobxml);
   conf.writeXml(out);
   out.close();
   try {
     yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
   } catch (IOException io) {
     LOG.info("Logging exception:", io);
     assertTrue(io.getLocalizedMessage().contains(failString));
   }
 }
  @Test
  public void testReconnectOnAMRestart() throws IOException {
    // test not applicable when AM not reachable
    // as instantiateAMProxy is not called at all
    if (!isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);

    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport(null, 0))
          .thenReturn(getRunningApplicationReport(null, 0))
          .thenReturn(getRunningApplicationReport("am2", 90));
    } catch (YarnException e) {
      throw new IOException(e);
    }

    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport())
        .thenReturn(
            MRBuilderUtils.newJobReport(
                jobId,
                "jobName-firstGen",
                "user",
                JobState.RUNNING,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                "anything",
                null,
                false,
                ""));

    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse1)
        .thenThrow(new RuntimeException("AM is down!"));

    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport())
        .thenReturn(
            MRBuilderUtils.newJobReport(
                jobId,
                "jobName-secondGen",
                "user",
                JobState.RUNNING,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                "anything",
                null,
                false,
                ""));

    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse2);

    ClientServiceDelegate clientServiceDelegate =
        spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests
    // should use the same proxy to AM2 and so instantiateProxy shouldn't be
    // called.
    doReturn(firstGenAMProxy)
        .doReturn(secondGenAMProxy)
        .when(clientServiceDelegate)
        .instantiateAMProxy(any(InetSocketAddress.class));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class));
  }
Ejemplo n.º 13
0
  @Test(timeout = 20000)
  public void testResourceMgrDelegate() throws Exception {
    /* we not want a mock of resource mgr delegate */
    final ClientRMProtocol clientRMProtocol = mock(ClientRMProtocol.class);
    ResourceMgrDelegate delegate =
        new ResourceMgrDelegate(conf) {
          @Override
          public synchronized void start() {
            this.rmClient = clientRMProtocol;
          }
        };
    /* make sure kill calls finish application master */
    when(clientRMProtocol.forceKillApplication(any(KillApplicationRequest.class))).thenReturn(null);
    delegate.killApplication(appId);
    verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class));

    /* make sure getalljobs calls get all applications */
    when(clientRMProtocol.getAllApplications(any(GetAllApplicationsRequest.class)))
        .thenReturn(recordFactory.newRecordInstance(GetAllApplicationsResponse.class));
    delegate.getAllJobs();
    verify(clientRMProtocol).getAllApplications(any(GetAllApplicationsRequest.class));

    /* make sure getapplication report is called */
    when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class)))
        .thenReturn(recordFactory.newRecordInstance(GetApplicationReportResponse.class));
    delegate.getApplicationReport(appId);
    verify(clientRMProtocol).getApplicationReport(any(GetApplicationReportRequest.class));

    /* make sure metrics is called */
    GetClusterMetricsResponse clusterMetricsResponse =
        recordFactory.newRecordInstance(GetClusterMetricsResponse.class);
    clusterMetricsResponse.setClusterMetrics(
        recordFactory.newRecordInstance(YarnClusterMetrics.class));
    when(clientRMProtocol.getClusterMetrics(any(GetClusterMetricsRequest.class)))
        .thenReturn(clusterMetricsResponse);
    delegate.getClusterMetrics();
    verify(clientRMProtocol).getClusterMetrics(any(GetClusterMetricsRequest.class));

    when(clientRMProtocol.getClusterNodes(any(GetClusterNodesRequest.class)))
        .thenReturn(recordFactory.newRecordInstance(GetClusterNodesResponse.class));
    delegate.getActiveTrackers();
    verify(clientRMProtocol).getClusterNodes(any(GetClusterNodesRequest.class));

    GetNewApplicationResponse newAppResponse =
        recordFactory.newRecordInstance(GetNewApplicationResponse.class);
    newAppResponse.setApplicationId(appId);
    when(clientRMProtocol.getNewApplication(any(GetNewApplicationRequest.class)))
        .thenReturn(newAppResponse);
    delegate.getNewJobID();
    verify(clientRMProtocol).getNewApplication(any(GetNewApplicationRequest.class));

    GetQueueInfoResponse queueInfoResponse =
        recordFactory.newRecordInstance(GetQueueInfoResponse.class);
    queueInfoResponse.setQueueInfo(recordFactory.newRecordInstance(QueueInfo.class));
    when(clientRMProtocol.getQueueInfo(any(GetQueueInfoRequest.class)))
        .thenReturn(queueInfoResponse);
    delegate.getQueues();
    verify(clientRMProtocol).getQueueInfo(any(GetQueueInfoRequest.class));

    GetQueueUserAclsInfoResponse aclResponse =
        recordFactory.newRecordInstance(GetQueueUserAclsInfoResponse.class);
    when(clientRMProtocol.getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class)))
        .thenReturn(aclResponse);
    delegate.getQueueAclsForCurrentUser();
    verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class));
  }