예제 #1
0
  private RMAppImpl createAndPopulateNewRMApp(
      ApplicationSubmissionContext submissionContext,
      long submitTime,
      String user,
      boolean isRecovery)
      throws YarnException {
    ApplicationId applicationId = submissionContext.getApplicationId();
    ResourceRequest amReq = validateAndCreateResourceRequest(submissionContext, isRecovery);

    // Verify and get the update application priority and set back to
    // submissionContext
    Priority appPriority =
        rmContext
            .getScheduler()
            .checkAndGetApplicationPriority(
                submissionContext.getPriority(), user, submissionContext.getQueue(), applicationId);
    submissionContext.setPriority(appPriority);

    // Create RMApp
    RMAppImpl application =
        new RMAppImpl(
            applicationId,
            rmContext,
            this.conf,
            submissionContext.getApplicationName(),
            user,
            submissionContext.getQueue(),
            submissionContext,
            this.scheduler,
            this.masterService,
            submitTime,
            submissionContext.getApplicationType(),
            submissionContext.getApplicationTags(),
            amReq);

    // Concurrent app submissions with same applicationId will fail here
    // Concurrent app submissions with different applicationIds will not
    // influence each other
    if (rmContext.getRMApps().putIfAbsent(applicationId, application) != null) {
      String message =
          "Application with id " + applicationId + " is already present! Cannot add a duplicate!";
      LOG.warn(message);
      throw new YarnException(message);
    }
    // Inform the ACLs Manager
    this.applicationACLsManager.addApplication(
        applicationId, submissionContext.getAMContainerSpec().getApplicationACLs());
    String appViewACLs =
        submissionContext
            .getAMContainerSpec()
            .getApplicationACLs()
            .get(ApplicationAccessType.VIEW_APP);
    rmContext
        .getSystemMetricsPublisher()
        .appACLsUpdated(application, appViewACLs, System.currentTimeMillis());
    return application;
  }
예제 #2
0
  private RMAppImpl createAndPopulateNewRMApp(
      ApplicationSubmissionContext submissionContext, long submitTime, String user)
      throws YarnException {
    ApplicationId applicationId = submissionContext.getApplicationId();
    validateResourceRequest(submissionContext);
    // Create RMApp
    RMAppImpl application =
        new RMAppImpl(
            applicationId,
            rmContext,
            this.conf,
            submissionContext.getApplicationName(),
            user,
            submissionContext.getQueue(),
            submissionContext,
            this.scheduler,
            this.masterService,
            submitTime,
            submissionContext.getApplicationType(),
            submissionContext.getApplicationTags());

    // Concurrent app submissions with same applicationId will fail here
    // Concurrent app submissions with different applicationIds will not
    // influence each other
    if (rmContext.getRMApps().putIfAbsent(applicationId, application) != null) {
      String message =
          "Application with id " + applicationId + " is already present! Cannot add a duplicate!";
      LOG.warn(message);
      throw RPCUtil.getRemoteException(message);
    }
    // Inform the ACLs Manager
    this.applicationACLsManager.addApplication(
        applicationId, submissionContext.getAMContainerSpec().getApplicationACLs());
    return application;
  }
  /**
   * Start a Tez Session
   *
   * @throws TezException
   * @throws IOException
   */
  public synchronized void start() throws TezException, IOException {
    yarnClient = YarnClient.createYarnClient();
    yarnClient.init(sessionConfig.getYarnConfiguration());
    yarnClient.start();

    tezJarResources =
        TezClientUtils.setupTezJarsLocalResources(sessionConfig.getTezConfiguration());

    try {
      if (applicationId == null) {
        applicationId =
            yarnClient.createApplication().getNewApplicationResponse().getApplicationId();
      }

      ApplicationSubmissionContext appContext =
          TezClientUtils.createApplicationSubmissionContext(
              sessionConfig.getTezConfiguration(),
              applicationId,
              null,
              sessionName,
              sessionConfig.getAMConfiguration(),
              tezJarResources);
      // Set Tez Sessions to not retry on AM crashes
      appContext.setMaxAppAttempts(1);
      tezConfPBLRsrc =
          appContext
              .getAMContainerSpec()
              .getLocalResources()
              .get(TezConfiguration.TEZ_PB_BINARY_CONF_NAME);
      yarnClient.submitApplication(appContext);
    } catch (YarnException e) {
      throw new TezException(e);
    }
    sessionStarted = true;
  }
예제 #4
0
 private Credentials parseCredentials(ApplicationSubmissionContext application)
     throws IOException {
   Credentials credentials = new Credentials();
   DataInputByteBuffer dibb = new DataInputByteBuffer();
   ByteBuffer tokens = application.getAMContainerSpec().getTokens();
   if (tokens != null) {
     dibb.reset(tokens);
     credentials.readTokenStorageStream(dibb);
     tokens.rewind();
   }
   return credentials;
 }
예제 #5
0
  @Override
  public ApplicationId submitApplication(ApplicationSubmissionContext appContext)
      throws YarnException, IOException {
    ApplicationId applicationId = appContext.getApplicationId();
    if (applicationId == null) {
      throw new ApplicationIdNotProvidedException(
          "ApplicationId is not provided in ApplicationSubmissionContext");
    }
    SubmitApplicationRequest request = Records.newRecord(SubmitApplicationRequest.class);
    request.setApplicationSubmissionContext(appContext);

    // Automatically add the timeline DT into the CLC
    // Only when the security and the timeline service are both enabled
    if (isSecurityEnabled() && timelineServiceEnabled) {
      addTimelineDelegationToken(appContext.getAMContainerSpec());
    }

    // TODO: YARN-1763:Handle RM failovers during the submitApplication call.
    rmClient.submitApplication(request);

    int pollCount = 0;
    long startTime = System.currentTimeMillis();
    EnumSet<YarnApplicationState> waitingStates =
        EnumSet.of(
            YarnApplicationState.NEW,
            YarnApplicationState.NEW_SAVING,
            YarnApplicationState.SUBMITTED);
    EnumSet<YarnApplicationState> failToSubmitStates =
        EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.KILLED);
    while (true) {
      try {
        ApplicationReport appReport = getApplicationReport(applicationId);
        YarnApplicationState state = appReport.getYarnApplicationState();
        if (!waitingStates.contains(state)) {
          if (failToSubmitStates.contains(state)) {
            throw new YarnException(
                "Failed to submit " + applicationId + " to YARN : " + appReport.getDiagnostics());
          }
          LOG.info("Submitted application " + applicationId);
          break;
        }

        long elapsedMillis = System.currentTimeMillis() - startTime;
        if (enforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis) {
          throw new YarnException(
              "Timed out while waiting for application "
                  + applicationId
                  + " to be submitted successfully");
        }

        // Notify the client through the log every 10 poll, in case the client
        // is blocked here too long.
        if (++pollCount % 10 == 0) {
          LOG.info(
              "Application submission is not finished, "
                  + "submitted application "
                  + applicationId
                  + " is still in "
                  + state);
        }
        try {
          Thread.sleep(submitPollIntervalMillis);
        } catch (InterruptedException ie) {
          LOG.error(
              "Interrupted while waiting for application "
                  + applicationId
                  + " to be successfully submitted.");
        }
      } catch (ApplicationNotFoundException ex) {
        // FailOver or RM restart happens before RMStateStore saves
        // ApplicationState
        LOG.info(
            "Re-submit application "
                + applicationId
                + "with the "
                + "same ApplicationSubmissionContext");
        rmClient.submitApplication(request);
      }
    }

    return applicationId;
  }
예제 #6
0
  @Test
  public void testAutomaticTimelineDelegationTokenLoading() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
    TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier();
    final Token<TimelineDelegationTokenIdentifier> dToken =
        new Token<TimelineDelegationTokenIdentifier>(
            timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text());
    // crate a mock client
    YarnClientImpl client =
        spy(
            new YarnClientImpl() {
              @Override
              protected void serviceInit(Configuration conf) throws Exception {
                if (getConfig()
                    .getBoolean(
                        YarnConfiguration.TIMELINE_SERVICE_ENABLED,
                        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
                  timelineServiceEnabled = true;
                  timelineClient = mock(TimelineClient.class);
                  when(timelineClient.getDelegationToken(any(String.class))).thenReturn(dToken);
                  timelineClient.init(getConfig());
                  timelineService = TimelineUtils.buildTimelineTokenService(getConfig());
                }
                this.setConfig(conf);
              }

              @Override
              protected void serviceStart() throws Exception {
                rmClient = mock(ApplicationClientProtocol.class);
              }

              @Override
              protected void serviceStop() throws Exception {}

              @Override
              public ApplicationReport getApplicationReport(ApplicationId appId) {
                ApplicationReport report = mock(ApplicationReport.class);
                when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.SUBMITTED);
                return report;
              }

              @Override
              public boolean isSecurityEnabled() {
                return true;
              }
            });
    client.init(conf);
    client.start();
    try {
      // when i == 0, timeline DT already exists, no need to get one more
      // when i == 1, timeline DT doesn't exist, need to get one more
      for (int i = 0; i < 2; ++i) {
        ApplicationSubmissionContext context = mock(ApplicationSubmissionContext.class);
        ApplicationId applicationId = ApplicationId.newInstance(0, i + 1);
        when(context.getApplicationId()).thenReturn(applicationId);
        DataOutputBuffer dob = new DataOutputBuffer();
        Credentials credentials = new Credentials();
        if (i == 0) {
          credentials.addToken(client.timelineService, dToken);
        }
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        ContainerLaunchContext clc =
            ContainerLaunchContext.newInstance(null, null, null, null, tokens, null);
        when(context.getAMContainerSpec()).thenReturn(clc);
        client.submitApplication(context);
        if (i == 0) {
          // GetTimelineDelegationToken shouldn't be called
          verify(client, never()).getTimelineDelegationToken();
        }
        // In either way, token should be there
        credentials = new Credentials();
        DataInputByteBuffer dibb = new DataInputByteBuffer();
        tokens = clc.getTokens();
        if (tokens != null) {
          dibb.reset(tokens);
          credentials.readTokenStorageStream(dibb);
          tokens.rewind();
        }
        Collection<Token<? extends TokenIdentifier>> dTokens = credentials.getAllTokens();
        Assert.assertEquals(1, dTokens.size());
        Assert.assertEquals(dToken, dTokens.iterator().next());
      }
    } finally {
      client.stop();
    }
  }
예제 #7
0
  @Test(timeout = 20000)
  public void testAMAdminCommandOpts() throws Exception {
    JobConf jobConf = new JobConf();

    jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true");
    jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m");

    YARNRunner yarnRunner = new YARNRunner(jobConf);

    File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
    OutputStream out = new FileOutputStream(jobxml);
    conf.writeXml(out);
    out.close();

    File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
    out = new FileOutputStream(jobsplit);
    out.close();

    File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO);
    out = new FileOutputStream(jobsplitmetainfo);
    out.close();

    File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE);
    out = new FileOutputStream(appTokens);
    out.close();

    ApplicationSubmissionContext submissionContext =
        yarnRunner.createApplicationSubmissionContext(
            jobConf, testWorkDir.toString(), new Credentials());

    ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
    List<String> commands = containerSpec.getCommands();

    int index = 0;
    int adminIndex = 0;
    int adminPos = -1;
    int userIndex = 0;
    int userPos = -1;

    for (String command : commands) {
      if (command != null) {
        adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true");
        if (adminPos >= 0) adminIndex = index;

        userPos = command.indexOf("-Xmx1024m");
        if (userPos >= 0) userIndex = index;
      }

      index++;
    }

    // Check both admin java opts and user java opts are in the commands
    assertTrue("AM admin command opts not in the commands.", adminPos > 0);
    assertTrue("AM user command opts not in the commands.", userPos > 0);

    // Check the admin java opts is before user java opts in the commands
    if (adminIndex == userIndex) {
      assertTrue("AM admin command opts is after user command opts.", adminPos < userPos);
    } else {
      assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex);
    }
  }