private ApplicationId submitAppAndGetAppId(String submitter, String queueName, boolean setupACLs)
      throws Exception {

    GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.newInstance();

    ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
    ApplicationId applicationId =
        submitterClient.getNewApplication(newAppRequest).getApplicationId();

    Resource resource = BuilderUtils.newResource(1024, 1);
    Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
    ContainerLaunchContext amContainerSpec =
        ContainerLaunchContext.newInstance(null, null, null, null, null, acls);

    ApplicationSubmissionContext appSubmissionContext =
        ApplicationSubmissionContext.newInstance(
            applicationId,
            "applicationName",
            queueName,
            null,
            amContainerSpec,
            false,
            true,
            1,
            resource,
            "applicationType");
    appSubmissionContext.setApplicationId(applicationId);
    appSubmissionContext.setQueue(queueName);

    SubmitApplicationRequest submitRequest =
        SubmitApplicationRequest.newInstance(appSubmissionContext);
    submitterClient.submitApplication(submitRequest);
    resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
    return applicationId;
  }
  private void verifyKillAppSuccess(
      String submitter, String killer, String queueName, boolean setupACLs) throws Exception {
    ApplicationId applicationId = submitAppAndGetAppId(submitter, queueName, setupACLs);

    final KillApplicationRequest finishAppRequest =
        KillApplicationRequest.newInstance(applicationId);

    ApplicationClientProtocol ownerClient = getRMClientForUser(killer);

    // Kill app as killer
    ownerClient.forceKillApplication(finishAppRequest);
    resourceManager.waitForState(applicationId, RMAppState.KILLED);
  }
예제 #3
0
 @Override
 public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
     throws YarnException, IOException {
   return rmClient
       .getLabelsToNodes(GetLabelsToNodesRequest.newInstance(labels))
       .getLabelsToNodes();
 }
예제 #4
0
 @Override
 public void moveApplicationAcrossQueues(ApplicationId appId, String queue)
     throws YarnException, IOException {
   MoveApplicationAcrossQueuesRequest request =
       MoveApplicationAcrossQueuesRequest.newInstance(appId, queue);
   rmClient.moveApplicationAcrossQueues(request);
 }
예제 #5
0
  @Override
  public void killApplication(ApplicationId applicationId) throws YarnException, IOException {
    KillApplicationRequest request = Records.newRecord(KillApplicationRequest.class);
    request.setApplicationId(applicationId);

    try {
      int pollCount = 0;
      long startTime = System.currentTimeMillis();

      while (true) {
        KillApplicationResponse response = rmClient.forceKillApplication(request);
        if (response.getIsKillCompleted()) {
          LOG.info("Killed application " + applicationId);
          break;
        }

        long elapsedMillis = System.currentTimeMillis() - startTime;
        if (enforceAsyncAPITimeout() && elapsedMillis >= this.asyncApiPollTimeoutMillis) {
          throw new YarnException(
              "Timed out while waiting for application " + applicationId + " to be killed.");
        }

        if (++pollCount % 10 == 0) {
          LOG.info("Waiting for application " + applicationId + " to be killed.");
        }
        Thread.sleep(asyncApiPollIntervalMillis);
      }
    } catch (InterruptedException e) {
      LOG.error("Interrupted while waiting for application " + applicationId + " to be killed.");
    }
  }
예제 #6
0
 @Override
 public Token getRMDelegationToken(Text renewer) throws YarnException, IOException {
   /* get the token from RM */
   GetDelegationTokenRequest rmDTRequest = Records.newRecord(GetDelegationTokenRequest.class);
   rmDTRequest.setRenewer(renewer.toString());
   GetDelegationTokenResponse response = rmClient.getDelegationToken(rmDTRequest);
   return response.getRMDelegationToken();
 }
  private void verifyGetClientAMToken(
      String submitter, String queueAdmin, String queueName, boolean setupACLs) throws Exception {
    ApplicationId applicationId = submitAppAndGetAppId(submitter, queueName, setupACLs);
    final GetApplicationReportRequest appReportRequest =
        GetApplicationReportRequest.newInstance(applicationId);

    ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
    ApplicationClientProtocol adMinUserClient = getRMClientForUser(queueAdmin);

    GetApplicationReportResponse submitterGetReport =
        submitterClient.getApplicationReport(appReportRequest);
    GetApplicationReportResponse adMinUserGetReport =
        adMinUserClient.getApplicationReport(appReportRequest);

    Assert.assertEquals(
        submitterGetReport.getApplicationReport().getClientToAMToken(),
        adMinUserGetReport.getApplicationReport().getClientToAMToken());
  }
예제 #8
0
  @Override
  public List<QueueInfo> getRootQueueInfos() throws YarnException, IOException {
    List<QueueInfo> queues = new ArrayList<QueueInfo>();

    QueueInfo rootQueue =
        rmClient.getQueueInfo(getQueueInfoRequest(ROOT, false, true, true)).getQueueInfo();
    getChildQueues(rootQueue, queues, false);
    return queues;
  }
예제 #9
0
  @Override
  public List<QueueInfo> getChildQueueInfos(String parent) throws YarnException, IOException {
    List<QueueInfo> queues = new ArrayList<QueueInfo>();

    QueueInfo parentQueue =
        rmClient.getQueueInfo(getQueueInfoRequest(parent, false, true, false)).getQueueInfo();
    getChildQueues(parentQueue, queues, true);
    return queues;
  }
예제 #10
0
 @Override
 public List<ApplicationReport> getApplications(
     Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates)
     throws YarnException, IOException {
   GetApplicationsRequest request =
       GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
   GetApplicationsResponse response = rmClient.getApplications(request);
   return response.getApplicationList();
 }
예제 #11
0
 @Override
 public List<NodeReport> getNodeReports(NodeState... states) throws YarnException, IOException {
   EnumSet<NodeState> statesSet =
       (states.length == 0) ? EnumSet.allOf(NodeState.class) : EnumSet.noneOf(NodeState.class);
   for (NodeState state : states) {
     statesSet.add(state);
   }
   GetClusterNodesRequest request = GetClusterNodesRequest.newInstance(statesSet);
   GetClusterNodesResponse response = rmClient.getClusterNodes(request);
   return response.getNodeReports();
 }
  private void verifyKillAppFailure(
      String submitter, String killer, String queueName, boolean setupACLs) throws Exception {

    ApplicationId applicationId = submitAppAndGetAppId(submitter, queueName, setupACLs);

    final KillApplicationRequest finishAppRequest =
        KillApplicationRequest.newInstance(applicationId);

    ApplicationClientProtocol killerClient = getRMClientForUser(killer);

    // Kill app as the killer
    try {
      killerClient.forceKillApplication(finishAppRequest);
      Assert.fail("App killing by the enemy should fail!!");
    } catch (YarnException e) {
      LOG.info("Got exception while killing app as the enemy", e);
      Assert.assertTrue(
          e.getMessage()
              .contains(
                  "User " + killer + " cannot perform operation MODIFY_APP on " + applicationId));
    }

    getRMClientForUser(submitter).forceKillApplication(finishAppRequest);
  }
 /**
  * Get all the nodes in the cluster, this method generate RPC
  *
  * @return host names
  * @throws YarnException
  */
 private List<String> getClusterNodes() throws YarnException {
   List<String> result = new ArrayList<String>();
   GetClusterNodesRequest clusterNodesReq = Records.newRecord(GetClusterNodesRequest.class);
   try {
     GetClusterNodesResponse clusterNodesResp =
         applicationsManager.getClusterNodes(clusterNodesReq);
     List<NodeReport> nodeReports = clusterNodesResp.getNodeReports();
     for (NodeReport nodeReport : nodeReports) {
       result.add(nodeReport.getNodeId().getHost());
     }
   } catch (IOException e) {
     LOG.error("error getting cluster nodes from AM");
     throw new YarnException(e);
   }
   return result;
 }
예제 #14
0
 @Override
 public ContainerReport getContainerReport(ContainerId containerId)
     throws YarnException, IOException {
   try {
     GetContainerReportRequest request = Records.newRecord(GetContainerReportRequest.class);
     request.setContainerId(containerId);
     GetContainerReportResponse response = rmClient.getContainerReport(request);
     return response.getContainerReport();
   } catch (YarnException e) {
     if (!historyServiceEnabled) {
       // Just throw it as usual if historyService is not enabled.
       throw e;
     }
     // Even if history-service is enabled, treat all exceptions still the same
     // except the following
     if (e.getClass() != ApplicationNotFoundException.class
         && e.getClass() != ContainerNotFoundException.class) {
       throw e;
     }
     return historyClient.getContainerReport(containerId);
   }
 }
예제 #15
0
 @Override
 public List<ApplicationAttemptReport> getApplicationAttempts(ApplicationId appId)
     throws YarnException, IOException {
   try {
     GetApplicationAttemptsRequest request =
         Records.newRecord(GetApplicationAttemptsRequest.class);
     request.setApplicationId(appId);
     GetApplicationAttemptsResponse response = rmClient.getApplicationAttempts(request);
     return response.getApplicationAttemptList();
   } catch (YarnException e) {
     if (!historyServiceEnabled) {
       // Just throw it as usual if historyService is not enabled.
       throw e;
     }
     // Even if history-service is enabled, treat all exceptions still the same
     // except the following
     if (e.getClass() != ApplicationNotFoundException.class) {
       throw e;
     }
     return historyClient.getApplicationAttempts(appId);
   }
 }
예제 #16
0
 @Override
 public ReservationUpdateResponse updateReservation(ReservationUpdateRequest request)
     throws YarnException, IOException {
   return rmClient.updateReservation(request);
 }
예제 #17
0
 @Override
 public ReservationSubmissionResponse submitReservation(ReservationSubmissionRequest request)
     throws YarnException, IOException {
   return rmClient.submitReservation(request);
 }
예제 #18
0
  @Override
  public List<ContainerReport> getContainers(ApplicationAttemptId applicationAttemptId)
      throws YarnException, IOException {
    List<ContainerReport> containersForAttempt = new ArrayList<ContainerReport>();
    boolean appNotFoundInRM = false;
    try {
      GetContainersRequest request = Records.newRecord(GetContainersRequest.class);
      request.setApplicationAttemptId(applicationAttemptId);
      GetContainersResponse response = rmClient.getContainers(request);
      containersForAttempt.addAll(response.getContainerList());
    } catch (YarnException e) {
      if (e.getClass() != ApplicationNotFoundException.class || !historyServiceEnabled) {
        // If Application is not in RM and history service is enabled then we
        // need to check with history service else throw exception.
        throw e;
      }
      appNotFoundInRM = true;
    }

    if (historyServiceEnabled) {
      // Check with AHS even if found in RM because to capture info of finished
      // containers also
      List<ContainerReport> containersListFromAHS = null;
      try {
        containersListFromAHS = historyClient.getContainers(applicationAttemptId);
      } catch (IOException e) {
        // History service access might be enabled but system metrics publisher
        // is disabled hence app not found exception is possible
        if (appNotFoundInRM) {
          // app not found in bothM and RM then propagate the exception.
          throw e;
        }
      }

      if (null != containersListFromAHS && containersListFromAHS.size() > 0) {
        // remove duplicates

        Set<ContainerId> containerIdsToBeKeptFromAHS = new HashSet<ContainerId>();
        Iterator<ContainerReport> tmpItr = containersListFromAHS.iterator();
        while (tmpItr.hasNext()) {
          containerIdsToBeKeptFromAHS.add(tmpItr.next().getContainerId());
        }

        Iterator<ContainerReport> rmContainers = containersForAttempt.iterator();
        while (rmContainers.hasNext()) {
          ContainerReport tmp = rmContainers.next();
          containerIdsToBeKeptFromAHS.remove(tmp.getContainerId());
          // Remove containers from AHS as container from RM will have latest
          // information
        }

        if (containerIdsToBeKeptFromAHS.size() > 0
            && containersListFromAHS.size() != containerIdsToBeKeptFromAHS.size()) {
          Iterator<ContainerReport> containersFromHS = containersListFromAHS.iterator();
          while (containersFromHS.hasNext()) {
            ContainerReport containerReport = containersFromHS.next();
            if (containerIdsToBeKeptFromAHS.contains(containerReport.getContainerId())) {
              containersForAttempt.add(containerReport);
            }
          }
        } else if (containersListFromAHS.size() == containerIdsToBeKeptFromAHS.size()) {
          containersForAttempt.addAll(containersListFromAHS);
        }
      }
    }
    return containersForAttempt;
  }
예제 #19
0
 private GetNewApplicationResponse getNewApplication() throws YarnException, IOException {
   GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class);
   return rmClient.getNewApplication(request);
 }
예제 #20
0
 @Override
 public ReservationDeleteResponse deleteReservation(ReservationDeleteRequest request)
     throws YarnException, IOException {
   return rmClient.deleteReservation(request);
 }
예제 #21
0
 @Override
 public YarnClusterMetrics getYarnClusterMetrics() throws YarnException, IOException {
   GetClusterMetricsRequest request = Records.newRecord(GetClusterMetricsRequest.class);
   GetClusterMetricsResponse response = rmClient.getClusterMetrics(request);
   return response.getClusterMetrics();
 }
예제 #22
0
  @Override
  public ApplicationId submitApplication(ApplicationSubmissionContext appContext)
      throws YarnException, IOException {
    ApplicationId applicationId = appContext.getApplicationId();
    if (applicationId == null) {
      throw new ApplicationIdNotProvidedException(
          "ApplicationId is not provided in ApplicationSubmissionContext");
    }
    SubmitApplicationRequest request = Records.newRecord(SubmitApplicationRequest.class);
    request.setApplicationSubmissionContext(appContext);

    // Automatically add the timeline DT into the CLC
    // Only when the security and the timeline service are both enabled
    if (isSecurityEnabled() && timelineServiceEnabled) {
      addTimelineDelegationToken(appContext.getAMContainerSpec());
    }

    // TODO: YARN-1763:Handle RM failovers during the submitApplication call.
    rmClient.submitApplication(request);

    int pollCount = 0;
    long startTime = System.currentTimeMillis();
    EnumSet<YarnApplicationState> waitingStates =
        EnumSet.of(
            YarnApplicationState.NEW,
            YarnApplicationState.NEW_SAVING,
            YarnApplicationState.SUBMITTED);
    EnumSet<YarnApplicationState> failToSubmitStates =
        EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.KILLED);
    while (true) {
      try {
        ApplicationReport appReport = getApplicationReport(applicationId);
        YarnApplicationState state = appReport.getYarnApplicationState();
        if (!waitingStates.contains(state)) {
          if (failToSubmitStates.contains(state)) {
            throw new YarnException(
                "Failed to submit " + applicationId + " to YARN : " + appReport.getDiagnostics());
          }
          LOG.info("Submitted application " + applicationId);
          break;
        }

        long elapsedMillis = System.currentTimeMillis() - startTime;
        if (enforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis) {
          throw new YarnException(
              "Timed out while waiting for application "
                  + applicationId
                  + " to be submitted successfully");
        }

        // Notify the client through the log every 10 poll, in case the client
        // is blocked here too long.
        if (++pollCount % 10 == 0) {
          LOG.info(
              "Application submission is not finished, "
                  + "submitted application "
                  + applicationId
                  + " is still in "
                  + state);
        }
        try {
          Thread.sleep(submitPollIntervalMillis);
        } catch (InterruptedException ie) {
          LOG.error(
              "Interrupted while waiting for application "
                  + applicationId
                  + " to be successfully submitted.");
        }
      } catch (ApplicationNotFoundException ex) {
        // FailOver or RM restart happens before RMStateStore saves
        // ApplicationState
        LOG.info(
            "Re-submit application "
                + applicationId
                + "with the "
                + "same ApplicationSubmissionContext");
        rmClient.submitApplication(request);
      }
    }

    return applicationId;
  }
예제 #23
0
 @Override
 public Map<NodeId, Set<String>> getNodeToLabels() throws YarnException, IOException {
   return rmClient.getNodeToLabels(GetNodesToLabelsRequest.newInstance()).getNodeToLabels();
 }
예제 #24
0
 @Override
 public QueueInfo getQueueInfo(String queueName) throws YarnException, IOException {
   GetQueueInfoRequest request = getQueueInfoRequest(queueName, true, false, false);
   Records.newRecord(GetQueueInfoRequest.class);
   return rmClient.getQueueInfo(request).getQueueInfo();
 }
예제 #25
0
 @Override
 public Set<String> getClusterNodeLabels() throws YarnException, IOException {
   return rmClient.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance()).getNodeLabels();
 }
예제 #26
0
 @Override
 public List<QueueUserACLInfo> getQueueAclsInfo() throws YarnException, IOException {
   GetQueueUserAclsInfoRequest request = Records.newRecord(GetQueueUserAclsInfoRequest.class);
   return rmClient.getQueueUserAcls(request).getUserAclsInfoList();
 }