Esempio n. 1
0
  @SuppressWarnings("deprecation")
  @Override
  protected void serviceInit(Configuration conf) throws Exception {
    asyncApiPollIntervalMillis =
        conf.getLong(
            YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS,
            YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
    asyncApiPollTimeoutMillis =
        conf.getLong(
            YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
            YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS);
    submitPollIntervalMillis = asyncApiPollIntervalMillis;
    if (conf.get(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS) != null) {
      submitPollIntervalMillis =
          conf.getLong(
              YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
              YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
    }

    if (conf.getBoolean(
        YarnConfiguration.APPLICATION_HISTORY_ENABLED,
        YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
      historyServiceEnabled = true;
      historyClient = AHSClient.createAHSClient();
      historyClient.init(conf);
    }

    if (conf.getBoolean(
        YarnConfiguration.TIMELINE_SERVICE_ENABLED,
        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
      timelineServiceEnabled = true;
      timelineClient = createTimelineClient();
      timelineClient.init(conf);
      timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
      timelineService = TimelineUtils.buildTimelineTokenService(conf);
    }

    timelineServiceBestEffort =
        conf.getBoolean(
            YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);
    super.serviceInit(conf);
  }
Esempio n. 2
0
 @Override
 protected void serviceStop() throws Exception {
   if (this.rmClient != null) {
     RPC.stopProxy(this.rmClient);
   }
   if (historyServiceEnabled) {
     historyClient.stop();
   }
   if (timelineServiceEnabled) {
     timelineClient.stop();
   }
   super.serviceStop();
 }
Esempio n. 3
0
 @Override
 protected void serviceStart() throws Exception {
   try {
     rmClient = ClientRMProxy.createRMProxy(getConfig(), ApplicationClientProtocol.class);
     if (historyServiceEnabled) {
       historyClient.start();
     }
     if (timelineServiceEnabled) {
       timelineClient.start();
     }
   } catch (IOException e) {
     throw new YarnRuntimeException(e);
   }
   super.serviceStart();
 }
Esempio n. 4
0
 @Override
 public ContainerReport getContainerReport(ContainerId containerId)
     throws YarnException, IOException {
   try {
     GetContainerReportRequest request = Records.newRecord(GetContainerReportRequest.class);
     request.setContainerId(containerId);
     GetContainerReportResponse response = rmClient.getContainerReport(request);
     return response.getContainerReport();
   } catch (YarnException e) {
     if (!historyServiceEnabled) {
       // Just throw it as usual if historyService is not enabled.
       throw e;
     }
     // Even if history-service is enabled, treat all exceptions still the same
     // except the following
     if (e.getClass() != ApplicationNotFoundException.class
         && e.getClass() != ContainerNotFoundException.class) {
       throw e;
     }
     return historyClient.getContainerReport(containerId);
   }
 }
Esempio n. 5
0
 @Override
 public List<ApplicationAttemptReport> getApplicationAttempts(ApplicationId appId)
     throws YarnException, IOException {
   try {
     GetApplicationAttemptsRequest request =
         Records.newRecord(GetApplicationAttemptsRequest.class);
     request.setApplicationId(appId);
     GetApplicationAttemptsResponse response = rmClient.getApplicationAttempts(request);
     return response.getApplicationAttemptList();
   } catch (YarnException e) {
     if (!historyServiceEnabled) {
       // Just throw it as usual if historyService is not enabled.
       throw e;
     }
     // Even if history-service is enabled, treat all exceptions still the same
     // except the following
     if (e.getClass() != ApplicationNotFoundException.class) {
       throw e;
     }
     return historyClient.getApplicationAttempts(appId);
   }
 }
Esempio n. 6
0
  @Override
  public List<ContainerReport> getContainers(ApplicationAttemptId applicationAttemptId)
      throws YarnException, IOException {
    List<ContainerReport> containersForAttempt = new ArrayList<ContainerReport>();
    boolean appNotFoundInRM = false;
    try {
      GetContainersRequest request = Records.newRecord(GetContainersRequest.class);
      request.setApplicationAttemptId(applicationAttemptId);
      GetContainersResponse response = rmClient.getContainers(request);
      containersForAttempt.addAll(response.getContainerList());
    } catch (YarnException e) {
      if (e.getClass() != ApplicationNotFoundException.class || !historyServiceEnabled) {
        // If Application is not in RM and history service is enabled then we
        // need to check with history service else throw exception.
        throw e;
      }
      appNotFoundInRM = true;
    }

    if (historyServiceEnabled) {
      // Check with AHS even if found in RM because to capture info of finished
      // containers also
      List<ContainerReport> containersListFromAHS = null;
      try {
        containersListFromAHS = historyClient.getContainers(applicationAttemptId);
      } catch (IOException e) {
        // History service access might be enabled but system metrics publisher
        // is disabled hence app not found exception is possible
        if (appNotFoundInRM) {
          // app not found in bothM and RM then propagate the exception.
          throw e;
        }
      }

      if (null != containersListFromAHS && containersListFromAHS.size() > 0) {
        // remove duplicates

        Set<ContainerId> containerIdsToBeKeptFromAHS = new HashSet<ContainerId>();
        Iterator<ContainerReport> tmpItr = containersListFromAHS.iterator();
        while (tmpItr.hasNext()) {
          containerIdsToBeKeptFromAHS.add(tmpItr.next().getContainerId());
        }

        Iterator<ContainerReport> rmContainers = containersForAttempt.iterator();
        while (rmContainers.hasNext()) {
          ContainerReport tmp = rmContainers.next();
          containerIdsToBeKeptFromAHS.remove(tmp.getContainerId());
          // Remove containers from AHS as container from RM will have latest
          // information
        }

        if (containerIdsToBeKeptFromAHS.size() > 0
            && containersListFromAHS.size() != containerIdsToBeKeptFromAHS.size()) {
          Iterator<ContainerReport> containersFromHS = containersListFromAHS.iterator();
          while (containersFromHS.hasNext()) {
            ContainerReport containerReport = containersFromHS.next();
            if (containerIdsToBeKeptFromAHS.contains(containerReport.getContainerId())) {
              containersForAttempt.add(containerReport);
            }
          }
        } else if (containersListFromAHS.size() == containerIdsToBeKeptFromAHS.size()) {
          containersForAttempt.addAll(containersListFromAHS);
        }
      }
    }
    return containersForAttempt;
  }