@SuppressWarnings("deprecation")
@Override
protected void serviceInit(Configuration conf) throws Exception {
asyncApiPollIntervalMillis =
conf.getLong(
YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
asyncApiPollTimeoutMillis =
conf.getLong(
YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS);
submitPollIntervalMillis = asyncApiPollIntervalMillis;
if (conf.get(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS) != null) {
submitPollIntervalMillis =
conf.getLong(
YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
}
if (conf.getBoolean(
YarnConfiguration.APPLICATION_HISTORY_ENABLED,
YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
historyServiceEnabled = true;
historyClient = AHSClient.createAHSClient();
historyClient.init(conf);
}
if (conf.getBoolean(
YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
timelineServiceEnabled = true;
timelineClient = createTimelineClient();
timelineClient.init(conf);
timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
timelineService = TimelineUtils.buildTimelineTokenService(conf);
}
timelineServiceBestEffort =
conf.getBoolean(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);
super.serviceInit(conf);
}
@Override
protected void serviceStop() throws Exception {
if (this.rmClient != null) {
RPC.stopProxy(this.rmClient);
}
if (historyServiceEnabled) {
historyClient.stop();
}
if (timelineServiceEnabled) {
timelineClient.stop();
}
super.serviceStop();
}
@Override
protected void serviceStart() throws Exception {
try {
rmClient = ClientRMProxy.createRMProxy(getConfig(), ApplicationClientProtocol.class);
if (historyServiceEnabled) {
historyClient.start();
}
if (timelineServiceEnabled) {
timelineClient.start();
}
} catch (IOException e) {
throw new YarnRuntimeException(e);
}
super.serviceStart();
}
@Override
public ContainerReport getContainerReport(ContainerId containerId)
throws YarnException, IOException {
try {
GetContainerReportRequest request = Records.newRecord(GetContainerReportRequest.class);
request.setContainerId(containerId);
GetContainerReportResponse response = rmClient.getContainerReport(request);
return response.getContainerReport();
} catch (YarnException e) {
if (!historyServiceEnabled) {
// Just throw it as usual if historyService is not enabled.
throw e;
}
// Even if history-service is enabled, treat all exceptions still the same
// except the following
if (e.getClass() != ApplicationNotFoundException.class
&& e.getClass() != ContainerNotFoundException.class) {
throw e;
}
return historyClient.getContainerReport(containerId);
}
}
@Override
public List<ApplicationAttemptReport> getApplicationAttempts(ApplicationId appId)
throws YarnException, IOException {
try {
GetApplicationAttemptsRequest request =
Records.newRecord(GetApplicationAttemptsRequest.class);
request.setApplicationId(appId);
GetApplicationAttemptsResponse response = rmClient.getApplicationAttempts(request);
return response.getApplicationAttemptList();
} catch (YarnException e) {
if (!historyServiceEnabled) {
// Just throw it as usual if historyService is not enabled.
throw e;
}
// Even if history-service is enabled, treat all exceptions still the same
// except the following
if (e.getClass() != ApplicationNotFoundException.class) {
throw e;
}
return historyClient.getApplicationAttempts(appId);
}
}
@Override
public List<ContainerReport> getContainers(ApplicationAttemptId applicationAttemptId)
throws YarnException, IOException {
List<ContainerReport> containersForAttempt = new ArrayList<ContainerReport>();
boolean appNotFoundInRM = false;
try {
GetContainersRequest request = Records.newRecord(GetContainersRequest.class);
request.setApplicationAttemptId(applicationAttemptId);
GetContainersResponse response = rmClient.getContainers(request);
containersForAttempt.addAll(response.getContainerList());
} catch (YarnException e) {
if (e.getClass() != ApplicationNotFoundException.class || !historyServiceEnabled) {
// If Application is not in RM and history service is enabled then we
// need to check with history service else throw exception.
throw e;
}
appNotFoundInRM = true;
}
if (historyServiceEnabled) {
// Check with AHS even if found in RM because to capture info of finished
// containers also
List<ContainerReport> containersListFromAHS = null;
try {
containersListFromAHS = historyClient.getContainers(applicationAttemptId);
} catch (IOException e) {
// History service access might be enabled but system metrics publisher
// is disabled hence app not found exception is possible
if (appNotFoundInRM) {
// app not found in bothM and RM then propagate the exception.
throw e;
}
}
if (null != containersListFromAHS && containersListFromAHS.size() > 0) {
// remove duplicates
Set<ContainerId> containerIdsToBeKeptFromAHS = new HashSet<ContainerId>();
Iterator<ContainerReport> tmpItr = containersListFromAHS.iterator();
while (tmpItr.hasNext()) {
containerIdsToBeKeptFromAHS.add(tmpItr.next().getContainerId());
}
Iterator<ContainerReport> rmContainers = containersForAttempt.iterator();
while (rmContainers.hasNext()) {
ContainerReport tmp = rmContainers.next();
containerIdsToBeKeptFromAHS.remove(tmp.getContainerId());
// Remove containers from AHS as container from RM will have latest
// information
}
if (containerIdsToBeKeptFromAHS.size() > 0
&& containersListFromAHS.size() != containerIdsToBeKeptFromAHS.size()) {
Iterator<ContainerReport> containersFromHS = containersListFromAHS.iterator();
while (containersFromHS.hasNext()) {
ContainerReport containerReport = containersFromHS.next();
if (containerIdsToBeKeptFromAHS.contains(containerReport.getContainerId())) {
containersForAttempt.add(containerReport);
}
}
} else if (containersListFromAHS.size() == containerIdsToBeKeptFromAHS.size()) {
containersForAttempt.addAll(containersListFromAHS);
}
}
}
return containersForAttempt;
}
