public static String appReportToString(ApplicationReport r, String separator) {
StringBuilder builder = new StringBuilder(512);
builder.append("application ").append(r.getName()).append("/").append(r.getApplicationType());
builder.append(separator).append("state: ").append(r.getYarnApplicationState());
builder.append(separator).append("URL: ").append(r.getTrackingUrl());
builder
.append(separator)
.append("Started ")
.append(new Date(r.getStartTime()).toLocaleString());
long finishTime = r.getFinishTime();
if (finishTime > 0) {
builder.append(separator).append("Finished ").append(new Date(finishTime).toLocaleString());
}
builder
.append(separator)
.append("RPC :")
.append(r.getHost())
.append(':')
.append(r.getRpcPort());
String diagnostics = r.getDiagnostics();
if (!diagnostics.isEmpty()) {
builder.append(separator).append("Diagnostics :").append(diagnostics);
}
return builder.toString();
}
@Override
public boolean hasFailed() {
if (!isConnected) {
throw new IllegalStateException("The cluster has been connected to the ApplicationMaster.");
}
if (pollingRunner == null) {
LOG.warn(
"FlinkYarnCluster.hasFailed() has been called on an uninitialized cluster."
+ "The system might be in an erroneous state");
}
ApplicationReport lastReport = pollingRunner.getLastReport();
if (lastReport == null) {
LOG.warn(
"FlinkYarnCluster.hasFailed() has been called on a cluster that didn't receive a status so far."
+ "The system might be in an erroneous state");
return false;
} else {
YarnApplicationState appState = lastReport.getYarnApplicationState();
boolean status =
(appState == YarnApplicationState.FAILED || appState == YarnApplicationState.KILLED);
if (status) {
LOG.warn("YARN reported application state {}", appState);
LOG.warn("Diagnostics: {}", lastReport.getDiagnostics());
}
return status;
}
}
@Override
public String getDiagnostics() {
if (!isConnected) {
throw new IllegalStateException("The cluster has been connected to the ApplicationMaster.");
}
if (!hasFailed()) {
LOG.warn("getDiagnostics() called for cluster which is not in failed state");
}
ApplicationReport lastReport = pollingRunner.getLastReport();
if (lastReport == null) {
LOG.warn("Last report is null");
return null;
} else {
return lastReport.getDiagnostics();
}
}
@Test(timeout = 20000)
public void testJobSubmissionFailure() throws Exception {
when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class)))
.thenReturn(appId);
ApplicationReport report = mock(ApplicationReport.class);
when(report.getApplicationId()).thenReturn(appId);
when(report.getDiagnostics()).thenReturn(failString);
when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED);
when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report);
Credentials credentials = new Credentials();
File jobxml = new File(testWorkDir, "job.xml");
OutputStream out = new FileOutputStream(jobxml);
conf.writeXml(out);
out.close();
try {
yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
} catch (IOException io) {
LOG.info("Logging exception:", io);
assertTrue(io.getLocalizedMessage().contains(failString));
}
}
@Override
public ApplicationId submitApplication(ApplicationSubmissionContext appContext)
throws YarnException, IOException {
ApplicationId applicationId = appContext.getApplicationId();
if (applicationId == null) {
throw new ApplicationIdNotProvidedException(
"ApplicationId is not provided in ApplicationSubmissionContext");
}
SubmitApplicationRequest request = Records.newRecord(SubmitApplicationRequest.class);
request.setApplicationSubmissionContext(appContext);
// Automatically add the timeline DT into the CLC
// Only when the security and the timeline service are both enabled
if (isSecurityEnabled() && timelineServiceEnabled) {
addTimelineDelegationToken(appContext.getAMContainerSpec());
}
// TODO: YARN-1763:Handle RM failovers during the submitApplication call.
rmClient.submitApplication(request);
int pollCount = 0;
long startTime = System.currentTimeMillis();
EnumSet<YarnApplicationState> waitingStates =
EnumSet.of(
YarnApplicationState.NEW,
YarnApplicationState.NEW_SAVING,
YarnApplicationState.SUBMITTED);
EnumSet<YarnApplicationState> failToSubmitStates =
EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.KILLED);
while (true) {
try {
ApplicationReport appReport = getApplicationReport(applicationId);
YarnApplicationState state = appReport.getYarnApplicationState();
if (!waitingStates.contains(state)) {
if (failToSubmitStates.contains(state)) {
throw new YarnException(
"Failed to submit " + applicationId + " to YARN : " + appReport.getDiagnostics());
}
LOG.info("Submitted application " + applicationId);
break;
}
long elapsedMillis = System.currentTimeMillis() - startTime;
if (enforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis) {
throw new YarnException(
"Timed out while waiting for application "
+ applicationId
+ " to be submitted successfully");
}
// Notify the client through the log every 10 poll, in case the client
// is blocked here too long.
if (++pollCount % 10 == 0) {
LOG.info(
"Application submission is not finished, "
+ "submitted application "
+ applicationId
+ " is still in "
+ state);
}
try {
Thread.sleep(submitPollIntervalMillis);
} catch (InterruptedException ie) {
LOG.error(
"Interrupted while waiting for application "
+ applicationId
+ " to be successfully submitted.");
}
} catch (ApplicationNotFoundException ex) {
// FailOver or RM restart happens before RMStateStore saves
// ApplicationState
LOG.info(
"Re-submit application "
+ applicationId
+ "with the "
+ "same ApplicationSubmissionContext");
rmClient.submitApplication(request);
}
}
return applicationId;
}
