public void submitApplication(
ApplicationId appId,
String appName,
Map<String, String> env,
Map<String, LocalResource> localResources,
List<String> commands,
int memory)
throws URISyntaxException, IOException {
if (clientResourceManager == null)
throw new IllegalStateException(
"Cannot submit an application without connecting to resource manager!");
ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class);
appCtx.setApplicationId(appId);
appCtx.setApplicationName(appName);
appCtx.setQueue("default");
appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
// System.out.println( "Based on my current user I am: " +
// UserGroupInformation.getCurrentUser().getShortUserName() );
Priority prio = Records.newRecord(Priority.class);
prio.setPriority(0);
appCtx.setPriority(prio);
/*
for (Map.Entry<String, LocalResource> entry : localResources.entrySet())
{
System.out.println("IR:RM: " + entry.getKey() + "/" + entry.getValue().getResource());
}
*/
/*
for (Map.Entry<String, String> entry : env.entrySet())
{
System.out.println("IR:ResourceManager -> Env vars: " + entry.getKey() + "/" + entry.getValue() );
}
*/
// Launch ctx
ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class);
containerCtx.setLocalResources(localResources);
containerCtx.setCommands(commands);
containerCtx.setEnvironment(env);
containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(memory);
containerCtx.setResource(capability);
appCtx.setAMContainerSpec(containerCtx);
SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class);
submitReq.setApplicationSubmissionContext(appCtx);
LOG.info("Submitting application to ASM");
clientResourceManager.submitApplication(submitReq);
// Don't return anything, ASM#submit returns an empty response
}
public ApplicationId submitApplication(ApplicationSubmissionContext appContext)
throws IOException {
appContext.setApplicationId(applicationId);
SubmitApplicationRequest request =
recordFactory.newRecordInstance(SubmitApplicationRequest.class);
request.setApplicationSubmissionContext(appContext);
applicationsManager.submitApplication(request);
LOG.info("Submitted application " + applicationId + " to ResourceManager" + " at " + rmAddress);
return applicationId;
}
@Test(timeout = 20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(conf);
MockRM rm = new MockRM(conf);
ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes());
ContainerLaunchContext amContainer =
ContainerLaunchContext.newInstance(
new HashMap<String, LocalResource>(),
new HashMap<String, String>(),
new ArrayList<String>(),
new HashMap<String, ByteBuffer>(),
tokens,
new HashMap<ApplicationAccessType, String>());
ApplicationSubmissionContext appSubContext =
ApplicationSubmissionContext.newInstance(
ApplicationId.newInstance(1234121, 0),
"BOGUS",
"default",
Priority.UNDEFINED,
amContainer,
false,
true,
1,
Resource.newInstance(1024, 1),
"BOGUS");
SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext);
try {
rm.getClientRMService().submitApplication(request);
fail("Error was excepted.");
} catch (YarnException e) {
Assert.assertTrue(e.getMessage().contains("Bad header found in token storage"));
}
}
private ApplicationId submitAppAndGetAppId(String submitter, String queueName, boolean setupACLs)
throws Exception {
GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.newInstance();
ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
ApplicationId applicationId =
submitterClient.getNewApplication(newAppRequest).getApplicationId();
Resource resource = BuilderUtils.newResource(1024, 1);
Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
ContainerLaunchContext amContainerSpec =
ContainerLaunchContext.newInstance(null, null, null, null, null, acls);
ApplicationSubmissionContext appSubmissionContext =
ApplicationSubmissionContext.newInstance(
applicationId,
"applicationName",
queueName,
null,
amContainerSpec,
false,
true,
1,
resource,
"applicationType");
appSubmissionContext.setApplicationId(applicationId);
appSubmissionContext.setQueue(queueName);
SubmitApplicationRequest submitRequest =
SubmitApplicationRequest.newInstance(appSubmissionContext);
submitterClient.submitApplication(submitRequest);
resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
return applicationId;
}
@Override
public ApplicationId submitApplication(ApplicationSubmissionContext appContext)
throws YarnException, IOException {
ApplicationId applicationId = appContext.getApplicationId();
if (applicationId == null) {
throw new ApplicationIdNotProvidedException(
"ApplicationId is not provided in ApplicationSubmissionContext");
}
SubmitApplicationRequest request = Records.newRecord(SubmitApplicationRequest.class);
request.setApplicationSubmissionContext(appContext);
// Automatically add the timeline DT into the CLC
// Only when the security and the timeline service are both enabled
if (isSecurityEnabled() && timelineServiceEnabled) {
addTimelineDelegationToken(appContext.getAMContainerSpec());
}
// TODO: YARN-1763:Handle RM failovers during the submitApplication call.
rmClient.submitApplication(request);
int pollCount = 0;
long startTime = System.currentTimeMillis();
EnumSet<YarnApplicationState> waitingStates =
EnumSet.of(
YarnApplicationState.NEW,
YarnApplicationState.NEW_SAVING,
YarnApplicationState.SUBMITTED);
EnumSet<YarnApplicationState> failToSubmitStates =
EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.KILLED);
while (true) {
try {
ApplicationReport appReport = getApplicationReport(applicationId);
YarnApplicationState state = appReport.getYarnApplicationState();
if (!waitingStates.contains(state)) {
if (failToSubmitStates.contains(state)) {
throw new YarnException(
"Failed to submit " + applicationId + " to YARN : " + appReport.getDiagnostics());
}
LOG.info("Submitted application " + applicationId);
break;
}
long elapsedMillis = System.currentTimeMillis() - startTime;
if (enforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis) {
throw new YarnException(
"Timed out while waiting for application "
+ applicationId
+ " to be submitted successfully");
}
// Notify the client through the log every 10 poll, in case the client
// is blocked here too long.
if (++pollCount % 10 == 0) {
LOG.info(
"Application submission is not finished, "
+ "submitted application "
+ applicationId
+ " is still in "
+ state);
}
try {
Thread.sleep(submitPollIntervalMillis);
} catch (InterruptedException ie) {
LOG.error(
"Interrupted while waiting for application "
+ applicationId
+ " to be successfully submitted.");
}
} catch (ApplicationNotFoundException ex) {
// FailOver or RM restart happens before RMStateStore saves
// ApplicationState
LOG.info(
"Re-submit application "
+ applicationId
+ "with the "
+ "same ApplicationSubmissionContext");
rmClient.submitApplication(request);
}
}
return applicationId;
}