@Test(timeout = 20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(conf);
MockRM rm = new MockRM(conf);
ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes());
ContainerLaunchContext amContainer =
ContainerLaunchContext.newInstance(
new HashMap<String, LocalResource>(),
new HashMap<String, String>(),
new ArrayList<String>(),
new HashMap<String, ByteBuffer>(),
tokens,
new HashMap<ApplicationAccessType, String>());
ApplicationSubmissionContext appSubContext =
ApplicationSubmissionContext.newInstance(
ApplicationId.newInstance(1234121, 0),
"BOGUS",
"default",
Priority.UNDEFINED,
amContainer,
false,
true,
1,
Resource.newInstance(1024, 1),
"BOGUS");
SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext);
try {
rm.getClientRMService().submitApplication(request);
fail("Error was excepted.");
} catch (YarnException e) {
Assert.assertTrue(e.getMessage().contains("Bad header found in token storage"));
}
}
@SuppressWarnings("deprecation")
@Test(timeout = 30000)
public void testSubmitApplication() {
Configuration conf = new Configuration();
conf.setLong(
YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS, 100); // speed up tests
final YarnClient client = new MockYarnClient();
client.init(conf);
client.start();
YarnApplicationState[] exitStates =
new YarnApplicationState[] {
YarnApplicationState.SUBMITTED,
YarnApplicationState.ACCEPTED,
YarnApplicationState.RUNNING,
YarnApplicationState.FINISHED,
YarnApplicationState.FAILED,
YarnApplicationState.KILLED
};
// Submit an application without ApplicationId provided
// Should get ApplicationIdNotProvidedException
ApplicationSubmissionContext contextWithoutApplicationId =
mock(ApplicationSubmissionContext.class);
try {
client.submitApplication(contextWithoutApplicationId);
Assert.fail("Should throw the ApplicationIdNotProvidedException");
} catch (YarnException e) {
Assert.assertTrue(e instanceof ApplicationIdNotProvidedException);
Assert.assertTrue(
e.getMessage().contains("ApplicationId is not provided in ApplicationSubmissionContext"));
} catch (IOException e) {
Assert.fail("IOException is not expected.");
}
// Submit the application with applicationId provided
// Should be successful
for (int i = 0; i < exitStates.length; ++i) {
ApplicationSubmissionContext context = mock(ApplicationSubmissionContext.class);
ApplicationId applicationId = ApplicationId.newInstance(System.currentTimeMillis(), i);
when(context.getApplicationId()).thenReturn(applicationId);
((MockYarnClient) client).setYarnApplicationState(exitStates[i]);
try {
client.submitApplication(context);
} catch (YarnException e) {
Assert.fail("Exception is not expected.");
} catch (IOException e) {
Assert.fail("Exception is not expected.");
}
verify(((MockYarnClient) client).mockReport, times(4 * i + 4)).getYarnApplicationState();
}
client.stop();
}
private void refreshAll() throws ServiceFailedException {
try {
refreshQueues(RefreshQueuesRequest.newInstance());
refreshNodes(RefreshNodesRequest.newInstance());
refreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequest.newInstance());
refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest.newInstance());
if (getConfig()
.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
refreshServiceAcls(RefreshServiceAclsRequest.newInstance());
}
} catch (YarnException ex) {
throw new ServiceFailedException(ex.getMessage());
} catch (IOException ex) {
throw new ServiceFailedException(ex.getMessage());
}
}
@Override
public ContainerReport getContainerReport(ContainerId containerId)
throws YarnException, IOException {
try {
GetContainerReportRequest request = Records.newRecord(GetContainerReportRequest.class);
request.setContainerId(containerId);
GetContainerReportResponse response = rmClient.getContainerReport(request);
return response.getContainerReport();
} catch (YarnException e) {
if (!historyServiceEnabled) {
// Just throw it as usual if historyService is not enabled.
throw e;
}
// Even if history-service is enabled, treat all exceptions still the same
// except the following
if (e.getClass() != ApplicationNotFoundException.class
&& e.getClass() != ContainerNotFoundException.class) {
throw e;
}
return historyClient.getContainerReport(containerId);
}
}
@Override
public List<ApplicationAttemptReport> getApplicationAttempts(ApplicationId appId)
throws YarnException, IOException {
try {
GetApplicationAttemptsRequest request =
Records.newRecord(GetApplicationAttemptsRequest.class);
request.setApplicationId(appId);
GetApplicationAttemptsResponse response = rmClient.getApplicationAttempts(request);
return response.getApplicationAttemptList();
} catch (YarnException e) {
if (!historyServiceEnabled) {
// Just throw it as usual if historyService is not enabled.
throw e;
}
// Even if history-service is enabled, treat all exceptions still the same
// except the following
if (e.getClass() != ApplicationNotFoundException.class) {
throw e;
}
return historyClient.getApplicationAttempts(appId);
}
}
private void verifyKillAppFailure(
String submitter, String killer, String queueName, boolean setupACLs) throws Exception {
ApplicationId applicationId = submitAppAndGetAppId(submitter, queueName, setupACLs);
final KillApplicationRequest finishAppRequest =
KillApplicationRequest.newInstance(applicationId);
ApplicationClientProtocol killerClient = getRMClientForUser(killer);
// Kill app as the killer
try {
killerClient.forceKillApplication(finishAppRequest);
Assert.fail("App killing by the enemy should fail!!");
} catch (YarnException e) {
LOG.info("Got exception while killing app as the enemy", e);
Assert.assertTrue(
e.getMessage()
.contains(
"User " + killer + " cannot perform operation MODIFY_APP on " + applicationId));
}
getRMClientForUser(submitter).forceKillApplication(finishAppRequest);
}
@Override
public List<ContainerReport> getContainers(ApplicationAttemptId applicationAttemptId)
throws YarnException, IOException {
List<ContainerReport> containersForAttempt = new ArrayList<ContainerReport>();
boolean appNotFoundInRM = false;
try {
GetContainersRequest request = Records.newRecord(GetContainersRequest.class);
request.setApplicationAttemptId(applicationAttemptId);
GetContainersResponse response = rmClient.getContainers(request);
containersForAttempt.addAll(response.getContainerList());
} catch (YarnException e) {
if (e.getClass() != ApplicationNotFoundException.class || !historyServiceEnabled) {
// If Application is not in RM and history service is enabled then we
// need to check with history service else throw exception.
throw e;
}
appNotFoundInRM = true;
}
if (historyServiceEnabled) {
// Check with AHS even if found in RM because to capture info of finished
// containers also
List<ContainerReport> containersListFromAHS = null;
try {
containersListFromAHS = historyClient.getContainers(applicationAttemptId);
} catch (IOException e) {
// History service access might be enabled but system metrics publisher
// is disabled hence app not found exception is possible
if (appNotFoundInRM) {
// app not found in bothM and RM then propagate the exception.
throw e;
}
}
if (null != containersListFromAHS && containersListFromAHS.size() > 0) {
// remove duplicates
Set<ContainerId> containerIdsToBeKeptFromAHS = new HashSet<ContainerId>();
Iterator<ContainerReport> tmpItr = containersListFromAHS.iterator();
while (tmpItr.hasNext()) {
containerIdsToBeKeptFromAHS.add(tmpItr.next().getContainerId());
}
Iterator<ContainerReport> rmContainers = containersForAttempt.iterator();
while (rmContainers.hasNext()) {
ContainerReport tmp = rmContainers.next();
containerIdsToBeKeptFromAHS.remove(tmp.getContainerId());
// Remove containers from AHS as container from RM will have latest
// information
}
if (containerIdsToBeKeptFromAHS.size() > 0
&& containersListFromAHS.size() != containerIdsToBeKeptFromAHS.size()) {
Iterator<ContainerReport> containersFromHS = containersListFromAHS.iterator();
while (containersFromHS.hasNext()) {
ContainerReport containerReport = containersFromHS.next();
if (containerIdsToBeKeptFromAHS.contains(containerReport.getContainerId())) {
containersForAttempt.add(containerReport);
}
}
} else if (containersListFromAHS.size() == containerIdsToBeKeptFromAHS.size()) {
containersForAttempt.addAll(containersListFromAHS);
}
}
}
return containersForAttempt;
}
