public void submitApplication(
ApplicationId appId,
String appName,
Map<String, String> env,
Map<String, LocalResource> localResources,
List<String> commands,
int memory)
throws URISyntaxException, IOException {
if (clientResourceManager == null)
throw new IllegalStateException(
"Cannot submit an application without connecting to resource manager!");
ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class);
appCtx.setApplicationId(appId);
appCtx.setApplicationName(appName);
appCtx.setQueue("default");
appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
// System.out.println( "Based on my current user I am: " +
// UserGroupInformation.getCurrentUser().getShortUserName() );
Priority prio = Records.newRecord(Priority.class);
prio.setPriority(0);
appCtx.setPriority(prio);
/*
for (Map.Entry<String, LocalResource> entry : localResources.entrySet())
{
System.out.println("IR:RM: " + entry.getKey() + "/" + entry.getValue().getResource());
}
*/
/*
for (Map.Entry<String, String> entry : env.entrySet())
{
System.out.println("IR:ResourceManager -> Env vars: " + entry.getKey() + "/" + entry.getValue() );
}
*/
// Launch ctx
ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class);
containerCtx.setLocalResources(localResources);
containerCtx.setCommands(commands);
containerCtx.setEnvironment(env);
containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(memory);
containerCtx.setResource(capability);
appCtx.setAMContainerSpec(containerCtx);
SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class);
submitReq.setApplicationSubmissionContext(appCtx);
LOG.info("Submitting application to ASM");
clientResourceManager.submitApplication(submitReq);
// Don't return anything, ASM#submit returns an empty response
}
public void launchSupervisorOnContainer(Container container) throws IOException {
LOG.info("Connecting to ContainerManager for containerid=" + container.getId());
String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
LOG.info("Connecting to ContainerManager at " + cmIpPortStr);
ContainerManager proxy =
((ContainerManager) rpc.getProxy(ContainerManager.class, cmAddress, hadoopConf));
LOG.info("launchSupervisorOnContainer( id:" + container.getId() + " )");
ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
launchContext.setContainerId(container.getId());
launchContext.setResource(container.getResource());
try {
launchContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
} catch (IOException e) {
LOG.info(
"Getting current user info failed when trying to launch the container" + e.getMessage());
}
Map<String, String> env = new HashMap<String, String>();
launchContext.setEnvironment(env);
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
String stormVersion = Util.getStormVersion(this.storm_conf);
Path zip = new Path("/lib/storm/" + stormVersion + "/storm.zip");
FileSystem fs = FileSystem.get(this.hadoopConf);
localResources.put(
"storm",
Util.newYarnAppResource(
fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC));
String appHome = Util.getApplicationHomeForId(this.appAttemptId.toString());
Path dirDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf);
localResources.put("conf", Util.newYarnAppResource(fs, dirDst));
launchContext.setLocalResources(localResources);
List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf);
launchContext.setCommands(supervisorArgs);
StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class);
startRequest.setContainerLaunchContext(launchContext);
LOG.info(
"launchSupervisorOnContainer: startRequest prepared, calling startContainer. "
+ startRequest);
try {
StartContainerResponse response = proxy.startContainer(startRequest);
LOG.info("Got a start container response " + response);
} catch (Exception e) {
LOG.error("Caught an exception while trying to start a container", e);
System.exit(-1);
}
}
@Test
public void testSuccessfulContainerLaunch() throws InterruptedException, IOException {
FileContext localFS = FileContext.getLocalFSFileContext();
localFS.delete(new Path(localDir.getAbsolutePath()), true);
localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
localDir.mkdir();
localLogDir.mkdir();
remoteLogDir.mkdir();
Context context = new NMContext();
YarnConfiguration conf = new YarnConfiguration();
conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());
ContainerExecutor exec = new DefaultContainerExecutor();
exec.setConf(conf);
DeletionService del = new DeletionService(exec);
Dispatcher dispatcher = new AsyncDispatcher();
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
NodeManagerMetrics metrics = NodeManagerMetrics.create();
ContainerTokenSecretManager containerTokenSecretManager = new ContainerTokenSecretManager();
NodeStatusUpdater nodeStatusUpdater =
new NodeStatusUpdaterImpl(
context, dispatcher, healthChecker, metrics, containerTokenSecretManager) {
@Override
protected ResourceTracker getRMClient() {
return new LocalRMInterface();
};
@Override
protected void startStatusUpdater() {
return; // Don't start any updating thread.
}
};
DummyContainerManager containerManager =
new DummyContainerManager(
context,
exec,
del,
nodeStatusUpdater,
metrics,
containerTokenSecretManager,
new ApplicationACLsManager(conf),
dirsHandler);
containerManager.init(conf);
containerManager.start();
ContainerLaunchContext launchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
ContainerId cID = recordFactory.newRecordInstance(ContainerId.class);
ApplicationId applicationId = recordFactory.newRecordInstance(ApplicationId.class);
applicationId.setClusterTimestamp(0);
applicationId.setId(0);
ApplicationAttemptId applicationAttemptId =
recordFactory.newRecordInstance(ApplicationAttemptId.class);
applicationAttemptId.setApplicationId(applicationId);
applicationAttemptId.setAttemptId(0);
cID.setApplicationAttemptId(applicationAttemptId);
launchContext.setContainerId(cID);
launchContext.setUser("testing");
launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.RUNNING);
StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
stopRequest.setContainerId(cID);
containerManager.stopContainer(stopRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.COMPLETE);
containerManager.stop();
}
