@Override
protected void render(Block html) {
ContainerId containerID;
try {
containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
} catch (IllegalArgumentException e) {
html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
return;
}
DIV<Hamlet> div = html.div("#content");
Container container = this.nmContext.getContainers().get(containerID);
if (container == null) {
div.h1("Unknown Container. Container might have completed, "
+ "please go back to the previous page and retry.")._();
return;
}
ContainerStatus containerData = container.cloneAndGetContainerStatus();
int exitCode = containerData.getExitStatus();
String exiStatus =
(exitCode == YarnConfiguration.INVALID_CONTAINER_EXIT_STATUS) ?
"N/A" : String.valueOf(exitCode);
info("Container information")
._("ContainerID", $(CONTAINER_ID))
._("ContainerState", container.getContainerState())
._("ExitStatus", exiStatus)
._("Diagnostics", containerData.getDiagnostics())
._("User", container.getUser())
._("TotalMemoryNeeded",
container.getLaunchContext().getResource().getMemory())
._("logs", ujoin("containerlogs", $(CONTAINER_ID), container.getUser()),
"Link to logs");
html._(InfoBlock.class);
}
public static void main(String[] args) {
try {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
String containerIdStr = System.getenv(Environment.CONTAINER_ID.name());
String nodeHostString = System.getenv(Environment.NM_HOST.name());
String nodePortString = System.getenv(Environment.NM_PORT.name());
String nodeHttpPortString = System.getenv(Environment.NM_HTTP_PORT.name());
String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV);
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
long appSubmitTime = Long.parseLong(appSubmitTimeStr);
Configuration conf = new Configuration(new YarnConfiguration());
TezUtils.addUserSpecifiedTezConfiguration(conf);
String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name());
// Do not automatically close FileSystem objects so that in case of
// SIGTERM I have a chance to write out the job history. I'll be closing
// the objects myself.
conf.setBoolean("fs.automatic.close", false);
// Command line options
Options opts = new Options();
opts.addOption(
TezConstants.TEZ_SESSION_MODE_CLI_OPTION,
false,
"Run Tez Application Master in Session mode");
CommandLine cliParser = new GnuParser().parse(opts, args);
DAGAppMaster appMaster =
new DAGAppMaster(
applicationAttemptId,
containerId,
nodeHostString,
Integer.parseInt(nodePortString),
Integer.parseInt(nodeHttpPortString),
appSubmitTime,
cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION));
ShutdownHookManager.get()
.addShutdownHook(new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
initAndStartAppMaster(appMaster, conf, jobUserName);
} catch (Throwable t) {
LOG.fatal("Error starting DAGAppMaster", t);
System.exit(1);
}
}
public void fromProto(TaskAttemptStartedProto proto) {
this.taskAttemptId = TezTaskAttemptID.fromString(proto.getTaskAttemptId());
this.launchTime = proto.getStartTime();
this.containerId = ConverterUtils.toContainerId(proto.getContainerId());
this.nodeId = ConverterUtils.toNodeId(proto.getNodeId());
this.creationTime = proto.getCreationTime();
this.allocationTime = proto.getAllocationTime();
if (proto.hasCreationCausalTA()) {
this.creationCausalTA = TezTaskAttemptID.fromString(proto.getCreationCausalTA());
}
}
private ContainerId verifyAndGetContainerId(Block html) {
String containerIdStr = $(CONTAINER_ID);
if (containerIdStr == null || containerIdStr.isEmpty()) {
html.h1()._("Cannot get container logs without a ContainerId")._();
return null;
}
ContainerId containerId = null;
try {
containerId = ConverterUtils.toContainerId(containerIdStr);
} catch (IllegalArgumentException e) {
html.h1()
._("Cannot get container logs for invalid containerId: "
+ containerIdStr)._();
return null;
}
return containerId;
}
@GET
@Path("/containers/{containerid}")
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
public ContainerInfo getNodeContainer(@PathParam("containerid") String id) {
ContainerId containerId = null;
init();
try {
containerId = ConverterUtils.toContainerId(id);
} catch (Exception e) {
throw new BadRequestException("invalid container id, " + id);
}
Container container = nmContext.getContainers().get(containerId);
if (container == null) {
throw new NotFoundException("container with id, " + id + ", not found");
}
return new ContainerInfo(
this.nmContext, container, uriInfo.getBaseUri().toString(), webapp.name());
}
/**
* Initial DragonAppMaster Get and CheckOut necessary parameters from system environment eg:
* container_Id,host,port,http_port,submitTime
*
* @param args
*/
public static void main(String[] args) {
try {
String containerIdStr = System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV);
String nodeHostString = System.getenv(ApplicationConstants.NM_HOST_ENV);
String nodePortString = System.getenv(ApplicationConstants.NM_PORT_ENV);
String nodeHttpPortString = System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV);
String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
validateInputParam(containerIdStr, ApplicationConstants.AM_CONTAINER_ID_ENV);
validateInputParam(nodeHostString, ApplicationConstants.NM_HOST_ENV);
validateInputParam(nodePortString, ApplicationConstants.NM_PORT_ENV);
validateInputParam(nodeHttpPortString, ApplicationConstants.NM_HTTP_PORT_ENV);
validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV);
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
long appSubmitTime = Long.parseLong(appSubmitTimeStr);
DragonAppMaster appMaster =
new DragonAppMaster(
applicationAttemptId,
containerId,
nodeHostString,
Integer.parseInt(nodePortString),
Integer.parseInt(nodeHttpPortString),
appSubmitTime);
Runtime.getRuntime().addShutdownHook(new CompositeServiceShutdownHook(appMaster));
YarnConfiguration conf = new YarnConfiguration(new DragonConfiguration());
conf.addResource(new Path(DragonJobConfig.JOB_CONF_FILE));
String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name());
conf.set(DragonJobConfig.USER_NAME, jobUserName);
// Do not automatically close FileSystem objects so that in case of
// SIGTERM I have a chance to write out the job history. I'll be closing
// the objects myself.
conf.setBoolean("fs.automatic.close", false);
initAndStartAppMaster(appMaster, conf, jobUserName);
} catch (Throwable t) {
LOG.fatal("Error starting MRAppMaster", t);
System.exit(1);
}
}
public static void main(String[] args) throws Exception {
Map<String, String> env = System.getenv();
LOG.info("Starting app master with the following environment variables");
for (String key : env.keySet()) {
LOG.info(key + "\t\t=" + env.get(key));
}
Options opts;
opts = new Options();
opts.addOption("num_containers", true, "Number of containers");
// START ZOOKEEPER
String dataDir = "dataDir";
String logDir = "logDir";
IDefaultNameSpace defaultNameSpace =
new IDefaultNameSpace() {
@Override
public void createDefaultNameSpace(ZkClient zkClient) {}
};
try {
FileUtils.deleteDirectory(new File(dataDir));
FileUtils.deleteDirectory(new File(logDir));
} catch (IOException e) {
LOG.error(e);
}
final ZkServer server = new ZkServer(dataDir, logDir, defaultNameSpace);
server.start();
// start Generic AppMaster that interacts with Yarn RM
AppMasterConfig appMasterConfig = new AppMasterConfig();
String containerIdStr = appMasterConfig.getContainerId();
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
String configFile = AppMasterConfig.AppEnvironment.APP_SPEC_FILE.toString();
String className = appMasterConfig.getApplicationSpecFactory();
GenericApplicationMaster genericApplicationMaster = new GenericApplicationMaster(appAttemptID);
try {
genericApplicationMaster.start();
} catch (Exception e) {
LOG.error("Unable to start application master: ", e);
}
ApplicationSpecFactory factory = HelixYarnUtil.createInstance(className);
// TODO: Avoid setting static variable.
YarnProvisioner.applicationMaster = genericApplicationMaster;
YarnProvisioner.applicationMasterConfig = appMasterConfig;
ApplicationSpec applicationSpec = factory.fromYaml(new FileInputStream(configFile));
YarnProvisioner.applicationSpec = applicationSpec;
String zkAddress = appMasterConfig.getZKAddress();
String clusterName = appMasterConfig.getAppName();
// CREATE CLUSTER and setup the resources
// connect
ZkHelixConnection connection = new ZkHelixConnection(zkAddress);
connection.connect();
// create the cluster
ClusterId clusterId = ClusterId.from(clusterName);
ClusterAccessor clusterAccessor = connection.createClusterAccessor(clusterId);
StateModelDefinition statelessService =
new StateModelDefinition(StateModelConfigGenerator.generateConfigForStatelessService());
StateModelDefinition taskStateModel =
new StateModelDefinition(StateModelConfigGenerator.generateConfigForTaskStateModel());
clusterAccessor.createCluster(
new ClusterConfig.Builder(clusterId)
.addStateModelDefinition(statelessService)
.addStateModelDefinition(taskStateModel)
.build());
for (String service : applicationSpec.getServices()) {
String resourceName = service;
// add the resource with the local provisioner
ResourceId resourceId = ResourceId.from(resourceName);
ServiceConfig serviceConfig = applicationSpec.getServiceConfig(resourceName);
serviceConfig.setSimpleField("service_name", service);
int numContainers = serviceConfig.getIntField("num_containers", 1);
YarnProvisionerConfig provisionerConfig = new YarnProvisionerConfig(resourceId);
provisionerConfig.setNumContainers(numContainers);
AutoRebalanceModeISBuilder idealStateBuilder = new AutoRebalanceModeISBuilder(resourceId);
idealStateBuilder.setStateModelDefId(statelessService.getStateModelDefId());
idealStateBuilder.add(PartitionId.from(resourceId, "0"));
idealStateBuilder.setNumReplica(1);
ResourceConfig.Builder resourceConfigBuilder =
new ResourceConfig.Builder(ResourceId.from(resourceName));
ResourceConfig resourceConfig =
resourceConfigBuilder
.provisionerConfig(provisionerConfig)
.idealState(idealStateBuilder.build()) //
.build();
clusterAccessor.addResource(resourceConfig);
}
// start controller
ControllerId controllerId = ControllerId.from("controller1");
HelixController controller = connection.createController(clusterId, controllerId);
controller.start();
// Start any pre-specified jobs
List<TaskConfig> taskConfigs = applicationSpec.getTaskConfigs();
if (taskConfigs != null) {
YarnConfiguration conf = new YarnConfiguration();
FileSystem fs;
fs = FileSystem.get(conf);
for (TaskConfig taskConfig : taskConfigs) {
URI yamlUri = taskConfig.getYamlURI();
if (yamlUri != null && taskConfig.name != null) {
InputStream is =
readFromHDFS(
fs, taskConfig.name, yamlUri, applicationSpec, appAttemptID.getApplicationId());
Workflow workflow = Workflow.parse(is);
TaskDriver taskDriver = new TaskDriver(new ZKHelixManager(controller));
taskDriver.start(workflow);
}
}
}
Thread shutdownhook =
new Thread(
new Runnable() {
@Override
public void run() {
server.shutdown();
}
});
Runtime.getRuntime().addShutdownHook(shutdownhook);
Thread.sleep(10000);
}
public static void main(String[] args) throws Exception {
System.err.println("Inside the AM!!!");
LOG.info("Starting the AM!!!!");
Options opts = new Options();
opts.addOption(
"app_attempt_id", true, "App Attempt ID. Not to be used " + "unless for testing purposes");
CommandLine cl = new GnuParser().parse(opts, args);
Map<String, String> envs = System.getenv();
_appAttemptID = Records.newRecord(ApplicationAttemptId.class);
if (cl.hasOption("app_attempt_id")) {
String appIdStr = cl.getOptionValue("app_attempt_id", "");
_appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
} else if (envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) {
ContainerId containerId =
ConverterUtils.toContainerId(envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV));
_appAttemptID = containerId.getApplicationAttemptId();
} else {
throw new IllegalArgumentException(
"Container and application attempt IDs not set in the environment");
}
@SuppressWarnings("rawtypes")
Map storm_conf = Config.readStormConfig(null);
YarnConfiguration hadoopConf = new YarnConfiguration();
StormAMRMClient client = new StormAMRMClient(_appAttemptID, storm_conf, hadoopConf);
client.init(hadoopConf);
client.start();
BlockingQueue<Container> launcherQueue = new LinkedBlockingQueue<Container>();
try {
InetSocketAddress addr =
NetUtils.createSocketAddr(
hadoopConf.get(
YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
int port = Utils.getInt(storm_conf.get(Config.MASTER_THRIFT_PORT));
RegisterApplicationMasterResponse resp =
client.registerApplicationMaster(addr.getHostName(), port, null);
LOG.info("Got a registration response " + resp);
LOG.info("Max Capability " + resp.getMaximumResourceCapability());
client.setMaxResource(resp.getMaximumResourceCapability());
MasterServer server = new MasterServer(storm_conf, client);
LOG.info("Starting HB thread");
initAndStartHeartbeat(
client, launcherQueue, (Integer) storm_conf.get(Config.MASTER_HEARTBEAT_INTERVAL_MILLIS));
LOG.info("Starting launcher");
initAndStartLauncher(client, launcherQueue);
client.startAllSupervisors();
server.serve();
client.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "AllDone", null);
} finally {
client.stop();
}
System.err.println("See Ya!!!");
System.exit(0);
}
/**
* Parse command line options
*
* @param args Command line args
* @return Whether init successful and run should be invoked
* @throws ParseException
* @throws IOException
*/
public boolean init(String[] args) throws ParseException, IOException {
Options opts = new Options();
opts.addOption(
"app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes");
opts.addOption(
"shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs");
opts.addOption(
"container_memory",
true,
"Amount of memory in MB to be requested to run the shell command");
opts.addOption(
"container_vcores",
true,
"Amount of virtual cores to be requested to run the shell command");
opts.addOption(
"num_containers",
true,
"No. of containers on which the shell command needs to be executed");
opts.addOption("priority", true, "Application Priority. Default 0");
opts.addOption("debug", false, "Dump out debug information");
opts.addOption("help", false, "Print usage");
CommandLine cliParser = new GnuParser().parse(opts, args);
if (args.length == 0) {
printUsage(opts);
throw new IllegalArgumentException("No args specified for application master to initialize");
}
// Check whether customer log4j.properties file exists
if (fileExist(log4jPath)) {
try {
Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath);
} catch (Exception e) {
LOG.warn("Can not set up custom log4j properties. " + e);
}
}
if (cliParser.hasOption("help")) {
printUsage(opts);
return false;
}
if (cliParser.hasOption("debug")) {
dumpOutDebugInfo();
}
Map<String, String> envs = System.getenv();
if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
if (cliParser.hasOption("app_attempt_id")) {
String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
} else {
throw new IllegalArgumentException("Application Attempt Id not set in the environment");
}
} else {
ContainerId containerId =
ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
appAttemptID = containerId.getApplicationAttemptId();
}
if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
throw new RuntimeException(
ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment");
}
if (!envs.containsKey(Environment.NM_HOST.name())) {
throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment");
}
if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment");
}
if (!envs.containsKey(Environment.NM_PORT.name())) {
throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment");
}
LOG.info(
"Application master for app"
+ ", appId="
+ appAttemptID.getApplicationId().getId()
+ ", clustertimestamp="
+ appAttemptID.getApplicationId().getClusterTimestamp()
+ ", attemptId="
+ appAttemptID.getAttemptId());
if (!fileExist(shellCommandPath)
&& envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION).isEmpty()) {
throw new IllegalArgumentException(
"No shell command or shell script specified to be executed by application master");
}
if (fileExist(shellCommandPath)) {
shellCommand = readContent(shellCommandPath);
}
if (fileExist(shellArgsPath)) {
shellArgs = readContent(shellArgsPath);
}
if (cliParser.hasOption("shell_env")) {
String shellEnvs[] = cliParser.getOptionValues("shell_env");
for (String env : shellEnvs) {
env = env.trim();
int index = env.indexOf('=');
if (index == -1) {
shellEnv.put(env, "");
continue;
}
String key = env.substring(0, index);
String val = "";
if (index < (env.length() - 1)) {
val = env.substring(index + 1);
}
shellEnv.put(key, val);
}
}
if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) {
scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION);
if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) {
shellScriptPathTimestamp =
Long.valueOf(envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP));
}
if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)) {
shellScriptPathLen = Long.valueOf(envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
}
if (!scriptPath.isEmpty() && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) {
LOG.error(
"Illegal values in env for shell script path"
+ ", path="
+ scriptPath
+ ", len="
+ shellScriptPathLen
+ ", timestamp="
+ shellScriptPathTimestamp);
throw new IllegalArgumentException("Illegal values in env for shell script path");
}
}
containerMemory = Integer.parseInt(cliParser.getOptionValue("container_memory", "10"));
containerVirtualCores = Integer.parseInt(cliParser.getOptionValue("container_vcores", "1"));
numTotalContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "1"));
if (numTotalContainers == 0) {
throw new IllegalArgumentException("Cannot run distributed shell with no containers");
}
requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));
// Creating the Timeline Client
timelineClient = TimelineClient.createTimelineClient();
timelineClient.init(conf);
timelineClient.start();
return true;
}