@Override protected void render(Block html) { ContainerId containerID; try { containerID = ConverterUtils.toContainerId($(CONTAINER_ID)); } catch (IllegalArgumentException e) { html.p()._("Invalid containerId " + $(CONTAINER_ID))._(); return; } DIV<Hamlet> div = html.div("#content"); Container container = this.nmContext.getContainers().get(containerID); if (container == null) { div.h1("Unknown Container. Container might have completed, " + "please go back to the previous page and retry.")._(); return; } ContainerStatus containerData = container.cloneAndGetContainerStatus(); int exitCode = containerData.getExitStatus(); String exiStatus = (exitCode == YarnConfiguration.INVALID_CONTAINER_EXIT_STATUS) ? "N/A" : String.valueOf(exitCode); info("Container information") ._("ContainerID", $(CONTAINER_ID)) ._("ContainerState", container.getContainerState()) ._("ExitStatus", exiStatus) ._("Diagnostics", containerData.getDiagnostics()) ._("User", container.getUser()) ._("TotalMemoryNeeded", container.getLaunchContext().getResource().getMemory()) ._("logs", ujoin("containerlogs", $(CONTAINER_ID), container.getUser()), "Link to logs"); html._(InfoBlock.class); }
public static void main(String[] args) { try { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); String containerIdStr = System.getenv(Environment.CONTAINER_ID.name()); String nodeHostString = System.getenv(Environment.NM_HOST.name()); String nodePortString = System.getenv(Environment.NM_PORT.name()); String nodeHttpPortString = System.getenv(Environment.NM_HTTP_PORT.name()); String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId(); long appSubmitTime = Long.parseLong(appSubmitTimeStr); Configuration conf = new Configuration(new YarnConfiguration()); TezUtils.addUserSpecifiedTezConfiguration(conf); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); // Do not automatically close FileSystem objects so that in case of // SIGTERM I have a chance to write out the job history. I'll be closing // the objects myself. conf.setBoolean("fs.automatic.close", false); // Command line options Options opts = new Options(); opts.addOption( TezConstants.TEZ_SESSION_MODE_CLI_OPTION, false, "Run Tez Application Master in Session mode"); CommandLine cliParser = new GnuParser().parse(opts, args); DAGAppMaster appMaster = new DAGAppMaster( applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), Integer.parseInt(nodeHttpPortString), appSubmitTime, cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION)); ShutdownHookManager.get() .addShutdownHook(new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY); initAndStartAppMaster(appMaster, conf, jobUserName); } catch (Throwable t) { LOG.fatal("Error starting DAGAppMaster", t); System.exit(1); } }
public void fromProto(TaskAttemptStartedProto proto) { this.taskAttemptId = TezTaskAttemptID.fromString(proto.getTaskAttemptId()); this.launchTime = proto.getStartTime(); this.containerId = ConverterUtils.toContainerId(proto.getContainerId()); this.nodeId = ConverterUtils.toNodeId(proto.getNodeId()); this.creationTime = proto.getCreationTime(); this.allocationTime = proto.getAllocationTime(); if (proto.hasCreationCausalTA()) { this.creationCausalTA = TezTaskAttemptID.fromString(proto.getCreationCausalTA()); } }
private ContainerId verifyAndGetContainerId(Block html) { String containerIdStr = $(CONTAINER_ID); if (containerIdStr == null || containerIdStr.isEmpty()) { html.h1()._("Cannot get container logs without a ContainerId")._(); return null; } ContainerId containerId = null; try { containerId = ConverterUtils.toContainerId(containerIdStr); } catch (IllegalArgumentException e) { html.h1() ._("Cannot get container logs for invalid containerId: " + containerIdStr)._(); return null; } return containerId; }
@GET @Path("/containers/{containerid}") @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) public ContainerInfo getNodeContainer(@PathParam("containerid") String id) { ContainerId containerId = null; init(); try { containerId = ConverterUtils.toContainerId(id); } catch (Exception e) { throw new BadRequestException("invalid container id, " + id); } Container container = nmContext.getContainers().get(containerId); if (container == null) { throw new NotFoundException("container with id, " + id + ", not found"); } return new ContainerInfo( this.nmContext, container, uriInfo.getBaseUri().toString(), webapp.name()); }
/** * Initial DragonAppMaster Get and CheckOut necessary parameters from system environment eg: * container_Id,host,port,http_port,submitTime * * @param args */ public static void main(String[] args) { try { String containerIdStr = System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV); String nodeHostString = System.getenv(ApplicationConstants.NM_HOST_ENV); String nodePortString = System.getenv(ApplicationConstants.NM_PORT_ENV); String nodeHttpPortString = System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV); String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); validateInputParam(containerIdStr, ApplicationConstants.AM_CONTAINER_ID_ENV); validateInputParam(nodeHostString, ApplicationConstants.NM_HOST_ENV); validateInputParam(nodePortString, ApplicationConstants.NM_PORT_ENV); validateInputParam(nodeHttpPortString, ApplicationConstants.NM_HTTP_PORT_ENV); validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId(); long appSubmitTime = Long.parseLong(appSubmitTimeStr); DragonAppMaster appMaster = new DragonAppMaster( applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), Integer.parseInt(nodeHttpPortString), appSubmitTime); Runtime.getRuntime().addShutdownHook(new CompositeServiceShutdownHook(appMaster)); YarnConfiguration conf = new YarnConfiguration(new DragonConfiguration()); conf.addResource(new Path(DragonJobConfig.JOB_CONF_FILE)); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(DragonJobConfig.USER_NAME, jobUserName); // Do not automatically close FileSystem objects so that in case of // SIGTERM I have a chance to write out the job history. I'll be closing // the objects myself. conf.setBoolean("fs.automatic.close", false); initAndStartAppMaster(appMaster, conf, jobUserName); } catch (Throwable t) { LOG.fatal("Error starting MRAppMaster", t); System.exit(1); } }
public static void main(String[] args) throws Exception { Map<String, String> env = System.getenv(); LOG.info("Starting app master with the following environment variables"); for (String key : env.keySet()) { LOG.info(key + "\t\t=" + env.get(key)); } Options opts; opts = new Options(); opts.addOption("num_containers", true, "Number of containers"); // START ZOOKEEPER String dataDir = "dataDir"; String logDir = "logDir"; IDefaultNameSpace defaultNameSpace = new IDefaultNameSpace() { @Override public void createDefaultNameSpace(ZkClient zkClient) {} }; try { FileUtils.deleteDirectory(new File(dataDir)); FileUtils.deleteDirectory(new File(logDir)); } catch (IOException e) { LOG.error(e); } final ZkServer server = new ZkServer(dataDir, logDir, defaultNameSpace); server.start(); // start Generic AppMaster that interacts with Yarn RM AppMasterConfig appMasterConfig = new AppMasterConfig(); String containerIdStr = appMasterConfig.getContainerId(); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId(); String configFile = AppMasterConfig.AppEnvironment.APP_SPEC_FILE.toString(); String className = appMasterConfig.getApplicationSpecFactory(); GenericApplicationMaster genericApplicationMaster = new GenericApplicationMaster(appAttemptID); try { genericApplicationMaster.start(); } catch (Exception e) { LOG.error("Unable to start application master: ", e); } ApplicationSpecFactory factory = HelixYarnUtil.createInstance(className); // TODO: Avoid setting static variable. YarnProvisioner.applicationMaster = genericApplicationMaster; YarnProvisioner.applicationMasterConfig = appMasterConfig; ApplicationSpec applicationSpec = factory.fromYaml(new FileInputStream(configFile)); YarnProvisioner.applicationSpec = applicationSpec; String zkAddress = appMasterConfig.getZKAddress(); String clusterName = appMasterConfig.getAppName(); // CREATE CLUSTER and setup the resources // connect ZkHelixConnection connection = new ZkHelixConnection(zkAddress); connection.connect(); // create the cluster ClusterId clusterId = ClusterId.from(clusterName); ClusterAccessor clusterAccessor = connection.createClusterAccessor(clusterId); StateModelDefinition statelessService = new StateModelDefinition(StateModelConfigGenerator.generateConfigForStatelessService()); StateModelDefinition taskStateModel = new StateModelDefinition(StateModelConfigGenerator.generateConfigForTaskStateModel()); clusterAccessor.createCluster( new ClusterConfig.Builder(clusterId) .addStateModelDefinition(statelessService) .addStateModelDefinition(taskStateModel) .build()); for (String service : applicationSpec.getServices()) { String resourceName = service; // add the resource with the local provisioner ResourceId resourceId = ResourceId.from(resourceName); ServiceConfig serviceConfig = applicationSpec.getServiceConfig(resourceName); serviceConfig.setSimpleField("service_name", service); int numContainers = serviceConfig.getIntField("num_containers", 1); YarnProvisionerConfig provisionerConfig = new YarnProvisionerConfig(resourceId); provisionerConfig.setNumContainers(numContainers); AutoRebalanceModeISBuilder idealStateBuilder = new AutoRebalanceModeISBuilder(resourceId); idealStateBuilder.setStateModelDefId(statelessService.getStateModelDefId()); idealStateBuilder.add(PartitionId.from(resourceId, "0")); idealStateBuilder.setNumReplica(1); ResourceConfig.Builder resourceConfigBuilder = new ResourceConfig.Builder(ResourceId.from(resourceName)); ResourceConfig resourceConfig = resourceConfigBuilder .provisionerConfig(provisionerConfig) .idealState(idealStateBuilder.build()) // .build(); clusterAccessor.addResource(resourceConfig); } // start controller ControllerId controllerId = ControllerId.from("controller1"); HelixController controller = connection.createController(clusterId, controllerId); controller.start(); // Start any pre-specified jobs List<TaskConfig> taskConfigs = applicationSpec.getTaskConfigs(); if (taskConfigs != null) { YarnConfiguration conf = new YarnConfiguration(); FileSystem fs; fs = FileSystem.get(conf); for (TaskConfig taskConfig : taskConfigs) { URI yamlUri = taskConfig.getYamlURI(); if (yamlUri != null && taskConfig.name != null) { InputStream is = readFromHDFS( fs, taskConfig.name, yamlUri, applicationSpec, appAttemptID.getApplicationId()); Workflow workflow = Workflow.parse(is); TaskDriver taskDriver = new TaskDriver(new ZKHelixManager(controller)); taskDriver.start(workflow); } } } Thread shutdownhook = new Thread( new Runnable() { @Override public void run() { server.shutdown(); } }); Runtime.getRuntime().addShutdownHook(shutdownhook); Thread.sleep(10000); }
public static void main(String[] args) throws Exception { System.err.println("Inside the AM!!!"); LOG.info("Starting the AM!!!!"); Options opts = new Options(); opts.addOption( "app_attempt_id", true, "App Attempt ID. Not to be used " + "unless for testing purposes"); CommandLine cl = new GnuParser().parse(opts, args); Map<String, String> envs = System.getenv(); _appAttemptID = Records.newRecord(ApplicationAttemptId.class); if (cl.hasOption("app_attempt_id")) { String appIdStr = cl.getOptionValue("app_attempt_id", ""); _appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else if (envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) { ContainerId containerId = ConverterUtils.toContainerId(envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV)); _appAttemptID = containerId.getApplicationAttemptId(); } else { throw new IllegalArgumentException( "Container and application attempt IDs not set in the environment"); } @SuppressWarnings("rawtypes") Map storm_conf = Config.readStormConfig(null); YarnConfiguration hadoopConf = new YarnConfiguration(); StormAMRMClient client = new StormAMRMClient(_appAttemptID, storm_conf, hadoopConf); client.init(hadoopConf); client.start(); BlockingQueue<Container> launcherQueue = new LinkedBlockingQueue<Container>(); try { InetSocketAddress addr = NetUtils.createSocketAddr( hadoopConf.get( YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS)); int port = Utils.getInt(storm_conf.get(Config.MASTER_THRIFT_PORT)); RegisterApplicationMasterResponse resp = client.registerApplicationMaster(addr.getHostName(), port, null); LOG.info("Got a registration response " + resp); LOG.info("Max Capability " + resp.getMaximumResourceCapability()); client.setMaxResource(resp.getMaximumResourceCapability()); MasterServer server = new MasterServer(storm_conf, client); LOG.info("Starting HB thread"); initAndStartHeartbeat( client, launcherQueue, (Integer) storm_conf.get(Config.MASTER_HEARTBEAT_INTERVAL_MILLIS)); LOG.info("Starting launcher"); initAndStartLauncher(client, launcherQueue); client.startAllSupervisors(); server.serve(); client.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "AllDone", null); } finally { client.stop(); } System.err.println("See Ya!!!"); System.exit(0); }
/** * Parse command line options * * @param args Command line args * @return Whether init successful and run should be invoked * @throws ParseException * @throws IOException */ public boolean init(String[] args) throws ParseException, IOException { Options opts = new Options(); opts.addOption( "app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption( "shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"); opts.addOption( "container_memory", true, "Amount of memory in MB to be requested to run the shell command"); opts.addOption( "container_vcores", true, "Amount of virtual cores to be requested to run the shell command"); opts.addOption( "num_containers", true, "No. of containers on which the shell command needs to be executed"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("debug", false, "Dump out debug information"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { printUsage(opts); throw new IllegalArgumentException("No args specified for application master to initialize"); } // Check whether customer log4j.properties file exists if (fileExist(log4jPath)) { try { Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath); } catch (Exception e) { LOG.warn("Can not set up custom log4j properties. " + e); } } if (cliParser.hasOption("help")) { printUsage(opts); return false; } if (cliParser.hasOption("debug")) { dumpOutDebugInfo(); } Map<String, String> envs = System.getenv(); if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException( ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } LOG.info( "Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); if (!fileExist(shellCommandPath) && envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION).isEmpty()) { throw new IllegalArgumentException( "No shell command or shell script specified to be executed by application master"); } if (fileExist(shellCommandPath)) { shellCommand = readContent(shellCommandPath); } if (fileExist(shellArgsPath)) { shellArgs = readContent(shellArgsPath); } if (cliParser.hasOption("shell_env")) { String shellEnvs[] = cliParser.getOptionValues("shell_env"); for (String env : shellEnvs) { env = env.trim(); int index = env.indexOf('='); if (index == -1) { shellEnv.put(env, ""); continue; } String key = env.substring(0, index); String val = ""; if (index < (env.length() - 1)) { val = env.substring(index + 1); } shellEnv.put(key, val); } } if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) { scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION); if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) { shellScriptPathTimestamp = Long.valueOf(envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)); } if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)) { shellScriptPathLen = Long.valueOf(envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)); } if (!scriptPath.isEmpty() && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) { LOG.error( "Illegal values in env for shell script path" + ", path=" + scriptPath + ", len=" + shellScriptPathLen + ", timestamp=" + shellScriptPathTimestamp); throw new IllegalArgumentException("Illegal values in env for shell script path"); } } containerMemory = Integer.parseInt(cliParser.getOptionValue("container_memory", "10")); containerVirtualCores = Integer.parseInt(cliParser.getOptionValue("container_vcores", "1")); numTotalContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "1")); if (numTotalContainers == 0) { throw new IllegalArgumentException("Cannot run distributed shell with no containers"); } requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); // Creating the Timeline Client timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); return true; }