private ContainerLaunchContext newContainerLaunchContext( Container container, String helixInstanceName) throws IOException { Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId); Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME); Map<String, LocalResource> resourceMap = Maps.newHashMap(); addContainerLocalResources( new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap); addContainerLocalResources( new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap); addContainerLocalResources( new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap); if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) { addRemoteAppFiles( this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap); } ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(resourceMap); containerLaunchContext.setEnvironment( YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); containerLaunchContext.setCommands( Lists.newArrayList(buildContainerCommand(container, helixInstanceName))); if (UserGroupInformation.isSecurityEnabled()) { containerLaunchContext.setTokens(this.tokens.duplicate()); } return containerLaunchContext; }
private void addTimelineDelegationToken(ContainerLaunchContext clc) throws YarnException, IOException { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); ByteBuffer tokens = clc.getTokens(); if (tokens != null) { dibb.reset(tokens); credentials.readTokenStorageStream(dibb); tokens.rewind(); } // If the timeline delegation token is already in the CLC, no need to add // one more for (org.apache.hadoop.security.token.Token<? extends TokenIdentifier> token : credentials.getAllTokens()) { if (token.getKind().equals(TimelineDelegationTokenIdentifier.KIND_NAME)) { return; } } org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier> timelineDelegationToken = getTimelineDelegationToken(); if (timelineDelegationToken == null) { return; } credentials.addToken(timelineService, timelineDelegationToken); if (LOG.isDebugEnabled()) { LOG.debug("Add timline delegation token into credentials: " + timelineDelegationToken); } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); clc.setTokens(tokens); }
private int runAndBlock(ContainerId cId, String... cmd) throws IOException { String appId = "APP_" + getNextId(); Container container = mock(Container.class); ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap<String, String> env = new HashMap<String, String>(); when(container.getContainerId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(context.getEnvironment()).thenReturn(env); String script = writeScriptFile(cmd); Path scriptPath = new Path(script); Path tokensPath = new Path("/dev/null"); Path workDir = new Path(workSpace.getAbsolutePath()); Path pidFile = new Path(workDir, "pid.txt"); exec.activateContainer(cId, pidFile); return exec.launchContainer( container, scriptPath, tokensPath, appSubmitter, appId, workDir, dirsHandler.getLocalDirs(), dirsHandler.getLogDirs()); }
void launchWorkerNode(Container container) throws IOException, YarnException { Map<String, String> env = System.getenv(); ContainerLaunchContext workerContext = Records.newRecord(ContainerLaunchContext.class); LocalResource workerJar = Records.newRecord(LocalResource.class); setupWorkerJar(workerJar); workerContext.setLocalResources(Collections.singletonMap("socialite.jar", workerJar)); System.out.println("[Master] workerJar:" + workerJar); Map<String, String> workerEnv = new HashMap<String, String>(env); setupWorkerEnv(workerEnv); workerContext.setEnvironment(workerEnv); String opts = "-Dsocialite.master=" + NetUtils.getHostname().split("/")[1] + " "; opts += "-Dsocialite.worker.num_threads=" + ClusterConf.get().getNumWorkerThreads() + " "; workerContext.setCommands( Collections.singletonList( "$JAVA_HOME/bin/java -ea " + env.get("JVM_OPTS") + " " + env.get("SOCIALITE_OPTS") + " " + opts + " " + "socialite.dist.worker.WorkerNode" + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); nmClient.startContainer(container, workerContext); L.info("Launched worker node (container:" + container.getId() + ")"); }
@Test(timeout = 20000) public void testAppSubmissionWithInvalidDelegationToken() throws Exception { Configuration conf = new Configuration(); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); MockRM rm = new MockRM(conf); ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( new HashMap<String, LocalResource>(), new HashMap<String, String>(), new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens, new HashMap<ApplicationAccessType, String>()); ApplicationSubmissionContext appSubContext = ApplicationSubmissionContext.newInstance( ApplicationId.newInstance(1234121, 0), "BOGUS", "default", Priority.UNDEFINED, amContainer, false, true, 1, Resource.newInstance(1024, 1), "BOGUS"); SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext); try { rm.getClientRMService().submitApplication(request); fail("Error was excepted."); } catch (YarnException e) { Assert.assertTrue(e.getMessage().contains("Bad header found in token storage")); } }
public static void setTokensFor( ContainerLaunchContext amContainer, List<Path> paths, Configuration conf) throws IOException { Credentials credentials = new Credentials(); // for HDFS TokenCache.obtainTokensForNamenodes(credentials, paths.toArray(new Path[0]), conf); // for HBase obtainTokenForHBase(credentials, conf); // for user UserGroupInformation currUsr = UserGroupInformation.getCurrentUser(); Collection<Token<? extends TokenIdentifier>> usrTok = currUsr.getTokens(); for (Token<? extends TokenIdentifier> token : usrTok) { final Text id = new Text(token.getIdentifier()); LOG.info("Adding user token " + id + " with " + token); credentials.addToken(id, token); } try (DataOutputBuffer dob = new DataOutputBuffer()) { credentials.writeTokenStorageToStream(dob); if (LOG.isDebugEnabled()) { LOG.debug("Wrote tokens. Credentials buffer length: " + dob.getLength()); } ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(securityTokens); } }
private ApplicationId submitAppAndGetAppId(String submitter, String queueName, boolean setupACLs) throws Exception { GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.newInstance(); ApplicationClientProtocol submitterClient = getRMClientForUser(submitter); ApplicationId applicationId = submitterClient.getNewApplication(newAppRequest).getApplicationId(); Resource resource = BuilderUtils.newResource(1024, 1); Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs); ContainerLaunchContext amContainerSpec = ContainerLaunchContext.newInstance(null, null, null, null, null, acls); ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.newInstance( applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType"); appSubmissionContext.setApplicationId(applicationId); appSubmissionContext.setQueue(queueName); SubmitApplicationRequest submitRequest = SubmitApplicationRequest.newInstance(appSubmissionContext); submitterClient.submitApplication(submitRequest); resourceManager.waitForState(applicationId, RMAppState.ACCEPTED); return applicationId; }
public void launchSupervisorOnContainer(Container container) throws IOException { LOG.info("Connecting to ContainerManager for containerid=" + container.getId()); String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort(); InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr); LOG.info("Connecting to ContainerManager at " + cmIpPortStr); ContainerManager proxy = ((ContainerManager) rpc.getProxy(ContainerManager.class, cmAddress, hadoopConf)); LOG.info("launchSupervisorOnContainer( id:" + container.getId() + " )"); ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); launchContext.setContainerId(container.getId()); launchContext.setResource(container.getResource()); try { launchContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); } catch (IOException e) { LOG.info( "Getting current user info failed when trying to launch the container" + e.getMessage()); } Map<String, String> env = new HashMap<String, String>(); launchContext.setEnvironment(env); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); String stormVersion = Util.getStormVersion(this.storm_conf); Path zip = new Path("/lib/storm/" + stormVersion + "/storm.zip"); FileSystem fs = FileSystem.get(this.hadoopConf); localResources.put( "storm", Util.newYarnAppResource( fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC)); String appHome = Util.getApplicationHomeForId(this.appAttemptId.toString()); Path dirDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf); localResources.put("conf", Util.newYarnAppResource(fs, dirDst)); launchContext.setLocalResources(localResources); List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf); launchContext.setCommands(supervisorArgs); StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(launchContext); LOG.info( "launchSupervisorOnContainer: startRequest prepared, calling startContainer. " + startRequest); try { StartContainerResponse response = proxy.startContainer(startRequest); LOG.info("Got a start container response " + response); } catch (Exception e) { LOG.error("Caught an exception while trying to start a container", e); System.exit(-1); } }
/* * (non-Javadoc) * @see org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler#onContainersAllocated(java.util.List) */ @Override public void onContainersAllocated(List<Container> containers) { LOG.info("onContainersAllocated() called"); for (Container container : containers) { LOG.info("container allocated is " + container); // Launch container by create ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); String logDir = System.getenv(ApplicationConstants.LOG_DIR_EXPANSION_VAR); // String command = new StringBuilder("/bin/echo ").append(message) // .append(" 1> // ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/echo_stdout") // .append(" 2> // ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/echo_stderr") // .toString(); String command = new StringBuilder("$JAVA_HOME/bin/java") .append(" ") .append("-classpath") .append(" ") .append(this.jar) .append(" ") .append(HelloServer.class.getName()) .append(" 1> ") .append(ApplicationConstants.LOG_DIR_EXPANSION_VAR) .append("/http_stdout") .append(" 2> ") .append(ApplicationConstants.LOG_DIR_EXPANSION_VAR) .append("/http_stderr") .toString(); LOG.info("Command to execute: " + command); List<String> commands = Arrays.asList(command); ctx.setCommands(commands); LOG.info("[AM] Launching container " + container.getId()); nmClient.startContainerAsync(container, ctx); } }
public void submitApplication( ApplicationId appId, String appName, Map<String, String> env, Map<String, LocalResource> localResources, List<String> commands, int memory) throws URISyntaxException, IOException { if (clientResourceManager == null) throw new IllegalStateException( "Cannot submit an application without connecting to resource manager!"); ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class); appCtx.setApplicationId(appId); appCtx.setApplicationName(appName); appCtx.setQueue("default"); appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); // System.out.println( "Based on my current user I am: " + // UserGroupInformation.getCurrentUser().getShortUserName() ); Priority prio = Records.newRecord(Priority.class); prio.setPriority(0); appCtx.setPriority(prio); /* for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) { System.out.println("IR:RM: " + entry.getKey() + "/" + entry.getValue().getResource()); } */ /* for (Map.Entry<String, String> entry : env.entrySet()) { System.out.println("IR:ResourceManager -> Env vars: " + entry.getKey() + "/" + entry.getValue() ); } */ // Launch ctx ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class); containerCtx.setLocalResources(localResources); containerCtx.setCommands(commands); containerCtx.setEnvironment(env); containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); containerCtx.setResource(capability); appCtx.setAMContainerSpec(containerCtx); SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class); submitReq.setApplicationSubmissionContext(appCtx); LOG.info("Submitting application to ASM"); clientResourceManager.submitApplication(submitReq); // Don't return anything, ASM#submit returns an empty response }
public static ContainerLaunchContext newContainerLaunchContext( Map<String, LocalResource> localResources, Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData, ByteBuffer tokens, Map<ApplicationAccessType, String> acls) { ContainerLaunchContext container = recordFactory.newRecordInstance(ContainerLaunchContext.class); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); container.setTokens(tokens); container.setApplicationACLs(acls); return container; }
@Test(timeout = 20000) public void testAMAdminCommandOpts() throws Exception { JobConf jobConf = new JobConf(); jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true"); jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m"); YARNRunner yarnRunner = new YARNRunner(jobConf); File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE); OutputStream out = new FileOutputStream(jobxml); conf.writeXml(out); out.close(); File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT); out = new FileOutputStream(jobsplit); out.close(); File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO); out = new FileOutputStream(jobsplitmetainfo); out.close(); File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE); out = new FileOutputStream(appTokens); out.close(); ApplicationSubmissionContext submissionContext = yarnRunner.createApplicationSubmissionContext( jobConf, testWorkDir.toString(), new Credentials()); ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec(); List<String> commands = containerSpec.getCommands(); int index = 0; int adminIndex = 0; int adminPos = -1; int userIndex = 0; int userPos = -1; for (String command : commands) { if (command != null) { adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true"); if (adminPos >= 0) adminIndex = index; userPos = command.indexOf("-Xmx1024m"); if (userPos >= 0) userIndex = index; } index++; } // Check both admin java opts and user java opts are in the commands assertTrue("AM admin command opts not in the commands.", adminPos > 0); assertTrue("AM user command opts not in the commands.", userPos > 0); // Check the admin java opts is before user java opts in the commands if (adminIndex == userIndex) { assertTrue("AM admin command opts is after user command opts.", adminPos < userPos); } else { assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex); } }
/** * Connects to CM, sets up container launch context for shell command and eventually dispatches * the container start request to the CM. */ @Override public void run() { System.out.println( "Setting up container launch container for containerid=" + container.getId()); ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); // Set the environment StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar).append("./*"); for (String c : am.getConf() .getStrings( YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(':'); classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } if (am.getConf().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } am.getShellEnv().put("CLASSPATH", classPathEnv.toString()); // Set the environment ctx.setEnvironment(am.getShellEnv()); ctx.setLocalResources(task.buildScriptsAndSetResources(container)); // Set the necessary command to execute on the allocated container Vector<CharSequence> vargs = new Vector<>(5); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + am.getContainerMemory() + "m"); // Set class name vargs.add(HiWayConfiguration.HIWAY_WORKER_CLASS); vargs.add("--appId " + am.getAppId().toString()); vargs.add("--containerId " + container.getId().toString()); vargs.add("--workflowId " + task.getWorkflowId()); vargs.add("--taskId " + task.getTaskId()); vargs.add("--taskName " + task.getTaskName()); vargs.add("--langLabel " + task.getLanguageLabel()); vargs.add("--id " + task.getId()); for (Data inputData : task.getInputData()) { vargs.add( "--input " + inputData.getLocalPath() + "," + inputData.isInput() + "," + inputData.getContainerId()); } for (Data outputData : task.getOutputData()) { vargs.add("--output " + outputData.getLocalPath()); } if (am.isDetermineFileSizes()) { vargs.add("--size"); } String invocScript = task.getInvocScript(); if (invocScript.length() > 0) { vargs.add("--invocScript " + invocScript); } vargs.add(">> " + Invocation.STDOUT_FILENAME); vargs.add("2>> " + Invocation.STDERR_FILENAME); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } List<String> commands = new ArrayList<>(); commands.add(command.toString()); ctx.setCommands(commands); /* Set up tokens for the container. For normal shell commands, the container in distribute-shell doesn't need any tokens. We are populating them mainly * for NodeManagers to be able to download any files in the distributed file-system. The tokens are otherwise also useful in cases, for e.g., when one * is running a "hadoop dfs" command inside the distributed shell. */ ctx.setTokens(am.getAllTokens().duplicate()); containerListener.addContainer(container.getId(), container); am.getNmClientAsync().startContainerAsync(container, ctx); }
@Test public void testAutomaticTimelineDelegationTokenLoading() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier(); final Token<TimelineDelegationTokenIdentifier> dToken = new Token<TimelineDelegationTokenIdentifier>( timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text()); // crate a mock client YarnClientImpl client = spy( new YarnClientImpl() { @Override protected void serviceInit(Configuration conf) throws Exception { if (getConfig() .getBoolean( YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineServiceEnabled = true; timelineClient = mock(TimelineClient.class); when(timelineClient.getDelegationToken(any(String.class))).thenReturn(dToken); timelineClient.init(getConfig()); timelineService = TimelineUtils.buildTimelineTokenService(getConfig()); } this.setConfig(conf); } @Override protected void serviceStart() throws Exception { rmClient = mock(ApplicationClientProtocol.class); } @Override protected void serviceStop() throws Exception {} @Override public ApplicationReport getApplicationReport(ApplicationId appId) { ApplicationReport report = mock(ApplicationReport.class); when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.SUBMITTED); return report; } @Override public boolean isSecurityEnabled() { return true; } }); client.init(conf); client.start(); try { // when i == 0, timeline DT already exists, no need to get one more // when i == 1, timeline DT doesn't exist, need to get one more for (int i = 0; i < 2; ++i) { ApplicationSubmissionContext context = mock(ApplicationSubmissionContext.class); ApplicationId applicationId = ApplicationId.newInstance(0, i + 1); when(context.getApplicationId()).thenReturn(applicationId); DataOutputBuffer dob = new DataOutputBuffer(); Credentials credentials = new Credentials(); if (i == 0) { credentials.addToken(client.timelineService, dToken); } credentials.writeTokenStorageToStream(dob); ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); ContainerLaunchContext clc = ContainerLaunchContext.newInstance(null, null, null, null, tokens, null); when(context.getAMContainerSpec()).thenReturn(clc); client.submitApplication(context); if (i == 0) { // GetTimelineDelegationToken shouldn't be called verify(client, never()).getTimelineDelegationToken(); } // In either way, token should be there credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); tokens = clc.getTokens(); if (tokens != null) { dibb.reset(tokens); credentials.readTokenStorageStream(dibb); tokens.rewind(); } Collection<Token<? extends TokenIdentifier>> dTokens = credentials.getAllTokens(); Assert.assertEquals(1, dTokens.size()); Assert.assertEquals(dToken, dTokens.iterator().next()); } } finally { client.stop(); } }
/** * Launch application for the dag represented by this client. * * @throws YarnException * @throws IOException */ public void startApplication() throws YarnException, IOException { Class<?>[] defaultClasses; if (applicationType.equals(YARN_APPLICATION_TYPE)) { // TODO restrict the security check to only check if security is enabled for webservices. if (UserGroupInformation.isSecurityEnabled()) { defaultClasses = DATATORRENT_SECURITY_CLASSES; } else { defaultClasses = DATATORRENT_CLASSES; } } else { throw new IllegalStateException(applicationType + " is not a valid application type."); } LinkedHashSet<String> localJarFiles = findJars(dag, defaultClasses); if (resources != null) { localJarFiles.addAll(resources); } YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info( "Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); // GetClusterNodesRequest clusterNodesReq = Records.newRecord(GetClusterNodesRequest.class); // GetClusterNodesResponse clusterNodesResp = // rmClient.clientRM.getClusterNodes(clusterNodesReq); // LOG.info("Got Cluster node info from ASM"); // for (NodeReport node : clusterNodesResp.getNodeReports()) { // LOG.info("Got node report from ASM for" // + ", nodeId=" + node.getNodeId() // + ", nodeAddress" + node.getHttpAddress() // + ", nodeRackName" + node.getRackName() // + ", nodeNumContainers" + node.getNumContainers() // + ", nodeHealthStatus" + node.getHealthReport()); // } List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info( "User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } // Get a new application id YarnClientApplication newApp = yarnClient.createApplication(); appId = newApp.getNewApplicationResponse().getApplicationId(); // Dump out information about cluster capability as seen by the resource manager int maxMem = newApp.getNewApplicationResponse().getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int amMemory = dag.getMasterMemoryMB(); if (amMemory > maxMem) { LOG.info( "AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } if (dag.getAttributes().get(LogicalPlan.APPLICATION_ID) == null) { dag.setAttribute(LogicalPlan.APPLICATION_ID, appId.toString()); } // Create launch context for app master LOG.info("Setting up application submission context for ASM"); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName(dag.getValue(LogicalPlan.APPLICATION_NAME)); appContext.setApplicationType(this.applicationType); if (YARN_APPLICATION_TYPE.equals(this.applicationType)) { // appContext.setMaxAppAttempts(1); // no retries until Stram is HA } // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // Setup security tokens // If security is enabled get ResourceManager and NameNode delegation tokens. // Set these tokens on the container so that they are sent as part of application submission. // This also sets them up for renewal by ResourceManager. The NameNode delegation rmToken // is also used by ResourceManager to fetch the jars from HDFS and set them up for the // application master launch. if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. FileSystem fs = StramClientUtils.newFileSystemInstance(conf); try { final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } } finally { fs.close(); } addRMDelegationToken(tokenRenewer, credentials); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // copy required jar files to dfs, to be localized for containers FileSystem fs = StramClientUtils.newFileSystemInstance(conf); try { Path appsBasePath = new Path(StramClientUtils.getDTDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS); Path appPath = new Path(appsBasePath, appId.toString()); String libJarsCsv = copyFromLocal(fs, appPath, localJarFiles.toArray(new String[] {})); LOG.info("libjars: {}", libJarsCsv); dag.getAttributes().put(LogicalPlan.LIBRARY_JARS, libJarsCsv); LaunchContainerRunnable.addFilesToLocalResources( LocalResourceType.FILE, libJarsCsv, localResources, fs); if (archives != null) { String[] localFiles = archives.split(","); String archivesCsv = copyFromLocal(fs, appPath, localFiles); LOG.info("archives: {}", archivesCsv); dag.getAttributes().put(LogicalPlan.ARCHIVES, archivesCsv); LaunchContainerRunnable.addFilesToLocalResources( LocalResourceType.ARCHIVE, archivesCsv, localResources, fs); } if (files != null) { String[] localFiles = files.split(","); String filesCsv = copyFromLocal(fs, appPath, localFiles); LOG.info("files: {}", filesCsv); dag.getAttributes().put(LogicalPlan.FILES, filesCsv); LaunchContainerRunnable.addFilesToLocalResources( LocalResourceType.FILE, filesCsv, localResources, fs); } dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, appPath.toString()); if (dag.getAttributes().get(OperatorContext.STORAGE_AGENT) == null) { /* which would be the most likely case */ Path checkpointPath = new Path(appPath, LogicalPlan.SUBDIR_CHECKPOINTS); // use conf client side to pickup any proxy settings from dt-site.xml dag.setAttribute( OperatorContext.STORAGE_AGENT, new FSStorageAgent(checkpointPath.toString(), conf)); } if (dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR) == null) { dag.setAttribute( LogicalPlan.CONTAINER_OPTS_CONFIGURATOR, new BasicContainerOptConfigurator()); } // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); Path log4jDst = new Path(appPath, "log4j.props"); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); log4jRsrc.setType(LocalResourceType.FILE); log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION); log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri())); log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime()); log4jRsrc.setSize(log4jFileStatus.getLen()); localResources.put("log4j.properties", log4jRsrc); } if (originalAppId != null) { Path origAppPath = new Path(appsBasePath, this.originalAppId); LOG.info("Restart from {}", origAppPath); copyInitialState(origAppPath); } // push logical plan to DFS location Path cfgDst = new Path(appPath, LogicalPlan.SER_FILE_NAME); FSDataOutputStream outStream = fs.create(cfgDst, true); LogicalPlan.write(this.dag, outStream); outStream.close(); Path launchConfigDst = new Path(appPath, LogicalPlan.LAUNCH_CONFIG_FILE_NAME); outStream = fs.create(launchConfigDst, true); conf.writeXml(outStream); outStream.close(); FileStatus topologyFileStatus = fs.getFileStatus(cfgDst); LocalResource topologyRsrc = Records.newRecord(LocalResource.class); topologyRsrc.setType(LocalResourceType.FILE); topologyRsrc.setVisibility(LocalResourceVisibility.APPLICATION); topologyRsrc.setResource(ConverterUtils.getYarnUrlFromURI(cfgDst.toUri())); topologyRsrc.setTimestamp(topologyFileStatus.getModificationTime()); topologyRsrc.setSize(topologyFileStatus.getLen()); localResources.put(LogicalPlan.SER_FILE_NAME, topologyRsrc); // Set local resource info into app master container launch context amContainer.setLocalResources(localResources); // Set the necessary security tokens as needed // amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // Add application jar(s) location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar(s) // including ${CLASSPATH} will duplicate the class path in app master, removing it for now // StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*"); StringBuilder classPathEnv = new StringBuilder("./*"); String classpath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH); for (String c : StringUtils.isBlank(classpath) ? YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH : classpath.split(",")) { if (c.equals("$HADOOP_CLIENT_CONF_DIR")) { // SPOI-2501 continue; } classPathEnv.append(':'); classPathEnv.append(c.trim()); } env.put("CLASSPATH", classPathEnv.toString()); // propagate to replace node managers user name (effective in non-secure mode) env.put("HADOOP_USER_NAME", UserGroupInformation.getLoginUser().getUserName()); amContainer.setEnvironment(env); // Set the necessary command to execute the application master ArrayList<CharSequence> vargs = new ArrayList<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(javaCmd); if (dag.isDebug()) { vargs.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=n"); } // Set Xmx based on am memory size // default heap size 75% of total memory vargs.add("-Xmx" + (amMemory * 3 / 4) + "m"); vargs.add("-XX:+HeapDumpOnOutOfMemoryError"); vargs.add("-XX:HeapDumpPath=/tmp/dt-heap-" + appId.getId() + ".bin"); vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA"); vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath())); if (dag.isDebug()) { vargs.add("-Dlog4j.debug=true"); } String loggersLevel = conf.get(DTLoggerFactory.DT_LOGGERS_LEVEL); if (loggersLevel != null) { vargs.add(String.format("-D%s=%s", DTLoggerFactory.DT_LOGGERS_LEVEL, loggersLevel)); } vargs.add(StreamingAppMaster.class.getName()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final command StringBuilder command = new StringBuilder(9 * vargs.size()); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); amContainer.setCommands(commands); // Set up resource type requirements // For now, only memory is supported so we set memory requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(queueName); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = rmClient.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure String specStr = Objects.toStringHelper("Submitting application: ") .add("name", appContext.getApplicationName()) .add("queue", appContext.getQueue()) .add("user", UserGroupInformation.getLoginUser()) .add("resource", appContext.getResource()) .toString(); LOG.info(specStr); if (dag.isDebug()) { // LOG.info("Full submission context: " + appContext); } yarnClient.submitApplication(appContext); } finally { fs.close(); } }
@Override /** * Connects to CM, sets up container launch context for shell command and eventually dispatches * the container start request to the CM. */ public void run() { LOG.info("Setting up container launch container for containerid=" + container.getId()); ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); // Set the environment ctx.setEnvironment(shellEnv); // Set the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // The container for the eventual shell commands needs its own local // resources too. // In this scenario, if a shell script is specified, we need to have it // copied and made available to the container. if (!scriptPath.isEmpty()) { Path renamedScriptPath = null; if (Shell.WINDOWS) { renamedScriptPath = new Path(scriptPath + ".bat"); } else { renamedScriptPath = new Path(scriptPath + ".sh"); } try { // rename the script file based on the underlying OS syntax. renameScriptFile(renamedScriptPath); } catch (Exception e) { LOG.error("Not able to add suffix (.bat/.sh) to the shell script filename", e); // We know we cannot continue launching the container // so we should release it. numCompletedContainers.incrementAndGet(); numFailedContainers.incrementAndGet(); return; } LocalResource shellRsrc = Records.newRecord(LocalResource.class); shellRsrc.setType(LocalResourceType.FILE); shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION); try { shellRsrc.setResource( ConverterUtils.getYarnUrlFromURI(new URI(renamedScriptPath.toString()))); } catch (URISyntaxException e) { LOG.error( "Error when trying to use shell script path specified" + " in env, path=" + renamedScriptPath, e); // A failure scenario on bad input such as invalid shell script path // We know we cannot continue launching the container // so we should release it. // TODO numCompletedContainers.incrementAndGet(); numFailedContainers.incrementAndGet(); return; } shellRsrc.setTimestamp(shellScriptPathTimestamp); shellRsrc.setSize(shellScriptPathLen); localResources.put( Shell.WINDOWS ? ExecBatScripStringtPath : ExecShellStringPath, shellRsrc); shellCommand = Shell.WINDOWS ? windows_command : linux_bash_command; } ctx.setLocalResources(localResources); // Set the necessary command to execute on the allocated container Vector<CharSequence> vargs = new Vector<CharSequence>(5); // Set executable command vargs.add(shellCommand); // Set shell script path if (!scriptPath.isEmpty()) { vargs.add(Shell.WINDOWS ? ExecBatScripStringtPath : ExecShellStringPath); } // Set args for the shell command if any vargs.add(shellArgs); // Add log redirect params vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } List<String> commands = new ArrayList<String>(); commands.add(command.toString()); ctx.setCommands(commands); // Set up tokens for the container too. Today, for normal shell commands, // the container in distribute-shell doesn't need any tokens. We are // populating them mainly for NodeManagers to be able to download any // files in the distributed file-system. The tokens are otherwise also // useful in cases, for e.g., when one is running a "hadoop dfs" command // inside the distributed shell. ctx.setTokens(allTokens.duplicate()); containerListener.addContainer(container.getId(), container); nmClientAsync.startContainerAsync(container, ctx); }
public void run(String[] args) throws Exception { final String command = args[0]; final int n = Integer.valueOf(args[1]); final Path jarPath = new Path(args[2]); // Create yarnClient YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); amContainer.setCommands( Collections.singletonList( "$JAVA_HOME/bin/java" + " -Xmx256M" + " com.hortonworks.simpleyarnapp.ApplicationMaster" + " " + command + " " + String.valueOf(n) + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); // Setup jar for ApplicationMaster LocalResource appMasterJar = Records.newRecord(LocalResource.class); setupAppMasterJar(jarPath, appMasterJar); System.out.println("Jar name is " + jarPath.getName()); amContainer.setLocalResources(Collections.singletonMap(jarPath.getName(), appMasterJar)); // Setup CLASSPATH for ApplicationMaster Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); capability.setVirtualCores(1); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("apache-yarn-example"); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // queue // Submit application ApplicationId appId = appContext.getApplicationId(); System.out.println("Submitting application " + appId); yarnClient.submitApplication(appContext); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { System.out.println("App Status = " + appState); Thread.sleep(100); appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); } System.out.println( "Application " + appId + " finished with" + " state " + appState + " at " + appReport.getFinishTime()); }
@Override public ContainerState transition(ContainerImpl container, ContainerEvent event) { final ContainerLaunchContext ctxt = container.launchContext; container.metrics.initingContainer(); container .dispatcher .getEventHandler() .handle(new AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT, container)); // Inform the AuxServices about the opaque serviceData Map<String, ByteBuffer> csd = ctxt.getServiceData(); if (csd != null) { // This can happen more than once per Application as each container may // have distinct service data for (Map.Entry<String, ByteBuffer> service : csd.entrySet()) { container .dispatcher .getEventHandler() .handle( new AuxServicesEvent( AuxServicesEventType.APPLICATION_INIT, container.user, container.containerId.getApplicationAttemptId().getApplicationId(), service.getKey().toString(), service.getValue())); } } // Send requests for public, private resources Map<String, LocalResource> cntrRsrc = ctxt.getLocalResources(); if (!cntrRsrc.isEmpty()) { try { for (Map.Entry<String, LocalResource> rsrc : cntrRsrc.entrySet()) { try { LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue()); List<String> links = container.pendingResources.get(req); if (links == null) { links = new ArrayList<String>(); container.pendingResources.put(req, links); } links.add(rsrc.getKey()); switch (rsrc.getValue().getVisibility()) { case PUBLIC: container.publicRsrcs.add(req); break; case PRIVATE: container.privateRsrcs.add(req); break; case APPLICATION: container.appRsrcs.add(req); break; } } catch (URISyntaxException e) { LOG.info("Got exception parsing " + rsrc.getKey() + " and value " + rsrc.getValue()); throw e; } } } catch (URISyntaxException e) { // malformed resource; abort container launch LOG.warn("Failed to parse resource-request", e); container.cleanup(); container.metrics.endInitingContainer(); return ContainerState.LOCALIZATION_FAILED; } Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); if (!container.publicRsrcs.isEmpty()) { req.put(LocalResourceVisibility.PUBLIC, container.publicRsrcs); } if (!container.privateRsrcs.isEmpty()) { req.put(LocalResourceVisibility.PRIVATE, container.privateRsrcs); } if (!container.appRsrcs.isEmpty()) { req.put(LocalResourceVisibility.APPLICATION, container.appRsrcs); } container .dispatcher .getEventHandler() .handle(new ContainerLocalizationRequestEvent(container, req)); return ContainerState.LOCALIZING; } else { container .dispatcher .getEventHandler() .handle( new ContainersLauncherEvent( container, ContainersLauncherEventType.LAUNCH_CONTAINER)); container.metrics.endInitingContainer(); return ContainerState.LOCALIZED; } }
@Test public void testSuccessfulContainerLaunch() throws InterruptedException, IOException { FileContext localFS = FileContext.getLocalFSFileContext(); localFS.delete(new Path(localDir.getAbsolutePath()), true); localFS.delete(new Path(localLogDir.getAbsolutePath()), true); localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true); localDir.mkdir(); localLogDir.mkdir(); remoteLogDir.mkdir(); Context context = new NMContext(); YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath()); ContainerExecutor exec = new DefaultContainerExecutor(); exec.setConf(conf); DeletionService del = new DeletionService(exec); Dispatcher dispatcher = new AsyncDispatcher(); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.init(conf); LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); NodeManagerMetrics metrics = NodeManagerMetrics.create(); ContainerTokenSecretManager containerTokenSecretManager = new ContainerTokenSecretManager(); NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl( context, dispatcher, healthChecker, metrics, containerTokenSecretManager) { @Override protected ResourceTracker getRMClient() { return new LocalRMInterface(); }; @Override protected void startStatusUpdater() { return; // Don't start any updating thread. } }; DummyContainerManager containerManager = new DummyContainerManager( context, exec, del, nodeStatusUpdater, metrics, containerTokenSecretManager, new ApplicationACLsManager(conf), dirsHandler); containerManager.init(conf); containerManager.start(); ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); ContainerId cID = recordFactory.newRecordInstance(ContainerId.class); ApplicationId applicationId = recordFactory.newRecordInstance(ApplicationId.class); applicationId.setClusterTimestamp(0); applicationId.setId(0); ApplicationAttemptId applicationAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class); applicationAttemptId.setApplicationId(applicationId); applicationAttemptId.setAttemptId(0); cID.setApplicationAttemptId(applicationAttemptId); launchContext.setContainerId(cID); launchContext.setUser("testing"); launchContext.setResource(recordFactory.newRecordInstance(Resource.class)); StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); request.setContainerLaunchContext(launchContext); containerManager.startContainer(request); BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.RUNNING); StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); stopRequest.setContainerId(cID); containerManager.stopContainer(stopRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.COMPLETE); containerManager.stop(); }