private ContainerLaunchContext newContainerLaunchContext( Container container, String helixInstanceName) throws IOException { Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId); Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME); Map<String, LocalResource> resourceMap = Maps.newHashMap(); addContainerLocalResources( new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap); addContainerLocalResources( new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap); addContainerLocalResources( new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap); if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) { addRemoteAppFiles( this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap); } ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(resourceMap); containerLaunchContext.setEnvironment( YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); containerLaunchContext.setCommands( Lists.newArrayList(buildContainerCommand(container, helixInstanceName))); if (UserGroupInformation.isSecurityEnabled()) { containerLaunchContext.setTokens(this.tokens.duplicate()); } return containerLaunchContext; }
private void addRemoteAppFiles(String hdfsFileList, Map<String, LocalResource> resourceMap) throws IOException { for (String hdfsFilePath : SPLITTER.split(hdfsFileList)) { Path srcFilePath = new Path(hdfsFilePath); YarnHelixUtils.addFileAsLocalResource( srcFilePath.getFileSystem(this.yarnConfiguration), srcFilePath, LocalResourceType.FILE, resourceMap); } }
private void addContainerLocalResources(Path destDir, Map<String, LocalResource> resourceMap) throws IOException { if (!this.fs.exists(destDir)) { LOGGER.warn( String.format("Path %s does not exist so no container LocalResource to add", destDir)); return; } FileStatus[] statuses = this.fs.listStatus(destDir); if (statuses != null) { for (FileStatus status : statuses) { YarnHelixUtils.addFileAsLocalResource( this.fs, status.getPath(), LocalResourceType.FILE, resourceMap); } } }