private ContainerLaunchContext newContainerLaunchContext(
Container container, String helixInstanceName) throws IOException {
Path appWorkDir =
GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId);
Path containerWorkDir =
new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);
Map<String, LocalResource> resourceMap = Maps.newHashMap();
addContainerLocalResources(
new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap);
addContainerLocalResources(
new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap);
addContainerLocalResources(
new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap);
if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) {
addRemoteAppFiles(
this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY),
resourceMap);
}
ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
containerLaunchContext.setLocalResources(resourceMap);
containerLaunchContext.setEnvironment(
YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
containerLaunchContext.setCommands(
Lists.newArrayList(buildContainerCommand(container, helixInstanceName)));
if (UserGroupInformation.isSecurityEnabled()) {
containerLaunchContext.setTokens(this.tokens.duplicate());
}
return containerLaunchContext;
}
private void addRemoteAppFiles(String hdfsFileList, Map<String, LocalResource> resourceMap)
throws IOException {
for (String hdfsFilePath : SPLITTER.split(hdfsFileList)) {
Path srcFilePath = new Path(hdfsFilePath);
YarnHelixUtils.addFileAsLocalResource(
srcFilePath.getFileSystem(this.yarnConfiguration),
srcFilePath,
LocalResourceType.FILE,
resourceMap);
}
}
private void addContainerLocalResources(Path destDir, Map<String, LocalResource> resourceMap)
throws IOException {
if (!this.fs.exists(destDir)) {
LOGGER.warn(
String.format("Path %s does not exist so no container LocalResource to add", destDir));
return;
}
FileStatus[] statuses = this.fs.listStatus(destDir);
if (statuses != null) {
for (FileStatus status : statuses) {
YarnHelixUtils.addFileAsLocalResource(
this.fs, status.getPath(), LocalResourceType.FILE, resourceMap);
}
}
}
