private static void addUserSpecifiedClasspath(
StringBuilder classpathBuilder, Configuration conf) {
// Add any additional user-specified classpath
String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX);
if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) {
classpathBuilder.append(additionalClasspath).append(File.pathSeparator);
}
// Add PWD:PWD/*
classpathBuilder
.append(Environment.PWD.$())
.append(File.pathSeparator)
.append(Environment.PWD.$() + File.separator + "*")
.append(File.pathSeparator);
}
public static void setupDefaultEnv(
Map<String, String> env,
Configuration conf,
String userEnvKey,
String userEnvDefault,
String clusterDefaultEnvKey,
String clusterDefaultEnvDefault,
boolean usingArchive) {
// Setup the CLASSPATH in environment
// i.e. add { Hadoop jars, job jar, CWD } to classpath.
String classpath = getFrameworkClasspath(conf, usingArchive);
TezYARNUtils.addToEnvironment(
env, ApplicationConstants.Environment.CLASSPATH.name(), classpath, File.pathSeparator);
// Pre-pend pwd to LD_LIBRARY_PATH
// Done separately here because this is known to work platform
// independent
TezYARNUtils.addToEnvironment(
env, Environment.LD_LIBRARY_PATH.name(), Environment.PWD.$(), File.pathSeparator);
TezYARNUtils.appendToEnvFromInputString(
env, conf.get(userEnvKey, userEnvDefault), File.pathSeparator);
// set any env from config if it is not set already
TezYARNUtils.appendToEnvFromInputString(
env, conf.get(clusterDefaultEnvKey, clusterDefaultEnvDefault), File.pathSeparator);
}
void setupWorkerEnv(Map<String, String> workerEnv) {
for (String c :
configuration.getStrings(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
addToEnv(workerEnv, Environment.CLASSPATH.name(), c.trim());
}
addToEnv(workerEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*");
}
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
for (String c :
conf.getStrings(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
}
Apps.addToEnvironment(
appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*");
}
public static void setClasspath(Map<String, String> environment) throws IOException {
Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), MRJobConfig.JOB_JAR);
Apps.addToEnvironment(
environment, Environment.CLASSPATH.name(), Environment.PWD.$() + Path.SEPARATOR + "*");
MRApps.setMRFrameworkClasspath(environment);
}
public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) {
StringBuilder classpathBuilder = new StringBuilder();
boolean userClassesTakesPrecedence =
conf.getBoolean(
TezConfiguration.TEZ_USER_CLASSPATH_FIRST,
TezConfiguration.TEZ_USER_CLASSPATH_FIRST_DEFAULT);
if (userClassesTakesPrecedence) {
addUserSpecifiedClasspath(classpathBuilder, conf);
}
String[] tezLibUrisClassPath = conf.getStrings(TezConfiguration.TEZ_LIB_URIS_CLASSPATH);
if (!conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)
&& tezLibUrisClassPath != null
&& tezLibUrisClassPath.length != 0) {
for (String c : tezLibUrisClassPath) {
classpathBuilder.append(c.trim()).append(File.pathSeparator);
}
} else {
if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) {
LOG.info(
"Ignoring '"
+ TezConfiguration.TEZ_LIB_URIS
+ "' since '"
+ TezConfiguration.TEZ_IGNORE_LIB_URIS
+ "' is set to true ");
}
// Legacy: Next add the tez libs, if specified via an archive.
if (usingArchive) {
// Add PWD/tezlib/*
classpathBuilder
.append(Environment.PWD.$())
.append(File.separator)
.append(TezConstants.TEZ_TAR_LR_NAME)
.append(File.separator)
.append("*")
.append(File.pathSeparator);
// Legacy: Add PWD/tezlib/lib/*
classpathBuilder
.append(Environment.PWD.$())
.append(File.separator)
.append(TezConstants.TEZ_TAR_LR_NAME)
.append(File.separator)
.append("lib")
.append(File.separator)
.append("*")
.append(File.pathSeparator);
}
}
// Last add HADOOP_CLASSPATH, if it's required.
if (conf.getBoolean(
TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS,
TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT)) {
for (String c :
conf.getStrings(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
classpathBuilder.append(c.trim()).append(File.pathSeparator);
}
} else {
// Setup HADOOP_CONF_DIR after PWD and tez-libs, if it's required.
classpathBuilder.append(Environment.HADOOP_CONF_DIR.$()).append(File.pathSeparator);
}
if (!userClassesTakesPrecedence) {
addUserSpecifiedClasspath(classpathBuilder, conf);
}
String classpath = classpathBuilder.toString();
return StringInterner.weakIntern(classpath);
}
