示例#1
0
  private static void addUserSpecifiedClasspath(
      StringBuilder classpathBuilder, Configuration conf) {
    // Add any additional user-specified classpath
    String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX);
    if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) {
      classpathBuilder.append(additionalClasspath).append(File.pathSeparator);
    }

    // Add PWD:PWD/*
    classpathBuilder
        .append(Environment.PWD.$())
        .append(File.pathSeparator)
        .append(Environment.PWD.$() + File.separator + "*")
        .append(File.pathSeparator);
  }
示例#2
0
  public static void setupDefaultEnv(
      Map<String, String> env,
      Configuration conf,
      String userEnvKey,
      String userEnvDefault,
      String clusterDefaultEnvKey,
      String clusterDefaultEnvDefault,
      boolean usingArchive) {
    // Setup the CLASSPATH in environment
    // i.e. add { Hadoop jars, job jar, CWD } to classpath.
    String classpath = getFrameworkClasspath(conf, usingArchive);
    TezYARNUtils.addToEnvironment(
        env, ApplicationConstants.Environment.CLASSPATH.name(), classpath, File.pathSeparator);

    // Pre-pend pwd to LD_LIBRARY_PATH
    // Done separately here because this is known to work platform
    // independent
    TezYARNUtils.addToEnvironment(
        env, Environment.LD_LIBRARY_PATH.name(), Environment.PWD.$(), File.pathSeparator);
    TezYARNUtils.appendToEnvFromInputString(
        env, conf.get(userEnvKey, userEnvDefault), File.pathSeparator);
    // set any env from config if it is not set already
    TezYARNUtils.appendToEnvFromInputString(
        env, conf.get(clusterDefaultEnvKey, clusterDefaultEnvDefault), File.pathSeparator);
  }
 void setupWorkerEnv(Map<String, String> workerEnv) {
   for (String c :
       configuration.getStrings(
           YarnConfiguration.YARN_APPLICATION_CLASSPATH,
           YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
     addToEnv(workerEnv, Environment.CLASSPATH.name(), c.trim());
   }
   addToEnv(workerEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*");
 }
示例#4
0
 private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
   for (String c :
       conf.getStrings(
           YarnConfiguration.YARN_APPLICATION_CLASSPATH,
           YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
     Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
   }
   Apps.addToEnvironment(
       appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*");
 }
示例#5
0
 public static void setClasspath(Map<String, String> environment) throws IOException {
   Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), MRJobConfig.JOB_JAR);
   Apps.addToEnvironment(
       environment, Environment.CLASSPATH.name(), Environment.PWD.$() + Path.SEPARATOR + "*");
   MRApps.setMRFrameworkClasspath(environment);
 }
示例#6
0
  public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) {
    StringBuilder classpathBuilder = new StringBuilder();
    boolean userClassesTakesPrecedence =
        conf.getBoolean(
            TezConfiguration.TEZ_USER_CLASSPATH_FIRST,
            TezConfiguration.TEZ_USER_CLASSPATH_FIRST_DEFAULT);
    if (userClassesTakesPrecedence) {
      addUserSpecifiedClasspath(classpathBuilder, conf);
    }

    String[] tezLibUrisClassPath = conf.getStrings(TezConfiguration.TEZ_LIB_URIS_CLASSPATH);

    if (!conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)
        && tezLibUrisClassPath != null
        && tezLibUrisClassPath.length != 0) {
      for (String c : tezLibUrisClassPath) {
        classpathBuilder.append(c.trim()).append(File.pathSeparator);
      }
    } else {
      if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) {
        LOG.info(
            "Ignoring '"
                + TezConfiguration.TEZ_LIB_URIS
                + "' since  '"
                + TezConfiguration.TEZ_IGNORE_LIB_URIS
                + "' is set to true ");
      }

      // Legacy: Next add the tez libs, if specified via an archive.
      if (usingArchive) {
        // Add PWD/tezlib/*
        classpathBuilder
            .append(Environment.PWD.$())
            .append(File.separator)
            .append(TezConstants.TEZ_TAR_LR_NAME)
            .append(File.separator)
            .append("*")
            .append(File.pathSeparator);

        // Legacy: Add PWD/tezlib/lib/*
        classpathBuilder
            .append(Environment.PWD.$())
            .append(File.separator)
            .append(TezConstants.TEZ_TAR_LR_NAME)
            .append(File.separator)
            .append("lib")
            .append(File.separator)
            .append("*")
            .append(File.pathSeparator);
      }
    }
    // Last add HADOOP_CLASSPATH, if it's required.
    if (conf.getBoolean(
        TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS,
        TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT)) {
      for (String c :
          conf.getStrings(
              YarnConfiguration.YARN_APPLICATION_CLASSPATH,
              YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classpathBuilder.append(c.trim()).append(File.pathSeparator);
      }
    } else {
      // Setup HADOOP_CONF_DIR after PWD and tez-libs, if it's required.
      classpathBuilder.append(Environment.HADOOP_CONF_DIR.$()).append(File.pathSeparator);
    }

    if (!userClassesTakesPrecedence) {
      addUserSpecifiedClasspath(classpathBuilder, conf);
    }
    String classpath = classpathBuilder.toString();
    return StringInterner.weakIntern(classpath);
  }