Exemple #1
0
 private static void addPathToConfig(Configuration conf, File path) {
   // chain-in a new classloader
   URL fileUrl = null;
   try {
     fileUrl = path.toURL();
   } catch (MalformedURLException e) {
     throw new RuntimeException("Erroneous config file path", e);
   }
   URL[] urls = {fileUrl};
   ClassLoader cl = new URLClassLoader(urls, conf.getClassLoader());
   conf.setClassLoader(cl);
 }
 /**
  * Executes launcher.
  *
  * @param configuration the Hadoop configuration for the application
  * @param args the launcher arguments
  * @return the exit status
  */
 public static int exec(Configuration configuration, String... args) {
   if (LOG.isDebugEnabled()) {
     LOG.debug(
         MessageFormat.format(
             "Preparing application: {0}", //$NON-NLS-1$
             Arrays.toString(args)));
   }
   configuration.setBoolean(KEY_LAUNCHER_USED, true);
   LauncherOptions options;
   try {
     options = LauncherOptionsParser.parse(configuration, args);
   } catch (Exception e) {
     LOG.error(
         MessageFormat.format("Exception occurred in launcher: {0}", Arrays.toString(args)), e);
     return LAUNCH_ERROR;
   }
   try {
     Configuration conf = options.getConfiguration();
     conf.setClassLoader(options.getApplicationClassLoader());
     Tool tool;
     try {
       tool = ReflectionUtils.newInstance(options.getApplicationClass(), conf);
     } catch (Exception e) {
       LOG.error(
           MessageFormat.format(
               "Exception occurred in launcher: {0}{1}",
               options.getApplicationClass().getName(), options.getApplicationArguments()),
           e);
       return LAUNCH_ERROR;
     }
     try {
       return launch(conf, tool, options.getApplicationArgumentArray());
     } catch (Exception e) {
       LOG.error(
           MessageFormat.format(
               "Exception occurred in launcher: {0}{1}",
               options.getApplicationClass().getName(), options.getApplicationArguments()),
           e);
       return CLIENT_ERROR;
     }
   } finally {
     disposeClassLoader(options.getApplicationClassLoader());
     for (File file : options.getApplicationCacheDirectories()) {
       if (delete(file) == false) {
         LOG.warn(
             MessageFormat.format("Failed to delete the application cache directory: {0}", file));
       }
     }
   }
 }
  /**
   * Modify configuration according user-specified generic options
   *
   * @param conf Configuration to be modified
   * @param line User-specified generic options
   */
  private void processGeneralOptions(Configuration conf, CommandLine line) throws IOException {
    if (line.hasOption("fs")) {
      FileSystem.setDefaultUri(conf, line.getOptionValue("fs"));
    }

    if (line.hasOption("jt")) {
      conf.set("mapred.job.tracker", line.getOptionValue("jt"), "from -jt command line option");
    }
    if (line.hasOption("conf")) {
      String[] values = line.getOptionValues("conf");
      for (String value : values) {
        conf.addResource(new Path(value));
      }
    }
    if (line.hasOption("libjars")) {
      conf.set(
          "tmpjars",
          validateFiles(line.getOptionValue("libjars"), conf),
          "from -libjars command line option");
      // setting libjars in client classpath
      URL[] libjars = getLibJars(conf);
      if (libjars != null && libjars.length > 0) {
        conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
        Thread.currentThread()
            .setContextClassLoader(
                new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
      }
    }
    if (line.hasOption("files")) {
      conf.set(
          "tmpfiles",
          validateFiles(line.getOptionValue("files"), conf),
          "from -files command line option");
    }
    if (line.hasOption("archives")) {
      conf.set(
          "tmparchives",
          validateFiles(line.getOptionValue("archives"), conf),
          "from -archives command line option");
    }
    if (line.hasOption('D')) {
      String[] property = line.getOptionValues('D');
      for (String prop : property) {
        String[] keyval = prop.split("=", 2);
        if (keyval.length == 2) {
          conf.set(keyval[0], keyval[1], "from command line");
        }
      }
    }
    conf.setBoolean("mapreduce.client.genericoptionsparser.used", true);

    // tokensFile
    if (line.hasOption("tokenCacheFile")) {
      String fileName = line.getOptionValue("tokenCacheFile");
      // check if the local file exists
      FileSystem localFs = FileSystem.getLocal(conf);
      Path p = new Path(fileName);
      if (!localFs.exists(p)) {
        throw new FileNotFoundException("File " + fileName + " does not exist.");
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("setting conf tokensFile: " + fileName);
      }
      conf.set(
          "mapreduce.job.credentials.json",
          localFs.makeQualified(p).toString(),
          "from -tokenCacheFile command line option");
    }
  }