private static void addPathToConfig(Configuration conf, File path) { // chain-in a new classloader URL fileUrl = null; try { fileUrl = path.toURL(); } catch (MalformedURLException e) { throw new RuntimeException("Erroneous config file path", e); } URL[] urls = {fileUrl}; ClassLoader cl = new URLClassLoader(urls, conf.getClassLoader()); conf.setClassLoader(cl); }
/** * Executes launcher. * * @param configuration the Hadoop configuration for the application * @param args the launcher arguments * @return the exit status */ public static int exec(Configuration configuration, String... args) { if (LOG.isDebugEnabled()) { LOG.debug( MessageFormat.format( "Preparing application: {0}", //$NON-NLS-1$ Arrays.toString(args))); } configuration.setBoolean(KEY_LAUNCHER_USED, true); LauncherOptions options; try { options = LauncherOptionsParser.parse(configuration, args); } catch (Exception e) { LOG.error( MessageFormat.format("Exception occurred in launcher: {0}", Arrays.toString(args)), e); return LAUNCH_ERROR; } try { Configuration conf = options.getConfiguration(); conf.setClassLoader(options.getApplicationClassLoader()); Tool tool; try { tool = ReflectionUtils.newInstance(options.getApplicationClass(), conf); } catch (Exception e) { LOG.error( MessageFormat.format( "Exception occurred in launcher: {0}{1}", options.getApplicationClass().getName(), options.getApplicationArguments()), e); return LAUNCH_ERROR; } try { return launch(conf, tool, options.getApplicationArgumentArray()); } catch (Exception e) { LOG.error( MessageFormat.format( "Exception occurred in launcher: {0}{1}", options.getApplicationClass().getName(), options.getApplicationArguments()), e); return CLIENT_ERROR; } } finally { disposeClassLoader(options.getApplicationClassLoader()); for (File file : options.getApplicationCacheDirectories()) { if (delete(file) == false) { LOG.warn( MessageFormat.format("Failed to delete the application cache directory: {0}", file)); } } } }
/** * Modify configuration according user-specified generic options * * @param conf Configuration to be modified * @param line User-specified generic options */ private void processGeneralOptions(Configuration conf, CommandLine line) throws IOException { if (line.hasOption("fs")) { FileSystem.setDefaultUri(conf, line.getOptionValue("fs")); } if (line.hasOption("jt")) { conf.set("mapred.job.tracker", line.getOptionValue("jt"), "from -jt command line option"); } if (line.hasOption("conf")) { String[] values = line.getOptionValues("conf"); for (String value : values) { conf.addResource(new Path(value)); } } if (line.hasOption("libjars")) { conf.set( "tmpjars", validateFiles(line.getOptionValue("libjars"), conf), "from -libjars command line option"); // setting libjars in client classpath URL[] libjars = getLibJars(conf); if (libjars != null && libjars.length > 0) { conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader())); Thread.currentThread() .setContextClassLoader( new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader())); } } if (line.hasOption("files")) { conf.set( "tmpfiles", validateFiles(line.getOptionValue("files"), conf), "from -files command line option"); } if (line.hasOption("archives")) { conf.set( "tmparchives", validateFiles(line.getOptionValue("archives"), conf), "from -archives command line option"); } if (line.hasOption('D')) { String[] property = line.getOptionValues('D'); for (String prop : property) { String[] keyval = prop.split("=", 2); if (keyval.length == 2) { conf.set(keyval[0], keyval[1], "from command line"); } } } conf.setBoolean("mapreduce.client.genericoptionsparser.used", true); // tokensFile if (line.hasOption("tokenCacheFile")) { String fileName = line.getOptionValue("tokenCacheFile"); // check if the local file exists FileSystem localFs = FileSystem.getLocal(conf); Path p = new Path(fileName); if (!localFs.exists(p)) { throw new FileNotFoundException("File " + fileName + " does not exist."); } if (LOG.isDebugEnabled()) { LOG.debug("setting conf tokensFile: " + fileName); } conf.set( "mapreduce.job.credentials.json", localFs.makeQualified(p).toString(), "from -tokenCacheFile command line option"); } }