private static void addPathToConfig(Configuration conf, File path) {
// chain-in a new classloader
URL fileUrl = null;
try {
fileUrl = path.toURL();
} catch (MalformedURLException e) {
throw new RuntimeException("Erroneous config file path", e);
}
URL[] urls = {fileUrl};
ClassLoader cl = new URLClassLoader(urls, conf.getClassLoader());
conf.setClassLoader(cl);
}
private static int launch(Configuration conf, Tool tool, String[] args) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug(
MessageFormat.format(
"Launching application: {0}{1}", //$NON-NLS-1$
tool.getClass().getName(), Arrays.toString(args)));
}
ClassLoader contextClassLoader = switchContextClassLoader(conf.getClassLoader());
try {
return ToolRunner.run(conf, tool, args);
} finally {
switchContextClassLoader(contextClassLoader);
}
}
private static Dataset instantiate(@Nullable Configuration conf, Id.DatasetInstance dsInstanceId)
throws IOException {
ContextManager.Context context = ContextManager.getContext(conf);
if (context == null) {
throw new NullJobConfException();
}
Id.DatasetInstance datasetInstanceId =
dsInstanceId != null ? dsInstanceId : getDatasetInstanceId(conf);
if (datasetInstanceId == null) {
throw new IOException("Dataset name property could not be found.");
}
try {
DatasetFramework framework = context.getDatasetFramework();
if (conf == null) {
return framework.getDataset(datasetInstanceId, DatasetDefinition.NO_ARGUMENTS, null);
}
String queryId = conf.get(Constants.Explore.QUERY_ID);
if (queryId == null) {
throw new IOException("QueryId property could not be found");
}
QueryHandle queryHandle = QueryHandle.fromId(queryId);
ClassLoader classLoader =
DATASET_CLASSLOADER_MAP.getUnchecked(queryHandle).get(datasetInstanceId);
Dataset dataset;
if (classLoader == null) {
classLoader = conf.getClassLoader();
dataset = firstLoad(framework, queryHandle, datasetInstanceId, classLoader);
} else {
dataset =
framework.getDataset(datasetInstanceId, DatasetDefinition.NO_ARGUMENTS, classLoader);
}
return dataset;
} catch (DatasetManagementException e) {
throw new IOException(e);
} finally {
context.close();
}
}
/**
* Modify configuration according user-specified generic options
*
* @param conf Configuration to be modified
* @param line User-specified generic options
*/
private void processGeneralOptions(Configuration conf, CommandLine line) throws IOException {
if (line.hasOption("fs")) {
FileSystem.setDefaultUri(conf, line.getOptionValue("fs"));
}
if (line.hasOption("jt")) {
conf.set("mapred.job.tracker", line.getOptionValue("jt"), "from -jt command line option");
}
if (line.hasOption("conf")) {
String[] values = line.getOptionValues("conf");
for (String value : values) {
conf.addResource(new Path(value));
}
}
if (line.hasOption("libjars")) {
conf.set(
"tmpjars",
validateFiles(line.getOptionValue("libjars"), conf),
"from -libjars command line option");
// setting libjars in client classpath
URL[] libjars = getLibJars(conf);
if (libjars != null && libjars.length > 0) {
conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
Thread.currentThread()
.setContextClassLoader(
new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
}
}
if (line.hasOption("files")) {
conf.set(
"tmpfiles",
validateFiles(line.getOptionValue("files"), conf),
"from -files command line option");
}
if (line.hasOption("archives")) {
conf.set(
"tmparchives",
validateFiles(line.getOptionValue("archives"), conf),
"from -archives command line option");
}
if (line.hasOption('D')) {
String[] property = line.getOptionValues('D');
for (String prop : property) {
String[] keyval = prop.split("=", 2);
if (keyval.length == 2) {
conf.set(keyval[0], keyval[1], "from command line");
}
}
}
conf.setBoolean("mapreduce.client.genericoptionsparser.used", true);
// tokensFile
if (line.hasOption("tokenCacheFile")) {
String fileName = line.getOptionValue("tokenCacheFile");
// check if the local file exists
FileSystem localFs = FileSystem.getLocal(conf);
Path p = new Path(fileName);
if (!localFs.exists(p)) {
throw new FileNotFoundException("File " + fileName + " does not exist.");
}
if (LOG.isDebugEnabled()) {
LOG.debug("setting conf tokensFile: " + fileName);
}
conf.set(
"mapreduce.job.credentials.json",
localFs.makeQualified(p).toString(),
"from -tokenCacheFile command line option");
}
}
