/**
* Obtain the tokens needed by the job and put them in the UGI
*
* @param conf
*/
protected void downloadTokensAndSetupUGI(Configuration conf) {
try {
this.currentUser = UserGroupInformation.getCurrentUser();
if (UserGroupInformation.isSecurityEnabled()) {
// Read the file-system tokens from the localized tokens-file.
Path jobSubmitDir =
FileContext.getLocalFSFileContext()
.makeQualified(
new Path(new File(DragonJobConfig.JOB_SUBMIT_DIR).getAbsolutePath()));
Path jobTokenFile = new Path(jobSubmitDir, DragonJobConfig.APPLICATION_TOKENS_FILE);
fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf));
LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile=" + jobTokenFile);
for (Token<? extends TokenIdentifier> tk : fsTokens.getAllTokens()) {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Token of kind "
+ tk.getKind()
+ "in current ugi in the AppMaster for service "
+ tk.getService());
}
currentUser.addToken(tk); // For use by AppMaster itself.
}
}
} catch (IOException e) {
throw new YarnException(e);
}
}
@SuppressWarnings("unchecked")
private void readTokensFromFiles(Configuration conf, Credentials credentials) throws IOException {
// add tokens and secrets coming from a token storage file
String binaryTokenFilename = conf.get("mapreduce.job.credentials.binary");
if (binaryTokenFilename != null) {
Credentials binary =
Credentials.readTokenStorageFile(new Path("file:///" + binaryTokenFilename), conf);
credentials.addAll(binary);
}
// add secret keys coming from a json file
String tokensFileName = conf.get("mapreduce.job.credentials.json");
if (tokensFileName != null) {
LOG.info("loading user's secret keys from " + tokensFileName);
String localFileName = new Path(tokensFileName).toUri().getPath();
boolean json_error = false;
try {
// read JSON
ObjectMapper mapper = new ObjectMapper();
Map<String, String> nm = mapper.readValue(new File(localFileName), Map.class);
for (Map.Entry<String, String> ent : nm.entrySet()) {
credentials.addSecretKey(new Text(ent.getKey()), ent.getValue().getBytes());
}
} catch (JsonMappingException e) {
json_error = true;
} catch (JsonParseException e) {
json_error = true;
}
if (json_error) LOG.warn("couldn't parse Token Cache JSON file with user secret keys");
}
}
/**
* Modify configuration according user-specified generic options
*
* @param conf Configuration to be modified
* @param line User-specified generic options
*/
private void processGeneralOptions(Configuration conf, CommandLine line) throws IOException {
if (line.hasOption("fs")) {
FileSystem.setDefaultUri(conf, line.getOptionValue("fs"));
}
if (line.hasOption("jt")) {
String optionValue = line.getOptionValue("jt");
if (optionValue.equalsIgnoreCase("local")) {
conf.set("mapreduce.framework.name", optionValue);
}
conf.set("yarn.resourcemanager.address", optionValue, "from -jt command line option");
}
if (line.hasOption("conf")) {
String[] values = line.getOptionValues("conf");
for (String value : values) {
conf.addResource(new Path(value));
}
}
if (line.hasOption("libjars")) {
conf.set(
"tmpjars",
validateFiles(line.getOptionValue("libjars"), conf),
"from -libjars command line option");
// setting libjars in client classpath
URL[] libjars = getLibJars(conf);
if (libjars != null && libjars.length > 0) {
conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
Thread.currentThread()
.setContextClassLoader(
new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
}
}
if (line.hasOption("files")) {
conf.set(
"tmpfiles",
validateFiles(line.getOptionValue("files"), conf),
"from -files command line option");
}
if (line.hasOption("archives")) {
conf.set(
"tmparchives",
validateFiles(line.getOptionValue("archives"), conf),
"from -archives command line option");
}
if (line.hasOption('D')) {
String[] property = line.getOptionValues('D');
for (String prop : property) {
String[] keyval = prop.split("=", 2);
if (keyval.length == 2) {
conf.set(keyval[0], keyval[1], "from command line");
}
}
}
conf.setBoolean("mapreduce.client.genericoptionsparser.used", true);
// tokensFile
if (line.hasOption("tokenCacheFile")) {
String fileName = line.getOptionValue("tokenCacheFile");
// check if the local file exists
FileSystem localFs = FileSystem.getLocal(conf);
Path p = localFs.makeQualified(new Path(fileName));
if (!localFs.exists(p)) {
throw new FileNotFoundException("File " + fileName + " does not exist.");
}
if (LOG.isDebugEnabled()) {
LOG.debug("setting conf tokensFile: " + fileName);
}
UserGroupInformation.getCurrentUser()
.addCredentials(Credentials.readTokenStorageFile(p, conf));
conf.set(
"mapreduce.job.credentials.json",
p.toString(),
"from -tokenCacheFile command line option");
}
}
private static Collection<Token<?>> readTokens(Path file, Configuration conf) throws IOException {
Credentials creds = Credentials.readTokenStorageFile(file, conf);
return creds.getAllTokens();
}
