/** * Obtain the tokens needed by the job and put them in the UGI * * @param conf */ protected void downloadTokensAndSetupUGI(Configuration conf) { try { this.currentUser = UserGroupInformation.getCurrentUser(); if (UserGroupInformation.isSecurityEnabled()) { // Read the file-system tokens from the localized tokens-file. Path jobSubmitDir = FileContext.getLocalFSFileContext() .makeQualified( new Path(new File(DragonJobConfig.JOB_SUBMIT_DIR).getAbsolutePath())); Path jobTokenFile = new Path(jobSubmitDir, DragonJobConfig.APPLICATION_TOKENS_FILE); fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf)); LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile=" + jobTokenFile); for (Token<? extends TokenIdentifier> tk : fsTokens.getAllTokens()) { if (LOG.isDebugEnabled()) { LOG.debug( "Token of kind " + tk.getKind() + "in current ugi in the AppMaster for service " + tk.getService()); } currentUser.addToken(tk); // For use by AppMaster itself. } } } catch (IOException e) { throw new YarnException(e); } }
@SuppressWarnings("unchecked") private void readTokensFromFiles(Configuration conf, Credentials credentials) throws IOException { // add tokens and secrets coming from a token storage file String binaryTokenFilename = conf.get("mapreduce.job.credentials.binary"); if (binaryTokenFilename != null) { Credentials binary = Credentials.readTokenStorageFile(new Path("file:///" + binaryTokenFilename), conf); credentials.addAll(binary); } // add secret keys coming from a json file String tokensFileName = conf.get("mapreduce.job.credentials.json"); if (tokensFileName != null) { LOG.info("loading user's secret keys from " + tokensFileName); String localFileName = new Path(tokensFileName).toUri().getPath(); boolean json_error = false; try { // read JSON ObjectMapper mapper = new ObjectMapper(); Map<String, String> nm = mapper.readValue(new File(localFileName), Map.class); for (Map.Entry<String, String> ent : nm.entrySet()) { credentials.addSecretKey(new Text(ent.getKey()), ent.getValue().getBytes()); } } catch (JsonMappingException e) { json_error = true; } catch (JsonParseException e) { json_error = true; } if (json_error) LOG.warn("couldn't parse Token Cache JSON file with user secret keys"); } }
/** * Modify configuration according user-specified generic options * * @param conf Configuration to be modified * @param line User-specified generic options */ private void processGeneralOptions(Configuration conf, CommandLine line) throws IOException { if (line.hasOption("fs")) { FileSystem.setDefaultUri(conf, line.getOptionValue("fs")); } if (line.hasOption("jt")) { String optionValue = line.getOptionValue("jt"); if (optionValue.equalsIgnoreCase("local")) { conf.set("mapreduce.framework.name", optionValue); } conf.set("yarn.resourcemanager.address", optionValue, "from -jt command line option"); } if (line.hasOption("conf")) { String[] values = line.getOptionValues("conf"); for (String value : values) { conf.addResource(new Path(value)); } } if (line.hasOption("libjars")) { conf.set( "tmpjars", validateFiles(line.getOptionValue("libjars"), conf), "from -libjars command line option"); // setting libjars in client classpath URL[] libjars = getLibJars(conf); if (libjars != null && libjars.length > 0) { conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader())); Thread.currentThread() .setContextClassLoader( new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader())); } } if (line.hasOption("files")) { conf.set( "tmpfiles", validateFiles(line.getOptionValue("files"), conf), "from -files command line option"); } if (line.hasOption("archives")) { conf.set( "tmparchives", validateFiles(line.getOptionValue("archives"), conf), "from -archives command line option"); } if (line.hasOption('D')) { String[] property = line.getOptionValues('D'); for (String prop : property) { String[] keyval = prop.split("=", 2); if (keyval.length == 2) { conf.set(keyval[0], keyval[1], "from command line"); } } } conf.setBoolean("mapreduce.client.genericoptionsparser.used", true); // tokensFile if (line.hasOption("tokenCacheFile")) { String fileName = line.getOptionValue("tokenCacheFile"); // check if the local file exists FileSystem localFs = FileSystem.getLocal(conf); Path p = localFs.makeQualified(new Path(fileName)); if (!localFs.exists(p)) { throw new FileNotFoundException("File " + fileName + " does not exist."); } if (LOG.isDebugEnabled()) { LOG.debug("setting conf tokensFile: " + fileName); } UserGroupInformation.getCurrentUser() .addCredentials(Credentials.readTokenStorageFile(p, conf)); conf.set( "mapreduce.job.credentials.json", p.toString(), "from -tokenCacheFile command line option"); } }
private static Collection<Token<?>> readTokens(Path file, Configuration conf) throws IOException { Credentials creds = Credentials.readTokenStorageFile(file, conf); return creds.getAllTokens(); }