/**
* Getter for proxiedFs, using the passed parameters to create an instance of a proxiedFs.
*
* @param properties
* @param authType is either TOKEN or KEYTAB.
* @param authPath is the KEYTAB location if the authType is KEYTAB; otherwise, it is the token
* file.
* @param uri File system URI.
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
* @return proxiedFs
*/
public FileSystem getProxiedFileSystem(
State properties, AuthType authType, String authPath, String uri)
throws IOException, InterruptedException, URISyntaxException {
Preconditions.checkArgument(
StringUtils.isNotBlank(properties.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME)),
"State does not contain a proper proxy user name");
String proxyUserName = properties.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME);
UserGroupInformation proxyUser;
switch (authType) {
case KEYTAB: // If the authentication type is KEYTAB, log in a super user first before
// creating a proxy user.
Preconditions.checkArgument(
StringUtils.isNotBlank(
properties.getProp(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS)),
"State does not contain a proper proxy token file name");
String superUser = properties.getProp(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS);
UserGroupInformation.loginUserFromKeytab(superUser, authPath);
proxyUser =
UserGroupInformation.createProxyUser(
proxyUserName, UserGroupInformation.getLoginUser());
break;
case TOKEN: // If the authentication type is TOKEN, create a proxy user and then add the token
// to the user.
proxyUser =
UserGroupInformation.createProxyUser(
proxyUserName, UserGroupInformation.getLoginUser());
Optional<Token> proxyToken = this.getTokenFromSeqFile(authPath, proxyUserName);
if (proxyToken.isPresent()) {
proxyUser.addToken(proxyToken.get());
} else {
LOG.warn("No delegation token found for the current proxy user.");
}
break;
default:
LOG.warn(
"Creating a proxy user without authentication, which could not perform File system operations.");
proxyUser =
UserGroupInformation.createProxyUser(
proxyUserName, UserGroupInformation.getLoginUser());
break;
}
final Configuration conf = new Configuration();
JobConfigurationUtils.putStateIntoConfiguration(properties, conf);
final URI fsURI = URI.create(uri);
proxyUser.doAs(
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws IOException {
LOG.debug(
"Now performing file system operations as :"
+ UserGroupInformation.getCurrentUser());
proxiedFs = FileSystem.get(fsURI, conf);
return null;
}
});
return this.proxiedFs;
}
private HCatClient getHCatClient(URI uri, Configuration conf, String user)
throws HCatAccessorException {
final HiveConf hiveConf = new HiveConf(conf, this.getClass());
String serverURI = getMetastoreConnectURI(uri);
if (!serverURI.equals("")) {
hiveConf.set("hive.metastore.local", "false");
}
hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, serverURI);
try {
XLog.getLog(HCatURIHandler.class)
.info(
"Creating HCatClient for user [{0}] login_user [{1}] and server [{2}] ",
user, UserGroupInformation.getLoginUser(), serverURI);
// HiveMetastoreClient (hive 0.9) currently does not work if UGI has doAs
// We are good to connect as the oozie user since listPartitions does not require
// authorization
/*
UserGroupInformation ugi = ugiService.getProxyUser(user);
return ugi.doAs(new PrivilegedExceptionAction<HCatClient>() {
public HCatClient run() throws Exception {
return HCatClient.create(hiveConf);
}
});
*/
return HCatClient.create(hiveConf);
} catch (HCatException e) {
throw new HCatAccessorException(ErrorCode.E1501, e);
} catch (IOException e) {
throw new HCatAccessorException(ErrorCode.E1501, e);
}
}
private <T> T doWithState(RunningQueryState state, PrivilegedExceptionAction<T> action)
throws BeeswaxException {
try {
UserGroupInformation ugi;
if (UserGroupInformation.isSecurityEnabled())
ugi =
UserGroupInformation.createProxyUser(
state.query.hadoop_user, UserGroupInformation.getLoginUser());
else {
ugi = UserGroupInformation.createRemoteUser(state.query.hadoop_user);
}
return ugi.doAs(action);
} catch (UndeclaredThrowableException e) {
if (e.getUndeclaredThrowable() instanceof PrivilegedActionException) {
Throwable bwe = e.getUndeclaredThrowable().getCause();
if (bwe instanceof BeeswaxException) {
LOG.error("Caught BeeswaxException", (BeeswaxException) bwe);
throw (BeeswaxException) bwe;
}
}
LOG.error("Caught unexpected exception.", e);
throw new BeeswaxException(e.getMessage(), state.handle.log_context, state.handle);
} catch (IOException e) {
LOG.error("Caught IOException", e);
throw new BeeswaxException(e.getMessage(), state.handle.log_context, state.handle);
} catch (InterruptedException e) {
LOG.error("Caught InterruptedException", e);
throw new BeeswaxException(e.getMessage(), state.handle.log_context, state.handle);
}
}
private static DistributedFileSystem getDFSForToken(
Token<DelegationTokenIdentifier> token, final Configuration conf) throws Exception {
DistributedFileSystem dfs = null;
try {
// TODO: The service is usually an IPaddress:port. We convert
// it to dns name and then obtain the filesystem just so that
// we reuse the existing filesystem handle (that the jobtracker
// might have for this namenode; the namenode is usually
// specified as the dns name in the jobtracker).
// THIS IS A WORKAROUND FOR NOW. NEED TO SOLVE THIS PROBLEM
// IN A BETTER WAY.
String[] ipaddr = token.getService().toString().split(":");
InetAddress iaddr = InetAddress.getByName(ipaddr[0]);
String dnsName = iaddr.getCanonicalHostName();
final URI uri = new URI(SCHEME + "://" + dnsName + ":" + ipaddr[1]);
dfs =
(DistributedFileSystem)
UserGroupInformation.getLoginUser()
.doAs(
new PrivilegedExceptionAction<DistributedFileSystem>() {
public DistributedFileSystem run() throws IOException {
return (DistributedFileSystem) FileSystem.get(uri, conf);
}
});
} catch (Exception e) {
LOG.warn("Failed to create a dfs to renew for:" + token.getService(), e);
throw e;
}
return dfs;
}
/**
* Initiate a copy operation using a command-line style (arguments are specified as {@link
* String}s).
*
* @param arguments the copy arguments
*/
public void copy(String... arguments) {
Assert.notEmpty(arguments, "invalid number of arguments");
// sanitize the arguments
final List<String> parsedArguments = new ArrayList<String>();
for (String arg : arguments) {
parsedArguments.addAll(Arrays.asList(StringUtils.tokenizeToStringArray(arg, " ")));
}
try {
if (StringUtils.hasText(user)) {
UserGroupInformation ugi =
UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
ugi.doAs(
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
invokeCopy(
configuration, parsedArguments.toArray(new String[parsedArguments.size()]));
return null;
}
});
} else {
invokeCopy(configuration, parsedArguments.toArray(new String[parsedArguments.size()]));
}
} catch (Exception ex) {
throw new IllegalStateException("Cannot run distCp impersonated as '" + user + "'", ex);
}
}
// 初始化环境
static {
// Properties props = getProperties("hadoop.properties");
conf = new Configuration();
conf.addResource(new Path("F:\\hdfs-site.xml"));
conf.addResource(new Path("F:\\hbase-site.xml"));
conf.set("fs.defaultFS", "hdfs://ju51nn");
try {
String ugi = "";
if (Strings.isNullOrEmpty(ugi)) {
fs = FileSystem.get(conf);
System.out.println("======" + fs);
} else {
UserGroupInformation.createProxyUser(ugi, UserGroupInformation.getLoginUser())
.doAs(
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
fs = FileSystem.get(conf);
return null;
}
});
}
} catch (Exception e) {
LOG.error("初始化FileSytem对象异常: ", e.getMessage());
e.printStackTrace();
}
}
@Test(timeout = 2000)
public void testCleanupQueueClosesFilesystem()
throws IOException, InterruptedException, NoSuchFieldException, IllegalAccessException {
Configuration conf = new Configuration();
File file = new File("afile.txt");
file.createNewFile();
Path path = new Path(file.getAbsoluteFile().toURI());
FileSystem.get(conf);
Assert.assertEquals(1, getFileSystemCacheSize());
// With UGI, should close FileSystem
CleanupQueue cleanupQueue = new CleanupQueue();
PathDeletionContext context =
new PathDeletionContext(path, conf, UserGroupInformation.getLoginUser(), null, null);
cleanupQueue.addToQueue(context);
while (getFileSystemCacheSize() > 0) {
Thread.sleep(100);
}
file.createNewFile();
FileSystem.get(conf);
Assert.assertEquals(1, getFileSystemCacheSize());
// Without UGI, should not close FileSystem
context = new PathDeletionContext(path, conf);
cleanupQueue.addToQueue(context);
while (file.exists()) {
Thread.sleep(100);
}
Assert.assertEquals(1, getFileSystemCacheSize());
}
/** Test login method */
@Test
public void testLogin() throws Exception {
// login from unix
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
assertEquals(UserGroupInformation.getCurrentUser(), UserGroupInformation.getLoginUser());
assertTrue(ugi.getGroupNames().length >= 1);
// ensure that doAs works correctly
UserGroupInformation userGroupInfo =
UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES);
UserGroupInformation curUGI =
userGroupInfo.doAs(
new PrivilegedExceptionAction<UserGroupInformation>() {
public UserGroupInformation run() throws IOException {
return UserGroupInformation.getCurrentUser();
}
});
// make sure in the scope of the doAs, the right user is current
assertEquals(curUGI, userGroupInfo);
// make sure it is not the same as the login user
assertFalse(curUGI.equals(UserGroupInformation.getLoginUser()));
}
public HBaseUtils(
String quorum,
boolean useKerberos,
String keyTabUsername,
String kerberosEnv,
String keyTabFileLocation,
int regions)
throws IOException {
this.regions = regions;
conf.set("hbase.zookeeper.quorum", quorum);
if (useKerberos) {
conf.set("hadoop.security.authentication", "Kerberos");
conf.set("hbase.security.authentication", "Kerberos");
conf.set("hbase.master.kerberos.principal", "hbase/_HOST@" + kerberosEnv + ".YOURDOMAIN.COM");
conf.set(
"hbase.regionserver.kerberos.principal",
"hbase/_HOST@" + kerberosEnv + ".YOURDOMAIN.COM");
conf.set("hbase.client.keyvalue.maxsize", "-1");
UserGroupInformation.setConfiguration(conf);
try {
UserGroupInformation.loginUserFromKeytab(
keyTabUsername + "@" + kerberosEnv + ".YOURDOMAIN.COM", keyTabFileLocation);
valid = true;
} catch (IOException e) {
e.printStackTrace();
valid = false;
}
kerberosRefresher.scheduleAtFixedRate(
() -> {
try {
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
if (ugi == null) {
Logger.error("KERBEROS GOT LOGGED OUT");
UserGroupInformation.loginUserFromKeytab(
keyTabUsername + "@" + kerberosEnv + ".YOURDOMAIN.COM", keyTabFileLocation);
} else {
ugi.checkTGTAndReloginFromKeytab();
}
} catch (IOException e) {
e.printStackTrace();
}
},
KERBEROS_EXPIRATION_HOURS,
KERBEROS_EXPIRATION_HOURS,
TimeUnit.HOURS);
} else {
valid = true;
conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase-unsecure");
}
connection = ConnectionFactory.createConnection(conf);
}
public int run(final String[] argv) throws IOException, InterruptedException {
int val = -1;
final Configuration conf = getConf();
UserGroupInformation.setConfiguration(conf);
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
val =
ugi.doAs(
new PrivilegedExceptionAction<Integer>() {
public Integer run() throws Exception {
return runJob(conf, argv);
}
});
return val;
}
protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials)
throws IOException, InterruptedException {
// Get new hdfs tokens on behalf of this user
UserGroupInformation proxyUser =
UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
Token<?>[] newTokens =
proxyUser.doAs(
new PrivilegedExceptionAction<Token<?>[]>() {
@Override
public Token<?>[] run() throws Exception {
return FileSystem.get(getConfig())
.addDelegationTokens(
UserGroupInformation.getLoginUser().getUserName(), credentials);
}
});
return newTokens;
}
/**
* Initializes the staging directory and returns the path. It also keeps track of all necessary
* ownership & permissions
*
* @param client
* @param conf
*/
public static Path getStagingDir(JobClient client, Configuration conf)
throws IOException, InterruptedException {
Path stagingArea = client.getStagingAreaDir();
FileSystem fs = stagingArea.getFileSystem(conf);
String realUser;
String currentUser;
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
realUser = ugi.getShortUserName();
currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
if (fs.exists(stagingArea)) {
FileStatus fsStatus = fs.getFileStatus(stagingArea);
String owner = fsStatus.getOwner();
if (!(owner.equals(currentUser) || owner.equals(realUser))) {
throw new IOException(
"The ownership on the staging directory "
+ stagingArea
+ " is not as expected. "
+ "It is owned by "
+ owner
+ ". The directory must "
+ "be owned by the submitter "
+ currentUser
+ " or "
+ "by "
+ realUser);
}
if (!fsStatus.getPermission().equals(JOB_DIR_PERMISSION)) {
LOG.info(
"Permissions on staging directory "
+ stagingArea
+ " are "
+ "incorrect: "
+ fsStatus.getPermission()
+ ". Fixing permissions "
+ "to correct value "
+ JOB_DIR_PERMISSION);
fs.setPermission(stagingArea, JOB_DIR_PERMISSION);
}
} else {
fs.mkdirs(stagingArea, new FsPermission(JOB_DIR_PERMISSION));
}
return stagingArea;
}
public static void initProcessSecurity(Configuration conf)
throws IOException, BadConfigException {
log.info("Secure mode with kerberos realm {}", HoyaUtils.getKerberosRealm());
// this gets UGI to reset its previous world view (i.e simple auth)
// security
SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
UserGroupInformation.setConfiguration(conf);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
log.debug("Authenticating as " + ugi.toString());
log.debug("Login user is {}", UserGroupInformation.getLoginUser());
if (!UserGroupInformation.isSecurityEnabled()) {
throw new BadConfigException(
"Although secure mode is enabled,"
+ "the application has already set up its user as an insecure entity %s",
ugi);
}
HoyaUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
HoyaUtils.verifyPrincipalSet(conf, DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
}
// renew a token
@VisibleForTesting
protected void renewToken(final DelegationTokenToRenew dttr) throws IOException {
// need to use doAs so that http can find the kerberos tgt
// NOTE: token renewers should be responsible for the correct UGI!
try {
dttr.expirationDate =
UserGroupInformation.getLoginUser()
.doAs(
new PrivilegedExceptionAction<Long>() {
@Override
public Long run() throws Exception {
return dttr.token.renew(dttr.conf);
}
});
} catch (InterruptedException e) {
throw new IOException(e);
}
LOG.info("Renewed delegation-token= [" + dttr + "], for " + dttr.applicationId);
}
private void testDelegationTokenWithinDoAs(final Class fileSystemClass, boolean proxyUser)
throws Exception {
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab("client", "/Users/tucu/tucu.keytab");
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
if (proxyUser) {
ugi = UserGroupInformation.createProxyUser("foo", ugi);
}
conf = new Configuration();
UserGroupInformation.setConfiguration(conf);
ugi.doAs(
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
testDelegationTokenWithFS(fileSystemClass);
return null;
}
});
}
public void run() {
TokenWithConf tokenWithConf = null;
while (true) {
try {
tokenWithConf = queue.take();
final TokenWithConf current = tokenWithConf;
if (LOG.isDebugEnabled()) {
LOG.debug("Cancelling token " + tokenWithConf.token.getService());
}
// need to use doAs so that http can find the kerberos tgt
UserGroupInformation.getLoginUser()
.doAs(
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
current.token.cancel(current.conf);
return null;
}
});
} catch (IOException e) {
LOG.warn(
"Failed to cancel token "
+ tokenWithConf.token
+ " "
+ StringUtils.stringifyException(e));
} catch (RuntimeException e) {
LOG.warn(
"Failed to cancel token "
+ tokenWithConf.token
+ " "
+ StringUtils.stringifyException(e));
} catch (InterruptedException ie) {
return;
} catch (Throwable t) {
LOG.warn("Got exception " + StringUtils.stringifyException(t) + ". Exiting..");
System.exit(-1);
}
}
}
public int run(final String[] argv) throws IOException, InterruptedException {
int val = -1;
final Configuration conf = getConf();
UserGroupInformation.setConfiguration(conf);
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
val =
ugi.doAs(
new PrivilegedExceptionAction<Integer>() {
public Integer run() throws Exception {
return runJob(conf, argv);
}
});
// print the gridmix summary if the run was successful
if (val == 0) {
// print the run summary
System.out.print("\n\n");
System.out.println(summarizer.toString());
}
return val;
}
/** Create the job tracker and run it. */
public void run() {
try {
jc = (jc == null) ? createJobConf() : createJobConf(jc);
File f = new File("build/test/mapred/local").getAbsoluteFile();
jc.set(MRConfig.LOCAL_DIR, f.getAbsolutePath());
jc.setClass(
"topology.node.switch.mapping.impl", StaticMapping.class, DNSToSwitchMapping.class);
final String id = new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date());
if (ugi == null) {
ugi = UserGroupInformation.getLoginUser();
}
tracker =
ugi.doAs(
new PrivilegedExceptionAction<JobTracker>() {
public JobTracker run() throws InterruptedException, IOException {
return JobTracker.startTracker(jc, clock, id);
}
});
tracker.offerService();
} catch (Throwable e) {
LOG.error("Job tracker crashed", e);
isActive = false;
}
}
// todo: this doesn;t check if compaction is already running (even though Initiator does but we
// don't go through Initiator for user initiated compactions)
@Override
public void run() {
do {
boolean launchedJob = false;
// Make sure nothing escapes this run method and kills the metastore at large,
// so wrap it in a big catch Throwable statement.
try {
final CompactionInfo ci = txnHandler.findNextToCompact(name);
if (ci == null && !stop.get()) {
try {
Thread.sleep(SLEEP_TIME);
continue;
} catch (InterruptedException e) {
LOG.warn("Worker thread sleep interrupted " + e.getMessage());
continue;
}
}
// Find the table we will be working with.
Table t1 = null;
try {
t1 = resolveTable(ci);
if (t1 == null) {
LOG.info(
"Unable to find table "
+ ci.getFullTableName()
+ ", assuming it was dropped and moving on.");
txnHandler.markCleaned(ci);
continue;
}
} catch (MetaException e) {
txnHandler.markCleaned(ci);
continue;
}
// This chicanery is to get around the fact that the table needs to be final in order to
// go into the doAs below.
final Table t = t1;
// Find the partition we will be working with, if there is one.
Partition p = null;
try {
p = resolvePartition(ci);
if (p == null && ci.partName != null) {
LOG.info(
"Unable to find partition "
+ ci.getFullPartitionName()
+ ", assuming it was dropped and moving on.");
txnHandler.markCleaned(ci);
continue;
}
} catch (Exception e) {
txnHandler.markCleaned(ci);
continue;
}
// Find the appropriate storage descriptor
final StorageDescriptor sd = resolveStorageDescriptor(t, p);
// Check that the table or partition isn't sorted, as we don't yet support that.
if (sd.getSortCols() != null && !sd.getSortCols().isEmpty()) {
LOG.error("Attempt to compact sorted table, which is not yet supported!");
txnHandler.markCleaned(ci);
continue;
}
final boolean isMajor = ci.isMajorCompaction();
final ValidTxnList txns =
CompactionTxnHandler.createValidCompactTxnList(txnHandler.getOpenTxnsInfo());
LOG.debug("ValidCompactTxnList: " + txns.writeToString());
txnHandler.setCompactionHighestTxnId(ci, txns.getHighWatermark());
final StringBuilder jobName = new StringBuilder(name);
jobName.append("-compactor-");
jobName.append(ci.getFullPartitionName());
// Determine who to run as
String runAs;
if (ci.runAs == null) {
runAs = findUserToRunAs(sd.getLocation(), t);
txnHandler.setRunAs(ci.id, runAs);
} else {
runAs = ci.runAs;
}
LOG.info("Starting " + ci.type.toString() + " compaction for " + ci.getFullPartitionName());
final StatsUpdater su =
StatsUpdater.init(
ci,
txnHandler.findColumnsWithStats(ci),
conf,
runJobAsSelf(runAs) ? runAs : t.getOwner());
final CompactorMR mr = new CompactorMR();
launchedJob = true;
try {
if (runJobAsSelf(runAs)) {
mr.run(conf, jobName.toString(), t, sd, txns, ci, su);
} else {
UserGroupInformation ugi =
UserGroupInformation.createProxyUser(
t.getOwner(), UserGroupInformation.getLoginUser());
ugi.doAs(
new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
mr.run(conf, jobName.toString(), t, sd, txns, ci, su);
return null;
}
});
}
txnHandler.markCompacted(ci);
} catch (Exception e) {
LOG.error(
"Caught exception while trying to compact "
+ ci
+ ". Marking clean to avoid repeated failures, "
+ StringUtils.stringifyException(e));
txnHandler.markFailed(ci);
}
} catch (Throwable t) {
LOG.error(
"Caught an exception in the main loop of compactor worker "
+ name
+ ", "
+ StringUtils.stringifyException(t));
}
// If we didn't try to launch a job it either means there was no work to do or we got
// here as the result of a communication failure with the DB. Either way we want to wait
// a bit before we restart the loop.
if (!launchedJob && !stop.get()) {
try {
Thread.sleep(SLEEP_TIME);
} catch (InterruptedException e) {
}
}
} while (!stop.get());
}
/**
* Launch application for the dag represented by this client.
*
* @throws YarnException
* @throws IOException
*/
public void startApplication() throws YarnException, IOException {
Class<?>[] defaultClasses;
if (applicationType.equals(YARN_APPLICATION_TYPE)) {
// TODO restrict the security check to only check if security is enabled for webservices.
if (UserGroupInformation.isSecurityEnabled()) {
defaultClasses = DATATORRENT_SECURITY_CLASSES;
} else {
defaultClasses = DATATORRENT_CLASSES;
}
} else {
throw new IllegalStateException(applicationType + " is not a valid application type.");
}
LinkedHashSet<String> localJarFiles = findJars(dag, defaultClasses);
if (resources != null) {
localJarFiles.addAll(resources);
}
YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
LOG.info(
"Got Cluster metric info from ASM"
+ ", numNodeManagers="
+ clusterMetrics.getNumNodeManagers());
// GetClusterNodesRequest clusterNodesReq = Records.newRecord(GetClusterNodesRequest.class);
// GetClusterNodesResponse clusterNodesResp =
// rmClient.clientRM.getClusterNodes(clusterNodesReq);
// LOG.info("Got Cluster node info from ASM");
// for (NodeReport node : clusterNodesResp.getNodeReports()) {
// LOG.info("Got node report from ASM for"
// + ", nodeId=" + node.getNodeId()
// + ", nodeAddress" + node.getHttpAddress()
// + ", nodeRackName" + node.getRackName()
// + ", nodeNumContainers" + node.getNumContainers()
// + ", nodeHealthStatus" + node.getHealthReport());
// }
List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
for (QueueUserACLInfo aclInfo : listAclInfo) {
for (QueueACL userAcl : aclInfo.getUserAcls()) {
LOG.info(
"User ACL Info for Queue"
+ ", queueName="
+ aclInfo.getQueueName()
+ ", userAcl="
+ userAcl.name());
}
}
// Get a new application id
YarnClientApplication newApp = yarnClient.createApplication();
appId = newApp.getNewApplicationResponse().getApplicationId();
// Dump out information about cluster capability as seen by the resource manager
int maxMem = newApp.getNewApplicationResponse().getMaximumResourceCapability().getMemory();
LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
int amMemory = dag.getMasterMemoryMB();
if (amMemory > maxMem) {
LOG.info(
"AM memory specified above max threshold of cluster. Using max value."
+ ", specified="
+ amMemory
+ ", max="
+ maxMem);
amMemory = maxMem;
}
if (dag.getAttributes().get(LogicalPlan.APPLICATION_ID) == null) {
dag.setAttribute(LogicalPlan.APPLICATION_ID, appId.toString());
}
// Create launch context for app master
LOG.info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
// set the application id
appContext.setApplicationId(appId);
// set the application name
appContext.setApplicationName(dag.getValue(LogicalPlan.APPLICATION_NAME));
appContext.setApplicationType(this.applicationType);
if (YARN_APPLICATION_TYPE.equals(this.applicationType)) {
// appContext.setMaxAppAttempts(1); // no retries until Stram is HA
}
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
// Setup security tokens
// If security is enabled get ResourceManager and NameNode delegation tokens.
// Set these tokens on the container so that they are sent as part of application submission.
// This also sets them up for renewal by ResourceManager. The NameNode delegation rmToken
// is also used by ResourceManager to fetch the jars from HDFS and set them up for the
// application master launch.
if (UserGroupInformation.isSecurityEnabled()) {
Credentials credentials = new Credentials();
String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
if (tokenRenewer == null || tokenRenewer.length() == 0) {
throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
}
// For now, only getting tokens for the default file-system.
FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
try {
final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
if (tokens != null) {
for (Token<?> token : tokens) {
LOG.info("Got dt for " + fs.getUri() + "; " + token);
}
}
} finally {
fs.close();
}
addRMDelegationToken(tokenRenewer, credentials);
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
amContainer.setTokens(fsTokens);
}
// set local resources for the application master
// local files or archives as needed
// In this scenario, the jar file for the application master is part of the local resources
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
// copy required jar files to dfs, to be localized for containers
FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
try {
Path appsBasePath =
new Path(StramClientUtils.getDTDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
Path appPath = new Path(appsBasePath, appId.toString());
String libJarsCsv = copyFromLocal(fs, appPath, localJarFiles.toArray(new String[] {}));
LOG.info("libjars: {}", libJarsCsv);
dag.getAttributes().put(LogicalPlan.LIBRARY_JARS, libJarsCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.FILE, libJarsCsv, localResources, fs);
if (archives != null) {
String[] localFiles = archives.split(",");
String archivesCsv = copyFromLocal(fs, appPath, localFiles);
LOG.info("archives: {}", archivesCsv);
dag.getAttributes().put(LogicalPlan.ARCHIVES, archivesCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.ARCHIVE, archivesCsv, localResources, fs);
}
if (files != null) {
String[] localFiles = files.split(",");
String filesCsv = copyFromLocal(fs, appPath, localFiles);
LOG.info("files: {}", filesCsv);
dag.getAttributes().put(LogicalPlan.FILES, filesCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.FILE, filesCsv, localResources, fs);
}
dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, appPath.toString());
if (dag.getAttributes().get(OperatorContext.STORAGE_AGENT) == null) {
/* which would be the most likely case */
Path checkpointPath = new Path(appPath, LogicalPlan.SUBDIR_CHECKPOINTS);
// use conf client side to pickup any proxy settings from dt-site.xml
dag.setAttribute(
OperatorContext.STORAGE_AGENT, new FSStorageAgent(checkpointPath.toString(), conf));
}
if (dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR) == null) {
dag.setAttribute(
LogicalPlan.CONTAINER_OPTS_CONFIGURATOR, new BasicContainerOptConfigurator());
}
// Set the log4j properties if needed
if (!log4jPropFile.isEmpty()) {
Path log4jSrc = new Path(log4jPropFile);
Path log4jDst = new Path(appPath, "log4j.props");
fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
log4jRsrc.setType(LocalResourceType.FILE);
log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
log4jRsrc.setSize(log4jFileStatus.getLen());
localResources.put("log4j.properties", log4jRsrc);
}
if (originalAppId != null) {
Path origAppPath = new Path(appsBasePath, this.originalAppId);
LOG.info("Restart from {}", origAppPath);
copyInitialState(origAppPath);
}
// push logical plan to DFS location
Path cfgDst = new Path(appPath, LogicalPlan.SER_FILE_NAME);
FSDataOutputStream outStream = fs.create(cfgDst, true);
LogicalPlan.write(this.dag, outStream);
outStream.close();
Path launchConfigDst = new Path(appPath, LogicalPlan.LAUNCH_CONFIG_FILE_NAME);
outStream = fs.create(launchConfigDst, true);
conf.writeXml(outStream);
outStream.close();
FileStatus topologyFileStatus = fs.getFileStatus(cfgDst);
LocalResource topologyRsrc = Records.newRecord(LocalResource.class);
topologyRsrc.setType(LocalResourceType.FILE);
topologyRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
topologyRsrc.setResource(ConverterUtils.getYarnUrlFromURI(cfgDst.toUri()));
topologyRsrc.setTimestamp(topologyFileStatus.getModificationTime());
topologyRsrc.setSize(topologyFileStatus.getLen());
localResources.put(LogicalPlan.SER_FILE_NAME, topologyRsrc);
// Set local resource info into app master container launch context
amContainer.setLocalResources(localResources);
// Set the necessary security tokens as needed
// amContainer.setContainerTokens(containerToken);
// Set the env variables to be setup in the env where the application master will be run
LOG.info("Set the environment for the application master");
Map<String, String> env = new HashMap<String, String>();
// Add application jar(s) location to classpath
// At some point we should not be required to add
// the hadoop specific classpaths to the env.
// It should be provided out of the box.
// For now setting all required classpaths including
// the classpath to "." for the application jar(s)
// including ${CLASSPATH} will duplicate the class path in app master, removing it for now
// StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*");
StringBuilder classPathEnv = new StringBuilder("./*");
String classpath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
for (String c :
StringUtils.isBlank(classpath)
? YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
: classpath.split(",")) {
if (c.equals("$HADOOP_CLIENT_CONF_DIR")) {
// SPOI-2501
continue;
}
classPathEnv.append(':');
classPathEnv.append(c.trim());
}
env.put("CLASSPATH", classPathEnv.toString());
// propagate to replace node managers user name (effective in non-secure mode)
env.put("HADOOP_USER_NAME", UserGroupInformation.getLoginUser().getUserName());
amContainer.setEnvironment(env);
// Set the necessary command to execute the application master
ArrayList<CharSequence> vargs = new ArrayList<CharSequence>(30);
// Set java executable command
LOG.info("Setting up app master command");
vargs.add(javaCmd);
if (dag.isDebug()) {
vargs.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=n");
}
// Set Xmx based on am memory size
// default heap size 75% of total memory
vargs.add("-Xmx" + (amMemory * 3 / 4) + "m");
vargs.add("-XX:+HeapDumpOnOutOfMemoryError");
vargs.add("-XX:HeapDumpPath=/tmp/dt-heap-" + appId.getId() + ".bin");
vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
if (dag.isDebug()) {
vargs.add("-Dlog4j.debug=true");
}
String loggersLevel = conf.get(DTLoggerFactory.DT_LOGGERS_LEVEL);
if (loggersLevel != null) {
vargs.add(String.format("-D%s=%s", DTLoggerFactory.DT_LOGGERS_LEVEL, loggersLevel));
}
vargs.add(StreamingAppMaster.class.getName());
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");
// Get final command
StringBuilder command = new StringBuilder(9 * vargs.size());
for (CharSequence str : vargs) {
command.append(str).append(" ");
}
LOG.info("Completed setting up app master command " + command.toString());
List<String> commands = new ArrayList<String>();
commands.add(command.toString());
amContainer.setCommands(commands);
// Set up resource type requirements
// For now, only memory is supported so we set memory requirements
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(amMemory);
appContext.setResource(capability);
// Service data is a binary blob that can be passed to the application
// Not needed in this scenario
// amContainer.setServiceData(serviceData);
appContext.setAMContainerSpec(amContainer);
// Set the priority for the application master
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(amPriority);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue(queueName);
// Submit the application to the applications manager
// SubmitApplicationResponse submitResp = rmClient.submitApplication(appRequest);
// Ignore the response as either a valid response object is returned on success
// or an exception thrown to denote some form of a failure
String specStr =
Objects.toStringHelper("Submitting application: ")
.add("name", appContext.getApplicationName())
.add("queue", appContext.getQueue())
.add("user", UserGroupInformation.getLoginUser())
.add("resource", appContext.getResource())
.toString();
LOG.info(specStr);
if (dag.isDebug()) {
// LOG.info("Full submission context: " + appContext);
}
yarnClient.submitApplication(appContext);
} finally {
fs.close();
}
}
@SuppressWarnings("rawtypes")
public void afterPropertiesSet() throws Exception {
final Configuration cfg = ConfigurationUtils.createFrom(configuration, properties);
buildGenericOptions(cfg);
if (StringUtils.hasText(user)) {
UserGroupInformation ugi =
UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
ugi.doAs(
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
job = new Job(cfg);
return null;
}
});
} else {
job = new Job(cfg);
}
ClassLoader loader =
(beanClassLoader != null
? beanClassLoader
: org.springframework.util.ClassUtils.getDefaultClassLoader());
if (jar != null) {
JobConf conf = (JobConf) job.getConfiguration();
conf.setJar(jar.getURI().toString());
loader = ExecutionUtils.createParentLastClassLoader(jar, beanClassLoader, cfg);
conf.setClassLoader(loader);
}
// set first to enable auto-detection of K/V to skip the key/value types to be specified
if (mapper != null) {
Class<? extends Mapper> mapperClass = resolveClass(mapper, loader, Mapper.class);
job.setMapperClass(mapperClass);
configureMapperTypesIfPossible(job, mapperClass);
}
if (reducer != null) {
Class<? extends Reducer> reducerClass = resolveClass(reducer, loader, Reducer.class);
job.setReducerClass(reducerClass);
configureReducerTypesIfPossible(job, reducerClass);
}
if (StringUtils.hasText(name)) {
job.setJobName(name);
}
if (combiner != null) {
job.setCombinerClass(resolveClass(combiner, loader, Reducer.class));
}
if (groupingComparator != null) {
job.setGroupingComparatorClass(resolveClass(groupingComparator, loader, RawComparator.class));
}
if (inputFormat != null) {
job.setInputFormatClass(resolveClass(inputFormat, loader, InputFormat.class));
}
if (mapKey != null) {
job.setMapOutputKeyClass(resolveClass(mapKey, loader, Object.class));
}
if (mapValue != null) {
job.setMapOutputValueClass(resolveClass(mapValue, loader, Object.class));
}
if (numReduceTasks != null) {
job.setNumReduceTasks(numReduceTasks);
}
if (key != null) {
job.setOutputKeyClass(resolveClass(key, loader, Object.class));
}
if (value != null) {
job.setOutputValueClass(resolveClass(value, loader, Object.class));
}
if (outputFormat != null) {
job.setOutputFormatClass(resolveClass(outputFormat, loader, OutputFormat.class));
}
if (partitioner != null) {
job.setPartitionerClass(resolveClass(partitioner, loader, Partitioner.class));
}
if (sortComparator != null) {
job.setSortComparatorClass(resolveClass(sortComparator, loader, RawComparator.class));
}
if (StringUtils.hasText(workingDir)) {
job.setWorkingDirectory(new Path(workingDir));
}
if (jarClass != null) {
job.setJarByClass(jarClass);
}
if (!CollectionUtils.isEmpty(inputPaths)) {
for (String path : inputPaths) {
FileInputFormat.addInputPath(job, new Path(path));
}
}
if (StringUtils.hasText(outputPath)) {
FileOutputFormat.setOutputPath(job, new Path(outputPath));
}
if (compressOutput != null) {
FileOutputFormat.setCompressOutput(job, compressOutput);
}
if (codecClass != null) {
FileOutputFormat.setOutputCompressorClass(
job, resolveClass(codecClass, loader, CompressionCodec.class));
}
processJob(job);
}
private void validateHadoopFS(List<ConfigIssue> issues) {
boolean validHapoopFsUri = true;
hadoopConf = getHadoopConfiguration(issues);
String hdfsUriInConf;
if (hdfsUri != null && !hdfsUri.isEmpty()) {
hadoopConf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, hdfsUri);
} else {
hdfsUriInConf = hadoopConf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
if (hdfsUriInConf == null) {
issues.add(
getContext().createConfigIssue(Groups.HADOOP_FS.name(), "hdfsUri", Errors.HADOOPFS_19));
return;
} else {
hdfsUri = hdfsUriInConf;
}
}
if (hdfsUri.contains("://")) {
try {
URI uri = new URI(hdfsUri);
if (!"hdfs".equals(uri.getScheme())) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
"hdfsUri",
Errors.HADOOPFS_12,
hdfsUri,
uri.getScheme()));
validHapoopFsUri = false;
} else if (uri.getAuthority() == null) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(), "hdfsUri", Errors.HADOOPFS_13, hdfsUri));
validHapoopFsUri = false;
}
} catch (Exception ex) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
"hdfsUri",
Errors.HADOOPFS_22,
hdfsUri,
ex.getMessage(),
ex));
validHapoopFsUri = false;
}
} else {
issues.add(
getContext()
.createConfigIssue(Groups.HADOOP_FS.name(), "hdfsUri", Errors.HADOOPFS_02, hdfsUri));
validHapoopFsUri = false;
}
StringBuilder logMessage = new StringBuilder();
try {
// forcing UGI to initialize with the security settings from the stage
UserGroupInformation.setConfiguration(hadoopConf);
Subject subject = Subject.getSubject(AccessController.getContext());
if (UserGroupInformation.isSecurityEnabled()) {
loginUgi = UserGroupInformation.getUGIFromSubject(subject);
} else {
UserGroupInformation.loginUserFromSubject(subject);
loginUgi = UserGroupInformation.getLoginUser();
}
LOG.info(
"Subject = {}, Principals = {}, Login UGI = {}",
subject,
subject == null ? "null" : subject.getPrincipals(),
loginUgi);
if (hdfsKerberos) {
logMessage.append("Using Kerberos");
if (loginUgi.getAuthenticationMethod()
!= UserGroupInformation.AuthenticationMethod.KERBEROS) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
"hdfsKerberos",
Errors.HADOOPFS_00,
loginUgi.getAuthenticationMethod(),
UserGroupInformation.AuthenticationMethod.KERBEROS));
}
} else {
logMessage.append("Using Simple");
hadoopConf.set(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
UserGroupInformation.AuthenticationMethod.SIMPLE.name());
}
if (validHapoopFsUri) {
getUGI()
.doAs(
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (FileSystem fs = getFileSystemForInitDestroy()) { // to trigger the close
}
return null;
}
});
}
} catch (Exception ex) {
LOG.info("Error connecting to FileSystem: " + ex, ex);
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
null,
Errors.HADOOPFS_11,
hdfsUri,
String.valueOf(ex),
ex));
}
LOG.info("Authentication Config: " + logMessage);
}
protected String getJobOwnerName() throws IOException {
return UserGroupInformation.getLoginUser().getUserName();
}
