public void fromProto(TaskAttemptStartedProto proto) {
this.taskAttemptId = TezTaskAttemptID.fromString(proto.getTaskAttemptId());
this.launchTime = proto.getStartTime();
this.containerId = ConverterUtils.toContainerId(proto.getContainerId());
this.nodeId = ConverterUtils.toNodeId(proto.getNodeId());
this.creationTime = proto.getCreationTime();
this.allocationTime = proto.getAllocationTime();
if (proto.hasCreationCausalTA()) {
this.creationCausalTA = TezTaskAttemptID.fromString(proto.getCreationCausalTA());
}
}
public void write(DataOutputStream out) throws IOException {
for (String rootLogDir : this.rootLogDirs) {
File appLogDir =
new File(
rootLogDir,
ConverterUtils.toString(
this.containerId.getApplicationAttemptId().getApplicationId()));
File containerLogDir = new File(appLogDir, ConverterUtils.toString(this.containerId));
if (!containerLogDir.isDirectory()) {
continue; // ContainerDir may have been deleted by the user.
}
// Write out log files in lexical order
File[] logFiles = containerLogDir.listFiles();
Arrays.sort(logFiles);
for (File logFile : logFiles) {
// Write the logFile Type
out.writeUTF(logFile.getName());
// Write the log length as UTF so that it is printable
out.writeUTF(String.valueOf(logFile.length()));
// Write the log itself
FileInputStream in = null;
try {
in = SecureIOUtils.openForRead(logFile, getUser(), null);
byte[] buf = new byte[65535];
int len = 0;
while ((len = in.read(buf)) != -1) {
out.write(buf, 0, len);
}
} catch (IOException e) {
String message =
"Error aggregating log file. Log file : "
+ logFile.getAbsolutePath()
+ e.getMessage();
LOG.error(message, e);
out.write(message.getBytes());
} finally {
if (in != null) {
in.close();
}
}
}
}
}
public AppAttemptInfo(RMAppAttempt attempt) {
this.startTime = 0;
this.containerId = "";
this.nodeHttpAddress = "";
this.nodeId = "";
this.logsLink = "";
if (attempt != null) {
this.id = attempt.getAppAttemptId().getAttemptId();
this.startTime = attempt.getStartTime();
Container masterContainer = attempt.getMasterContainer();
if (masterContainer != null) {
this.containerId = masterContainer.getId().toString();
this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
this.nodeId = masterContainer.getNodeId().toString();
this.logsLink =
join(
HttpConfig.getSchemePrefix(),
masterContainer.getNodeHttpAddress(),
"/node",
"/containerlogs/",
ConverterUtils.toString(masterContainer.getId()),
"/",
attempt.getSubmissionContext().getUser());
}
}
}
@Override
protected void render(Block html) {
ContainerId containerID;
try {
containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
} catch (IllegalArgumentException e) {
html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
return;
}
DIV<Hamlet> div = html.div("#content");
Container container = this.nmContext.getContainers().get(containerID);
if (container == null) {
div.h1("Unknown Container. Container might have completed, "
+ "please go back to the previous page and retry.")._();
return;
}
ContainerStatus containerData = container.cloneAndGetContainerStatus();
int exitCode = containerData.getExitStatus();
String exiStatus =
(exitCode == YarnConfiguration.INVALID_CONTAINER_EXIT_STATUS) ?
"N/A" : String.valueOf(exitCode);
info("Container information")
._("ContainerID", $(CONTAINER_ID))
._("ContainerState", container.getContainerState())
._("ExitStatus", exiStatus)
._("Diagnostics", containerData.getDiagnostics())
._("User", container.getUser())
._("TotalMemoryNeeded",
container.getLaunchContext().getResource().getMemory())
._("logs", ujoin("containerlogs", $(CONTAINER_ID), container.getUser()),
"Link to logs");
html._(InfoBlock.class);
}
private static CommitterEventHandler createCommitterEventHandler(
Dispatcher dispatcher, OutputCommitter committer) {
final SystemClock clock = new SystemClock();
AppContext appContext = mock(AppContext.class);
when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
when(appContext.getClock()).thenReturn(clock);
RMHeartbeatHandler heartbeatHandler =
new RMHeartbeatHandler() {
@Override
public long getLastHeartbeatTime() {
return clock.getTime();
}
@Override
public void runOnNextHeartbeat(Runnable callback) {
callback.run();
}
};
ApplicationAttemptId id =
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
when(appContext.getApplicationAttemptId()).thenReturn(id);
CommitterEventHandler handler =
new CommitterEventHandler(appContext, committer, heartbeatHandler);
dispatcher.register(CommitterEventType.class, handler);
return handler;
}
private void addRMDelegationToken(final String renewer, final Credentials credentials)
throws IOException, YarnException {
// Get the ResourceManager delegation rmToken
final org.apache.hadoop.yarn.api.records.Token rmDelegationToken =
yarnClient.getRMDelegationToken(new Text(renewer));
Token<RMDelegationTokenIdentifier> token;
// TODO: Use the utility method getRMDelegationTokenService in ClientRMProxy to remove the
// separate handling of
// TODO: HA and non-HA cases when hadoop dependency is changed to hadoop 2.4 or above
if (conf.getBoolean(RM_HA_ENABLED, DEFAULT_RM_HA_ENABLED)) {
LOG.info("Yarn Resource Manager HA is enabled");
token = getRMHAToken(rmDelegationToken);
} else {
LOG.info("Yarn Resource Manager HA is not enabled");
InetSocketAddress rmAddress =
conf.getSocketAddr(
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_PORT);
token = ConverterUtils.convertFromYarn(rmDelegationToken, rmAddress);
}
LOG.info("RM dt {}", token);
credentials.addToken(token.getService(), token);
}
private void setupAppMasterJar(Path jarPath, LocalResource appMasterJar) throws IOException {
FileStatus jarStat = FileSystem.get(conf).getFileStatus(jarPath);
appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
appMasterJar.setSize(jarStat.getLen());
appMasterJar.setTimestamp(jarStat.getModificationTime());
appMasterJar.setType(LocalResourceType.FILE);
appMasterJar.setVisibility(LocalResourceVisibility.PUBLIC);
}
@Override
public String toString() {
this.readLock.lock();
try {
return ConverterUtils.toString(this.containerId);
} finally {
this.readLock.unlock();
}
}
public static LocalResource newLocalResource(
URI uri,
LocalResourceType type,
LocalResourceVisibility visibility,
long size,
long timestamp) {
return newLocalResource(
ConverterUtils.getYarnUrlFromURI(uri), type, visibility, size, timestamp);
}
public static void registerLocalResource(
FileSystem fs, Path remoteRsrcPath, LocalResource localResource) throws IOException {
FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
localResource.setSize(jarStat.getLen());
localResource.setTimestamp(jarStat.getModificationTime());
localResource.setType(LocalResourceType.FILE);
localResource.setVisibility(LocalResourceVisibility.PUBLIC);
}
public org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> getAMRMToken(
ApplicationId appId) throws YarnException, IOException {
Token token = getApplicationReport(appId).getAMRMToken();
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken = null;
if (token != null) {
amrmToken = ConverterUtils.convertFromYarn(token, (Text) null);
}
return amrmToken;
}
@Test(timeout = 15000)
public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception {
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
RMState rmState = memStore.getState();
Map<RMDelegationTokenIdentifier, Long> rmDTState =
rmState.getRMDTSecretManagerState().getTokenState();
Set<DelegationKey> rmDTMasterKeyState = rmState.getRMDTSecretManagerState().getMasterKeyState();
MockRM rm1 = new MyMockRM(conf, memStore);
rm1.start();
// on rm start, two master keys are created.
// One is created at RMDTSecretMgr.startThreads.updateCurrentKey();
// the other is created on the first run of
// tokenRemoverThread.rollMasterKey()
RMDelegationTokenSecretManager dtSecretManager = rm1.getRMDTSecretManager();
// assert all master keys are saved
Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState);
Set<DelegationKey> expiringKeys = new HashSet<DelegationKey>();
expiringKeys.addAll(dtSecretManager.getAllMasterKeys());
// record the current key
DelegationKey oldCurrentKey =
((TestRMDelegationTokenSecretManager) dtSecretManager).getCurrentKey();
// request to generate a RMDelegationToken
GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class);
when(request.getRenewer()).thenReturn("renewer1");
GetDelegationTokenResponse response = rm1.getClientRMService().getDelegationToken(request);
org.apache.hadoop.yarn.api.records.Token delegationToken = response.getRMDelegationToken();
Token<RMDelegationTokenIdentifier> token1 =
ConverterUtils.convertFromYarn(delegationToken, null);
RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
// wait for the first rollMasterKey
while (((TestRMDelegationTokenSecretManager) dtSecretManager).numUpdatedKeys.get() < 1) {
Thread.sleep(200);
}
// assert old-current-key and new-current-key exist
Assert.assertTrue(rmDTMasterKeyState.contains(oldCurrentKey));
DelegationKey newCurrentKey =
((TestRMDelegationTokenSecretManager) dtSecretManager).getCurrentKey();
Assert.assertTrue(rmDTMasterKeyState.contains(newCurrentKey));
// wait for token to expire
// rollMasterKey is called every 1 second.
while (((TestRMDelegationTokenSecretManager) dtSecretManager).numUpdatedKeys.get() < 6) {
Thread.sleep(200);
}
Assert.assertFalse(rmDTState.containsKey(dtId1));
rm1.stop();
}
public static void main(String[] args) {
try {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
String containerIdStr = System.getenv(Environment.CONTAINER_ID.name());
String nodeHostString = System.getenv(Environment.NM_HOST.name());
String nodePortString = System.getenv(Environment.NM_PORT.name());
String nodeHttpPortString = System.getenv(Environment.NM_HTTP_PORT.name());
String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV);
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
long appSubmitTime = Long.parseLong(appSubmitTimeStr);
Configuration conf = new Configuration(new YarnConfiguration());
TezUtils.addUserSpecifiedTezConfiguration(conf);
String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name());
// Do not automatically close FileSystem objects so that in case of
// SIGTERM I have a chance to write out the job history. I'll be closing
// the objects myself.
conf.setBoolean("fs.automatic.close", false);
// Command line options
Options opts = new Options();
opts.addOption(
TezConstants.TEZ_SESSION_MODE_CLI_OPTION,
false,
"Run Tez Application Master in Session mode");
CommandLine cliParser = new GnuParser().parse(opts, args);
DAGAppMaster appMaster =
new DAGAppMaster(
applicationAttemptId,
containerId,
nodeHostString,
Integer.parseInt(nodePortString),
Integer.parseInt(nodeHttpPortString),
appSubmitTime,
cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION));
ShutdownHookManager.get()
.addShutdownHook(new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
initAndStartAppMaster(appMaster, conf, jobUserName);
} catch (Throwable t) {
LOG.fatal("Error starting DAGAppMaster", t);
System.exit(1);
}
}
public void fromProto(DAGSubmittedProto proto) {
this.dagID = TezDAGID.fromString(proto.getDagId());
this.dagPlan = proto.getDagPlan();
this.dagName = this.dagPlan.getName();
this.submitTime = proto.getSubmitTime();
this.applicationAttemptId =
ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
if (proto.hasCumulativeAdditionalAmResources()) {
this.cumulativeAdditionalLocalResources =
DagTypeConverters.convertFromPlanLocalResources(
proto.getCumulativeAdditionalAmResources());
}
}
static LocalResource newYarnAppResource(
FileSystem fs, Path path, LocalResourceType type, LocalResourceVisibility vis)
throws IOException {
Path qualified = fs.makeQualified(path);
FileStatus status = fs.getFileStatus(qualified);
LocalResource resource = Records.newRecord(LocalResource.class);
resource.setType(type);
resource.setVisibility(vis);
resource.setResource(ConverterUtils.getYarnUrlFromPath(qualified));
resource.setTimestamp(status.getModificationTime());
resource.setSize(status.getLen());
return resource;
}
private ApplicationAttemptStateData createAttemptState(String itemName, byte[] data)
throws IOException {
ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId(itemName);
ApplicationAttemptStateDataPBImpl attemptState =
new ApplicationAttemptStateDataPBImpl(ApplicationAttemptStateDataProto.parseFrom(data));
if (!attemptId.equals(attemptState.getAttemptId())) {
throw new YarnRuntimeException(
"The database entry for "
+ attemptId
+ " contains data for "
+ attemptState.getAttemptId());
}
return attemptState;
}
private ApplicationStateData createApplicationState(String appIdStr, byte[] data)
throws IOException {
ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
ApplicationStateDataPBImpl appState =
new ApplicationStateDataPBImpl(ApplicationStateDataProto.parseFrom(data));
if (!appId.equals(appState.getApplicationSubmissionContext().getApplicationId())) {
throw new YarnRuntimeException(
"The database entry for "
+ appId
+ " contains data for "
+ appState.getApplicationSubmissionContext().getApplicationId());
}
return appState;
}
/** Creates a LocalResource instance for the JAR file referenced by the given Path. */
public LocalResource getLocalResourceForPath(final Path jarPath, final LocalResourceType type)
throws IOException {
final FileStatus status =
FileContext.getFileContext(fileSystem.getUri()).getFileStatus(jarPath);
final LocalResource localResource = Records.newRecord(LocalResource.class);
localResource.setType(type);
localResource.setVisibility(LocalResourceVisibility.APPLICATION);
localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
localResource.setTimestamp(status.getModificationTime());
localResource.setSize(status.getLen());
return localResource;
}
@GET
@Path("/apps/{appid}")
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
public AppInfo getNodeApp(@PathParam("appid") String appId) {
init();
ApplicationId id = ConverterUtils.toApplicationId(recordFactory, appId);
if (id == null) {
throw new NotFoundException("app with id " + appId + " not found");
}
Application app = this.nmContext.getApplications().get(id);
if (app == null) {
throw new NotFoundException("app with id " + appId + " not found");
}
return new AppInfo(app);
}
public static LocalResource newLocalResource(
URI uri,
LocalResourceType type,
LocalResourceVisibility visibility,
long size,
long timestamp,
boolean shouldBeUploadedToSharedCache) {
return newLocalResource(
ConverterUtils.getYarnUrlFromURI(uri),
type,
visibility,
size,
timestamp,
shouldBeUploadedToSharedCache);
}
private NodeId verifyAndGetNodeId(Block html) {
String nodeIdStr = $(NM_NODENAME);
if (nodeIdStr == null || nodeIdStr.isEmpty()) {
html.h1()._("Cannot get container logs without a NodeId")._();
return null;
}
NodeId nodeId = null;
try {
nodeId = ConverterUtils.toNodeId(nodeIdStr);
} catch (IllegalArgumentException e) {
html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr)
._();
return null;
}
return nodeId;
}
/**
* @param file Path.
* @param fs File system.
* @param type Local resource type.
* @throws Exception If failed.
*/
public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type)
throws Exception {
LocalResource resource = Records.newRecord(LocalResource.class);
file = fs.makeQualified(file);
FileStatus stat = fs.getFileStatus(file);
resource.setResource(ConverterUtils.getYarnUrlFromPath(file));
resource.setSize(stat.getLen());
resource.setTimestamp(stat.getModificationTime());
resource.setType(type);
resource.setVisibility(LocalResourceVisibility.APPLICATION);
return resource;
}
@Override
public synchronized void storeApplicationAttemptState(
String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB) throws Exception {
ApplicationAttemptId appAttemptId = ConverterUtils.toApplicationAttemptId(attemptId);
Path appDirPath = getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
Path nodeCreatePath = getNodePath(appDirPath, attemptId);
LOG.info("Storing info for attempt: " + attemptId + " at: " + nodeCreatePath);
byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
try {
// currently throw all exceptions. May need to respond differently for HA
// based on whether we have lost the right to write to FS
writeFile(nodeCreatePath, attemptStateData);
} catch (Exception e) {
LOG.info("Error storing info for attempt: " + attemptId, e);
throw e;
}
}
private ContainerId verifyAndGetContainerId(Block html) {
String containerIdStr = $(CONTAINER_ID);
if (containerIdStr == null || containerIdStr.isEmpty()) {
html.h1()._("Cannot get container logs without a ContainerId")._();
return null;
}
ContainerId containerId = null;
try {
containerId = ConverterUtils.toContainerId(containerIdStr);
} catch (IllegalArgumentException e) {
html.h1()
._("Cannot get container logs for invalid containerId: "
+ containerIdStr)._();
return null;
}
return containerId;
}
/**
* Create an AM resource from the
*
* @param hdfs HDFS or other filesystem in use
* @param destPath dest path in filesystem
* @param resourceType resource type
* @return the resource set up wih application-level visibility and the timestamp & size set from
* the file stats.
*/
public static LocalResource createAmResource(
FileSystem hdfs, Path destPath, LocalResourceType resourceType) throws IOException {
FileStatus destStatus = hdfs.getFileStatus(destPath);
LocalResource amResource = Records.newRecord(LocalResource.class);
amResource.setType(resourceType);
// Set visibility of the resource
// Setting to most private option
amResource.setVisibility(LocalResourceVisibility.APPLICATION);
// Set the resource to be copied over
amResource.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
// Set timestamp and length of file so that the framework
// can do basic sanity checks for the local resource
// after it has been copied over to ensure it is the same
// resource the client intended to use with the application
amResource.setTimestamp(destStatus.getModificationTime());
amResource.setSize(destStatus.getLen());
return amResource;
}
@GET
@Path("/containers/{containerid}")
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
public ContainerInfo getNodeContainer(@PathParam("containerid") String id) {
ContainerId containerId = null;
init();
try {
containerId = ConverterUtils.toContainerId(id);
} catch (Exception e) {
throw new BadRequestException("invalid container id, " + id);
}
Container container = nmContext.getContainers().get(containerId);
if (container == null) {
throw new NotFoundException("container with id, " + id + ", not found");
}
return new ContainerInfo(
this.nmContext, container, uriInfo.getBaseUri().toString(), webapp.name());
}
/**
* Initial DragonAppMaster Get and CheckOut necessary parameters from system environment eg:
* container_Id,host,port,http_port,submitTime
*
* @param args
*/
public static void main(String[] args) {
try {
String containerIdStr = System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV);
String nodeHostString = System.getenv(ApplicationConstants.NM_HOST_ENV);
String nodePortString = System.getenv(ApplicationConstants.NM_PORT_ENV);
String nodeHttpPortString = System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV);
String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
validateInputParam(containerIdStr, ApplicationConstants.AM_CONTAINER_ID_ENV);
validateInputParam(nodeHostString, ApplicationConstants.NM_HOST_ENV);
validateInputParam(nodePortString, ApplicationConstants.NM_PORT_ENV);
validateInputParam(nodeHttpPortString, ApplicationConstants.NM_HTTP_PORT_ENV);
validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV);
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
long appSubmitTime = Long.parseLong(appSubmitTimeStr);
DragonAppMaster appMaster =
new DragonAppMaster(
applicationAttemptId,
containerId,
nodeHostString,
Integer.parseInt(nodePortString),
Integer.parseInt(nodeHttpPortString),
appSubmitTime);
Runtime.getRuntime().addShutdownHook(new CompositeServiceShutdownHook(appMaster));
YarnConfiguration conf = new YarnConfiguration(new DragonConfiguration());
conf.addResource(new Path(DragonJobConfig.JOB_CONF_FILE));
String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name());
conf.set(DragonJobConfig.USER_NAME, jobUserName);
// Do not automatically close FileSystem objects so that in case of
// SIGTERM I have a chance to write out the job history. I'll be closing
// the objects myself.
conf.setBoolean("fs.automatic.close", false);
initAndStartAppMaster(appMaster, conf, jobUserName);
} catch (Throwable t) {
LOG.fatal("Error starting MRAppMaster", t);
System.exit(1);
}
}
public AppLogAggregatorImpl(
Dispatcher dispatcher,
DeletionService deletionService,
Configuration conf,
ApplicationId appId,
UserGroupInformation userUgi,
LocalDirsHandlerService dirsHandler,
Path remoteNodeLogFileForApp,
ContainerLogsRetentionPolicy retentionPolicy,
Map<ApplicationAccessType, String> appAcls) {
this.dispatcher = dispatcher;
this.conf = conf;
this.delService = deletionService;
this.appId = appId;
this.applicationId = ConverterUtils.toString(appId);
this.userUgi = userUgi;
this.dirsHandler = dirsHandler;
this.remoteNodeLogFileForApp = remoteNodeLogFileForApp;
this.remoteNodeTmpLogFileForApp = getRemoteNodeTmpLogFileForApp();
this.retentionPolicy = retentionPolicy;
this.pendingContainers = new LinkedBlockingQueue<ContainerId>();
this.appAcls = appAcls;
}
/*
* Helper method to create a yarn local resource.
*/
private LocalResource createLocalResource(
FileSystem remoteFs, Path file, LocalResourceType type, LocalResourceVisibility visibility) {
FileStatus fstat = null;
try {
fstat = remoteFs.getFileStatus(file);
} catch (IOException e) {
e.printStackTrace();
}
URL resourceURL = ConverterUtils.getYarnUrlFromPath(file);
long resourceSize = fstat.getLen();
long resourceModificationTime = fstat.getModificationTime();
LOG.info("Resource modification time: " + resourceModificationTime);
LocalResource lr = Records.newRecord(LocalResource.class);
lr.setResource(resourceURL);
lr.setType(type);
lr.setSize(resourceSize);
lr.setVisibility(visibility);
lr.setTimestamp(resourceModificationTime);
return lr;
}
/**
* Launch application for the dag represented by this client.
*
* @throws YarnException
* @throws IOException
*/
public void startApplication() throws YarnException, IOException {
Class<?>[] defaultClasses;
if (applicationType.equals(YARN_APPLICATION_TYPE)) {
// TODO restrict the security check to only check if security is enabled for webservices.
if (UserGroupInformation.isSecurityEnabled()) {
defaultClasses = DATATORRENT_SECURITY_CLASSES;
} else {
defaultClasses = DATATORRENT_CLASSES;
}
} else {
throw new IllegalStateException(applicationType + " is not a valid application type.");
}
LinkedHashSet<String> localJarFiles = findJars(dag, defaultClasses);
if (resources != null) {
localJarFiles.addAll(resources);
}
YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
LOG.info(
"Got Cluster metric info from ASM"
+ ", numNodeManagers="
+ clusterMetrics.getNumNodeManagers());
// GetClusterNodesRequest clusterNodesReq = Records.newRecord(GetClusterNodesRequest.class);
// GetClusterNodesResponse clusterNodesResp =
// rmClient.clientRM.getClusterNodes(clusterNodesReq);
// LOG.info("Got Cluster node info from ASM");
// for (NodeReport node : clusterNodesResp.getNodeReports()) {
// LOG.info("Got node report from ASM for"
// + ", nodeId=" + node.getNodeId()
// + ", nodeAddress" + node.getHttpAddress()
// + ", nodeRackName" + node.getRackName()
// + ", nodeNumContainers" + node.getNumContainers()
// + ", nodeHealthStatus" + node.getHealthReport());
// }
List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
for (QueueUserACLInfo aclInfo : listAclInfo) {
for (QueueACL userAcl : aclInfo.getUserAcls()) {
LOG.info(
"User ACL Info for Queue"
+ ", queueName="
+ aclInfo.getQueueName()
+ ", userAcl="
+ userAcl.name());
}
}
// Get a new application id
YarnClientApplication newApp = yarnClient.createApplication();
appId = newApp.getNewApplicationResponse().getApplicationId();
// Dump out information about cluster capability as seen by the resource manager
int maxMem = newApp.getNewApplicationResponse().getMaximumResourceCapability().getMemory();
LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
int amMemory = dag.getMasterMemoryMB();
if (amMemory > maxMem) {
LOG.info(
"AM memory specified above max threshold of cluster. Using max value."
+ ", specified="
+ amMemory
+ ", max="
+ maxMem);
amMemory = maxMem;
}
if (dag.getAttributes().get(LogicalPlan.APPLICATION_ID) == null) {
dag.setAttribute(LogicalPlan.APPLICATION_ID, appId.toString());
}
// Create launch context for app master
LOG.info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
// set the application id
appContext.setApplicationId(appId);
// set the application name
appContext.setApplicationName(dag.getValue(LogicalPlan.APPLICATION_NAME));
appContext.setApplicationType(this.applicationType);
if (YARN_APPLICATION_TYPE.equals(this.applicationType)) {
// appContext.setMaxAppAttempts(1); // no retries until Stram is HA
}
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
// Setup security tokens
// If security is enabled get ResourceManager and NameNode delegation tokens.
// Set these tokens on the container so that they are sent as part of application submission.
// This also sets them up for renewal by ResourceManager. The NameNode delegation rmToken
// is also used by ResourceManager to fetch the jars from HDFS and set them up for the
// application master launch.
if (UserGroupInformation.isSecurityEnabled()) {
Credentials credentials = new Credentials();
String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
if (tokenRenewer == null || tokenRenewer.length() == 0) {
throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
}
// For now, only getting tokens for the default file-system.
FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
try {
final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
if (tokens != null) {
for (Token<?> token : tokens) {
LOG.info("Got dt for " + fs.getUri() + "; " + token);
}
}
} finally {
fs.close();
}
addRMDelegationToken(tokenRenewer, credentials);
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
amContainer.setTokens(fsTokens);
}
// set local resources for the application master
// local files or archives as needed
// In this scenario, the jar file for the application master is part of the local resources
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
// copy required jar files to dfs, to be localized for containers
FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
try {
Path appsBasePath =
new Path(StramClientUtils.getDTDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
Path appPath = new Path(appsBasePath, appId.toString());
String libJarsCsv = copyFromLocal(fs, appPath, localJarFiles.toArray(new String[] {}));
LOG.info("libjars: {}", libJarsCsv);
dag.getAttributes().put(LogicalPlan.LIBRARY_JARS, libJarsCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.FILE, libJarsCsv, localResources, fs);
if (archives != null) {
String[] localFiles = archives.split(",");
String archivesCsv = copyFromLocal(fs, appPath, localFiles);
LOG.info("archives: {}", archivesCsv);
dag.getAttributes().put(LogicalPlan.ARCHIVES, archivesCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.ARCHIVE, archivesCsv, localResources, fs);
}
if (files != null) {
String[] localFiles = files.split(",");
String filesCsv = copyFromLocal(fs, appPath, localFiles);
LOG.info("files: {}", filesCsv);
dag.getAttributes().put(LogicalPlan.FILES, filesCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.FILE, filesCsv, localResources, fs);
}
dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, appPath.toString());
if (dag.getAttributes().get(OperatorContext.STORAGE_AGENT) == null) {
/* which would be the most likely case */
Path checkpointPath = new Path(appPath, LogicalPlan.SUBDIR_CHECKPOINTS);
// use conf client side to pickup any proxy settings from dt-site.xml
dag.setAttribute(
OperatorContext.STORAGE_AGENT, new FSStorageAgent(checkpointPath.toString(), conf));
}
if (dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR) == null) {
dag.setAttribute(
LogicalPlan.CONTAINER_OPTS_CONFIGURATOR, new BasicContainerOptConfigurator());
}
// Set the log4j properties if needed
if (!log4jPropFile.isEmpty()) {
Path log4jSrc = new Path(log4jPropFile);
Path log4jDst = new Path(appPath, "log4j.props");
fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
log4jRsrc.setType(LocalResourceType.FILE);
log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
log4jRsrc.setSize(log4jFileStatus.getLen());
localResources.put("log4j.properties", log4jRsrc);
}
if (originalAppId != null) {
Path origAppPath = new Path(appsBasePath, this.originalAppId);
LOG.info("Restart from {}", origAppPath);
copyInitialState(origAppPath);
}
// push logical plan to DFS location
Path cfgDst = new Path(appPath, LogicalPlan.SER_FILE_NAME);
FSDataOutputStream outStream = fs.create(cfgDst, true);
LogicalPlan.write(this.dag, outStream);
outStream.close();
Path launchConfigDst = new Path(appPath, LogicalPlan.LAUNCH_CONFIG_FILE_NAME);
outStream = fs.create(launchConfigDst, true);
conf.writeXml(outStream);
outStream.close();
FileStatus topologyFileStatus = fs.getFileStatus(cfgDst);
LocalResource topologyRsrc = Records.newRecord(LocalResource.class);
topologyRsrc.setType(LocalResourceType.FILE);
topologyRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
topologyRsrc.setResource(ConverterUtils.getYarnUrlFromURI(cfgDst.toUri()));
topologyRsrc.setTimestamp(topologyFileStatus.getModificationTime());
topologyRsrc.setSize(topologyFileStatus.getLen());
localResources.put(LogicalPlan.SER_FILE_NAME, topologyRsrc);
// Set local resource info into app master container launch context
amContainer.setLocalResources(localResources);
// Set the necessary security tokens as needed
// amContainer.setContainerTokens(containerToken);
// Set the env variables to be setup in the env where the application master will be run
LOG.info("Set the environment for the application master");
Map<String, String> env = new HashMap<String, String>();
// Add application jar(s) location to classpath
// At some point we should not be required to add
// the hadoop specific classpaths to the env.
// It should be provided out of the box.
// For now setting all required classpaths including
// the classpath to "." for the application jar(s)
// including ${CLASSPATH} will duplicate the class path in app master, removing it for now
// StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*");
StringBuilder classPathEnv = new StringBuilder("./*");
String classpath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
for (String c :
StringUtils.isBlank(classpath)
? YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
: classpath.split(",")) {
if (c.equals("$HADOOP_CLIENT_CONF_DIR")) {
// SPOI-2501
continue;
}
classPathEnv.append(':');
classPathEnv.append(c.trim());
}
env.put("CLASSPATH", classPathEnv.toString());
// propagate to replace node managers user name (effective in non-secure mode)
env.put("HADOOP_USER_NAME", UserGroupInformation.getLoginUser().getUserName());
amContainer.setEnvironment(env);
// Set the necessary command to execute the application master
ArrayList<CharSequence> vargs = new ArrayList<CharSequence>(30);
// Set java executable command
LOG.info("Setting up app master command");
vargs.add(javaCmd);
if (dag.isDebug()) {
vargs.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=n");
}
// Set Xmx based on am memory size
// default heap size 75% of total memory
vargs.add("-Xmx" + (amMemory * 3 / 4) + "m");
vargs.add("-XX:+HeapDumpOnOutOfMemoryError");
vargs.add("-XX:HeapDumpPath=/tmp/dt-heap-" + appId.getId() + ".bin");
vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
if (dag.isDebug()) {
vargs.add("-Dlog4j.debug=true");
}
String loggersLevel = conf.get(DTLoggerFactory.DT_LOGGERS_LEVEL);
if (loggersLevel != null) {
vargs.add(String.format("-D%s=%s", DTLoggerFactory.DT_LOGGERS_LEVEL, loggersLevel));
}
vargs.add(StreamingAppMaster.class.getName());
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");
// Get final command
StringBuilder command = new StringBuilder(9 * vargs.size());
for (CharSequence str : vargs) {
command.append(str).append(" ");
}
LOG.info("Completed setting up app master command " + command.toString());
List<String> commands = new ArrayList<String>();
commands.add(command.toString());
amContainer.setCommands(commands);
// Set up resource type requirements
// For now, only memory is supported so we set memory requirements
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(amMemory);
appContext.setResource(capability);
// Service data is a binary blob that can be passed to the application
// Not needed in this scenario
// amContainer.setServiceData(serviceData);
appContext.setAMContainerSpec(amContainer);
// Set the priority for the application master
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(amPriority);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue(queueName);
// Submit the application to the applications manager
// SubmitApplicationResponse submitResp = rmClient.submitApplication(appRequest);
// Ignore the response as either a valid response object is returned on success
// or an exception thrown to denote some form of a failure
String specStr =
Objects.toStringHelper("Submitting application: ")
.add("name", appContext.getApplicationName())
.add("queue", appContext.getQueue())
.add("user", UserGroupInformation.getLoginUser())
.add("resource", appContext.getResource())
.toString();
LOG.info(specStr);
if (dag.isDebug()) {
// LOG.info("Full submission context: " + appContext);
}
yarnClient.submitApplication(appContext);
} finally {
fs.close();
}
}