private static CommitterEventHandler createCommitterEventHandler(
Dispatcher dispatcher, OutputCommitter committer) {
final SystemClock clock = new SystemClock();
AppContext appContext = mock(AppContext.class);
when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
when(appContext.getClock()).thenReturn(clock);
RMHeartbeatHandler heartbeatHandler =
new RMHeartbeatHandler() {
@Override
public long getLastHeartbeatTime() {
return clock.getTime();
}
@Override
public void runOnNextHeartbeat(Runnable callback) {
callback.run();
}
};
ApplicationAttemptId id =
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
when(appContext.getApplicationAttemptId()).thenReturn(id);
CommitterEventHandler handler =
new CommitterEventHandler(appContext, committer, heartbeatHandler);
dispatcher.register(CommitterEventType.class, handler);
return handler;
}
public void fromProto(DAGSubmittedProto proto) {
this.dagID = TezDAGID.fromString(proto.getDagId());
this.dagPlan = proto.getDagPlan();
this.dagName = this.dagPlan.getName();
this.submitTime = proto.getSubmitTime();
this.applicationAttemptId =
ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
if (proto.hasCumulativeAdditionalAmResources()) {
this.cumulativeAdditionalLocalResources =
DagTypeConverters.convertFromPlanLocalResources(
proto.getCumulativeAdditionalAmResources());
}
}
private ApplicationAttemptStateData createAttemptState(String itemName, byte[] data)
throws IOException {
ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId(itemName);
ApplicationAttemptStateDataPBImpl attemptState =
new ApplicationAttemptStateDataPBImpl(ApplicationAttemptStateDataProto.parseFrom(data));
if (!attemptId.equals(attemptState.getAttemptId())) {
throw new YarnRuntimeException(
"The database entry for "
+ attemptId
+ " contains data for "
+ attemptState.getAttemptId());
}
return attemptState;
}
@Override
public synchronized void storeApplicationAttemptState(
String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB) throws Exception {
ApplicationAttemptId appAttemptId = ConverterUtils.toApplicationAttemptId(attemptId);
Path appDirPath = getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
Path nodeCreatePath = getNodePath(appDirPath, attemptId);
LOG.info("Storing info for attempt: " + attemptId + " at: " + nodeCreatePath);
byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
try {
// currently throw all exceptions. May need to respond differently for HA
// based on whether we have lost the right to write to FS
writeFile(nodeCreatePath, attemptStateData);
} catch (Exception e) {
LOG.info("Error storing info for attempt: " + attemptId, e);
throw e;
}
}
public void fromProto(AMStartedProto proto) {
this.applicationAttemptId =
ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
this.startTime = proto.getStartTime();
}
private void loadRMAppState(RMState rmState) throws Exception {
try {
List<ApplicationAttemptState> attempts = new ArrayList<ApplicationAttemptState>();
for (FileStatus appDir : fs.listStatus(rmAppRoot)) {
for (FileStatus childNodeStatus : fs.listStatus(appDir.getPath())) {
assert childNodeStatus.isFile();
String childNodeName = childNodeStatus.getPath().getName();
byte[] childData = readFile(childNodeStatus.getPath(), childNodeStatus.getLen());
if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) {
// application
LOG.info("Loading application from node: " + childNodeName);
ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
ApplicationStateDataPBImpl appStateData =
new ApplicationStateDataPBImpl(ApplicationStateDataProto.parseFrom(childData));
ApplicationState appState =
new ApplicationState(
appStateData.getSubmitTime(),
appStateData.getApplicationSubmissionContext(),
appStateData.getUser());
// assert child node name is same as actual applicationId
assert appId.equals(appState.context.getApplicationId());
rmState.appState.put(appId, appState);
} else if (childNodeName.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
// attempt
LOG.info("Loading application attempt from node: " + childNodeName);
ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId(childNodeName);
ApplicationAttemptStateDataPBImpl attemptStateData =
new ApplicationAttemptStateDataPBImpl(
ApplicationAttemptStateDataProto.parseFrom(childData));
Credentials credentials = null;
if (attemptStateData.getAppAttemptTokens() != null) {
credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
dibb.reset(attemptStateData.getAppAttemptTokens());
credentials.readTokenStorageStream(dibb);
}
ApplicationAttemptState attemptState =
new ApplicationAttemptState(
attemptId, attemptStateData.getMasterContainer(), credentials);
// assert child node name is same as application attempt id
assert attemptId.equals(attemptState.getAttemptId());
attempts.add(attemptState);
} else {
LOG.info("Unknown child node with name: " + childNodeName);
}
}
}
// go through all attempts and add them to their apps, Ideally, each
// attempt node must have a corresponding app node, because remove
// directory operation remove both at the same time
for (ApplicationAttemptState attemptState : attempts) {
ApplicationId appId = attemptState.getAttemptId().getApplicationId();
ApplicationState appState = rmState.appState.get(appId);
assert appState != null;
appState.attempts.put(attemptState.getAttemptId(), attemptState);
}
} catch (Exception e) {
LOG.error("Failed to load state.", e);
throw e;
}
}
public static void main(String[] args) throws Exception {
System.err.println("Inside the AM!!!");
LOG.info("Starting the AM!!!!");
Options opts = new Options();
opts.addOption(
"app_attempt_id", true, "App Attempt ID. Not to be used " + "unless for testing purposes");
CommandLine cl = new GnuParser().parse(opts, args);
Map<String, String> envs = System.getenv();
_appAttemptID = Records.newRecord(ApplicationAttemptId.class);
if (cl.hasOption("app_attempt_id")) {
String appIdStr = cl.getOptionValue("app_attempt_id", "");
_appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
} else if (envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) {
ContainerId containerId =
ConverterUtils.toContainerId(envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV));
_appAttemptID = containerId.getApplicationAttemptId();
} else {
throw new IllegalArgumentException(
"Container and application attempt IDs not set in the environment");
}
@SuppressWarnings("rawtypes")
Map storm_conf = Config.readStormConfig(null);
YarnConfiguration hadoopConf = new YarnConfiguration();
StormAMRMClient client = new StormAMRMClient(_appAttemptID, storm_conf, hadoopConf);
client.init(hadoopConf);
client.start();
BlockingQueue<Container> launcherQueue = new LinkedBlockingQueue<Container>();
try {
InetSocketAddress addr =
NetUtils.createSocketAddr(
hadoopConf.get(
YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
int port = Utils.getInt(storm_conf.get(Config.MASTER_THRIFT_PORT));
RegisterApplicationMasterResponse resp =
client.registerApplicationMaster(addr.getHostName(), port, null);
LOG.info("Got a registration response " + resp);
LOG.info("Max Capability " + resp.getMaximumResourceCapability());
client.setMaxResource(resp.getMaximumResourceCapability());
MasterServer server = new MasterServer(storm_conf, client);
LOG.info("Starting HB thread");
initAndStartHeartbeat(
client, launcherQueue, (Integer) storm_conf.get(Config.MASTER_HEARTBEAT_INTERVAL_MILLIS));
LOG.info("Starting launcher");
initAndStartLauncher(client, launcherQueue);
client.startAllSupervisors();
server.serve();
client.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "AllDone", null);
} finally {
client.stop();
}
System.err.println("See Ya!!!");
System.exit(0);
}
/**
* Parse command line options
*
* @param args Command line args
* @return Whether init successful and run should be invoked
* @throws ParseException
* @throws IOException
*/
public boolean init(String[] args) throws ParseException, IOException {
Options opts = new Options();
opts.addOption(
"app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes");
opts.addOption(
"shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs");
opts.addOption(
"container_memory",
true,
"Amount of memory in MB to be requested to run the shell command");
opts.addOption(
"container_vcores",
true,
"Amount of virtual cores to be requested to run the shell command");
opts.addOption(
"num_containers",
true,
"No. of containers on which the shell command needs to be executed");
opts.addOption("priority", true, "Application Priority. Default 0");
opts.addOption("debug", false, "Dump out debug information");
opts.addOption("help", false, "Print usage");
CommandLine cliParser = new GnuParser().parse(opts, args);
if (args.length == 0) {
printUsage(opts);
throw new IllegalArgumentException("No args specified for application master to initialize");
}
// Check whether customer log4j.properties file exists
if (fileExist(log4jPath)) {
try {
Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath);
} catch (Exception e) {
LOG.warn("Can not set up custom log4j properties. " + e);
}
}
if (cliParser.hasOption("help")) {
printUsage(opts);
return false;
}
if (cliParser.hasOption("debug")) {
dumpOutDebugInfo();
}
Map<String, String> envs = System.getenv();
if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
if (cliParser.hasOption("app_attempt_id")) {
String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
} else {
throw new IllegalArgumentException("Application Attempt Id not set in the environment");
}
} else {
ContainerId containerId =
ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
appAttemptID = containerId.getApplicationAttemptId();
}
if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
throw new RuntimeException(
ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment");
}
if (!envs.containsKey(Environment.NM_HOST.name())) {
throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment");
}
if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment");
}
if (!envs.containsKey(Environment.NM_PORT.name())) {
throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment");
}
LOG.info(
"Application master for app"
+ ", appId="
+ appAttemptID.getApplicationId().getId()
+ ", clustertimestamp="
+ appAttemptID.getApplicationId().getClusterTimestamp()
+ ", attemptId="
+ appAttemptID.getAttemptId());
if (!fileExist(shellCommandPath)
&& envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION).isEmpty()) {
throw new IllegalArgumentException(
"No shell command or shell script specified to be executed by application master");
}
if (fileExist(shellCommandPath)) {
shellCommand = readContent(shellCommandPath);
}
if (fileExist(shellArgsPath)) {
shellArgs = readContent(shellArgsPath);
}
if (cliParser.hasOption("shell_env")) {
String shellEnvs[] = cliParser.getOptionValues("shell_env");
for (String env : shellEnvs) {
env = env.trim();
int index = env.indexOf('=');
if (index == -1) {
shellEnv.put(env, "");
continue;
}
String key = env.substring(0, index);
String val = "";
if (index < (env.length() - 1)) {
val = env.substring(index + 1);
}
shellEnv.put(key, val);
}
}
if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) {
scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION);
if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) {
shellScriptPathTimestamp =
Long.valueOf(envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP));
}
if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)) {
shellScriptPathLen = Long.valueOf(envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
}
if (!scriptPath.isEmpty() && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) {
LOG.error(
"Illegal values in env for shell script path"
+ ", path="
+ scriptPath
+ ", len="
+ shellScriptPathLen
+ ", timestamp="
+ shellScriptPathTimestamp);
throw new IllegalArgumentException("Illegal values in env for shell script path");
}
}
containerMemory = Integer.parseInt(cliParser.getOptionValue("container_memory", "10"));
containerVirtualCores = Integer.parseInt(cliParser.getOptionValue("container_vcores", "1"));
numTotalContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "1"));
if (numTotalContainers == 0) {
throw new IllegalArgumentException("Cannot run distributed shell with no containers");
}
requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));
// Creating the Timeline Client
timelineClient = TimelineClient.createTimelineClient();
timelineClient.init(conf);
timelineClient.start();
return true;
}