@Override
protected void serviceInit(Configuration conf) throws Exception {
this.exitOnDispatchException =
conf.getBoolean(
Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, Dispatcher.DEFAULT_DISPATCHER_EXIT_ON_ERROR);
super.serviceInit(conf);
}
@Override
protected void serviceStop() throws Exception {
if (drainEventsOnStop) {
blockNewEvents = true;
LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
synchronized (waitForDrained) {
while (!drained && eventHandlingThread.isAlive()) {
waitForDrained.wait(1000);
LOG.info(
"Waiting for AsyncDispatcher to drain. Thread state is :"
+ eventHandlingThread.getState());
}
}
}
stopped = true;
if (eventHandlingThread != null) {
eventHandlingThread.interrupt();
try {
eventHandlingThread.join();
} catch (InterruptedException ie) {
LOG.warn("Interrupted Exception while stopping", ie);
}
}
// stop all the components
super.serviceStop();
}
@Override
public void serviceInit(Configuration conf) throws Exception {
Constructor<?> cons;
try {
if (conf instanceof TajoConf) {
this.conf = (TajoConf) conf;
} else {
throw new TajoInternalError("conf must be a TajoConf instance");
}
Class<?> storeClass = this.conf.getClass(CatalogConstants.STORE_CLASS, DerbyStore.class);
LOG.info("Catalog Store Class: " + storeClass.getCanonicalName());
cons = storeClass.getConstructor(new Class[] {Configuration.class});
this.store = (CatalogStore) cons.newInstance(this.conf);
initBuiltinFunctions(builtingFuncs);
} catch (Throwable t) {
LOG.error("CatalogServer initialization failed", t);
throw new TajoInternalError(t);
}
super.serviceInit(conf);
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
rmPollInterval =
conf.getInt(
MRJobConfig.MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS,
MRJobConfig.DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS);
}
@Override
protected void serviceStop() throws Exception {
if (server != null) {
server.stop();
}
super.serviceStop();
amRunning = false;
}
@Override
protected void serviceStart() throws Exception {
// start all the components
super.serviceStart();
eventHandlingThread = new Thread(createThread());
eventHandlingThread.setName("AsyncDispatcher event handler");
eventHandlingThread.start();
}
@Override
protected void serviceStart() throws Exception {
scheduler = createSchedulerProxy();
register();
startAllocatorThread();
JobID id = TypeConverter.fromYarn(this.applicationId);
JobId jobId = TypeConverter.toYarn(id);
job = context.getJob(jobId);
super.serviceStart();
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
clientServiceBindAddress = RMADDRESS;
/*
clientServiceBindAddress = conf.get(
YarnConfiguration.APPSMANAGER_ADDRESS,
YarnConfiguration.DEFAULT_APPSMANAGER_BIND_ADDRESS);
*/
clientBindAddress = NetUtils.createSocketAddr(clientServiceBindAddress);
super.serviceInit(conf);
}
@Override
public void serviceStop() throws Exception {
if (isSession) {
sessionStopped.set(true);
}
if (this.dagSubmissionTimer != null) {
this.dagSubmissionTimer.cancel();
}
stopServices();
super.serviceStop();
}
@Override
protected void serviceStart() throws Exception {
// All the clients to appsManager are supposed to be authenticated via
// Kerberos if security is enabled, so no secretManager.
YarnRPC rpc = YarnRPC.create(getConfig());
Configuration clientServerConf = new Configuration(getConfig());
this.server =
rpc.getServer(
ApplicationClientProtocol.class, this, clientBindAddress, clientServerConf, null, 1);
this.server.start();
super.serviceStart();
}
@Override
protected void serviceStop() throws Exception {
if (proxyServer != null) {
try {
proxyServer.stop();
} catch (Exception e) {
LOG.fatal("Error stopping proxy web server", e);
throw new YarnRuntimeException("Error stopping proxy web server", e);
}
}
super.serviceStop();
}
@Override
public void serviceStop() throws Exception {
LOG.info("Catalog Server (" + bindAddressStr + ") shutdown");
// If CatalogServer shutdowns before it started, rpcServer and store may be NULL.
// So, we should check Nullity of them.
if (rpcServer != null) {
this.rpcServer.shutdown();
}
if (store != null) {
store.close();
}
super.serviceStop();
}
@Override
protected void serviceStop() throws Exception {
if (sched != null) {
sched.shutdown();
boolean terminated = false;
try {
terminated = sched.awaitTermination(10, SECONDS);
} catch (InterruptedException e) {
}
if (terminated != true) {
sched.shutdownNow();
}
}
super.serviceStop();
}
@Override
protected void serviceStart() throws Exception {
try {
proxyServer = new HttpServer("proxy", bindAddress, port, port == 0, getConfig(), acl);
proxyServer.addServlet(
ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC,
WebAppProxyServlet.class);
proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start();
} catch (IOException e) {
LOG.fatal("Could not start proxy web server", e);
throw new YarnRuntimeException("Could not start proxy web server", e);
}
super.serviceStart();
}
@Override
protected void serviceStart() throws Exception {
dtCancelThread.start();
if (tokenKeepAliveEnabled) {
delayedRemovalThread =
new Thread(new DelayedTokenRemovalRunnable(getConfig()), "DelayedTokenCanceller");
delayedRemovalThread.start();
}
setLocalSecretManagerAndServiceAddr();
serviceStateLock.writeLock().lock();
isServiceStarted = true;
serviceStateLock.writeLock().unlock();
while (!pendingEventQueue.isEmpty()) {
processDelegationTokenRenewerEvent(pendingEventQueue.take());
}
super.serviceStart();
}
public void start(Configuration conf) {
YarnRPC rpc = YarnRPC.create(conf);
// TODO : use fixed port ??
InetSocketAddress address = NetUtils.createSocketAddr(hostAddress);
InetAddress hostNameResolved = null;
try {
address.getAddress();
hostNameResolved = InetAddress.getLocalHost();
} catch (UnknownHostException e) {
throw new YarnRuntimeException(e);
}
server = rpc.getServer(protocol, this, address, conf, null, 1);
server.start();
this.bindAddress = NetUtils.getConnectAddress(server);
super.start();
amRunning = true;
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
this.conf = conf;
int serialNumberLowDigits = 3;
serialNumberFormat =
("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");
long maxFSWaitTime =
conf.getLong(
JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);
this.aclsMgr = new JobACLsManager(conf);
maxHistoryAge =
conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
jobListCache = createJobListCache();
serialNumberIndex =
new SerialNumberIndex(
conf.getInt(
JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));
int numMoveThreads =
conf.getInt(
JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
ThreadFactory tf =
new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
moveToDoneExecutor =
new ThreadPoolExecutor(
numMoveThreads,
numMoveThreads,
1,
TimeUnit.HOURS,
new LinkedBlockingQueue<Runnable>(),
tf);
super.serviceInit(conf);
}
@Override
protected void serviceStop() throws Exception {
if (stopped.getAndSet(true)) {
// return if already stopped
return;
}
if (allocatorThread != null) {
allocatorThread.interrupt();
try {
allocatorThread.join();
} catch (InterruptedException ie) {
LOG.warn("InterruptedException while stopping", ie);
}
}
if (shouldUnregister) {
unregister();
}
super.serviceStop();
}
@Override
public void serviceStart() throws Exception {
String serverAddr = conf.getVar(ConfVars.CATALOG_ADDRESS);
InetSocketAddress initIsa = NetUtils.createSocketAddr(serverAddr);
int workerNum = conf.getIntVar(ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM);
try {
this.rpcServer = new BlockingRpcServer(CatalogProtocol.class, handler, initIsa, workerNum);
this.rpcServer.start();
this.bindAddress = NetUtils.getConnectAddress(this.rpcServer.getListenAddress());
this.bindAddressStr = NetUtils.normalizeInetSocketAddress(bindAddress);
conf.setVar(ConfVars.CATALOG_ADDRESS, bindAddressStr);
} catch (Exception e) {
LOG.error("CatalogServer startup failed", e);
throw new TajoInternalError(e);
}
LOG.info("Catalog Server startup (" + bindAddressStr + ")");
super.serviceStart();
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat("DeletionService #%d").build();
if (conf != null) {
sched =
new ScheduledThreadPoolExecutor(
conf.getInt(
YarnConfiguration.NM_DELETE_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT),
tf);
debugDelay = conf.getInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0);
} else {
sched = new ScheduledThreadPoolExecutor(YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT, tf);
}
sched.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
sched.setKeepAliveTime(60L, SECONDS);
if (stateStore.canRecover()) {
recover(stateStore.loadDeletionServiceState());
}
super.serviceInit(conf);
}
@SuppressWarnings("unchecked")
@Override
public void serviceStart() throws Exception {
// start all the components
startServices();
super.serviceStart();
this.state = DAGAppMasterState.IDLE;
// metrics system init is really init & start.
// It's more test friendly to put it here.
DefaultMetricsSystem.initialize("DAGAppMaster");
this.appsStartTime = clock.getTime();
AMStartedEvent startEvent =
new AMStartedEvent(appAttemptID, startTime, appsStartTime, appSubmitTime);
dispatcher.getEventHandler().handle(new DAGHistoryEvent(startEvent));
this.lastDAGCompletionTime = clock.getTime();
if (!isSession) {
startDAG();
} else {
LOG.info("In Session mode. Waiting for DAG over RPC");
this.dagSubmissionTimer = new Timer(true);
this.dagSubmissionTimer.scheduleAtFixedRate(
new TimerTask() {
@Override
public void run() {
checkAndHandleSessionTimeout();
}
},
sessionTimeoutInterval,
sessionTimeoutInterval / 10);
}
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
this.hasProxyUserPrivileges =
conf.getBoolean(
YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED,
YarnConfiguration.DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED);
this.tokenKeepAliveEnabled =
conf.getBoolean(
YarnConfiguration.LOG_AGGREGATION_ENABLED,
YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED);
this.tokenRemovalDelayMs =
conf.getInt(
YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS);
this.credentialsValidTimeRemaining =
conf.getLong(
RM_SYSTEM_CREDENTIALS_VALID_TIME_REMAINING,
DEFAULT_RM_SYSTEM_CREDENTIALS_VALID_TIME_REMAINING);
setLocalSecretManagerAndServiceAddr();
renewerService = createNewThreadPoolService(conf);
pendingEventQueue = new LinkedBlockingQueue<DelegationTokenRenewerEvent>();
renewalTimer = new Timer(true);
super.serviceInit(conf);
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
String auth = conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
if (auth == null || "simple".equals(auth)) {
isSecurityEnabled = false;
} else if ("kerberos".equals(auth)) {
isSecurityEnabled = true;
} else {
LOG.warn(
"Unrecongized attribute value for "
+ CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION
+ " of "
+ auth);
}
String proxy = YarnConfiguration.getProxyHostAndPort(conf);
String[] proxyParts = proxy.split(":");
proxyHost = proxyParts[0];
fetcher = new AppReportFetcher(conf);
bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
if (bindAddress == null || bindAddress.isEmpty()) {
throw new YarnRuntimeException(
YarnConfiguration.PROXY_ADDRESS + " is not set so the proxy will not run.");
}
LOG.info("Instantiating Proxy at " + bindAddress);
String[] parts = StringUtils.split(bindAddress, ':');
port = 0;
if (parts.length == 2) {
bindAddress = parts[0];
port = Integer.parseInt(parts[1]);
}
acl =
new AccessControlList(
conf.get(YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
super.serviceInit(conf);
}
@Override
public void serviceStop() throws Exception {
ShutdownThreadsHelper.shutdownExecutorService(moveToDoneExecutor);
super.serviceStop();
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
this.conf = conf;
int serialNumberLowDigits = 3;
serialNumberFormat =
("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");
String doneDirPrefix = null;
doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
try {
doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
mkdir(
doneDirFc,
doneDirPrefixPath,
new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
} catch (IOException e) {
throw new YarnRuntimeException(
"Error creating done directory: [" + doneDirPrefixPath + "]", e);
}
String intermediateDoneDirPrefix = null;
intermediateDoneDirPrefix = JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
try {
intermediateDoneDirPath =
FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
mkdir(
intermediateDoneDirFc,
intermediateDoneDirPath,
new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
} catch (IOException e) {
LOG.info("error creating done directory on dfs " + e);
throw new YarnRuntimeException(
"Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
}
this.aclsMgr = new JobACLsManager(conf);
maxHistoryAge =
conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
jobListCache =
new JobListCache(
conf.getInt(
JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,
JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE),
maxHistoryAge);
serialNumberIndex =
new SerialNumberIndex(
conf.getInt(
JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));
int numMoveThreads =
conf.getInt(
JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
ThreadFactory tf =
new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
moveToDoneExecutor =
new ThreadPoolExecutor(
numMoveThreads,
numMoveThreads,
1,
TimeUnit.HOURS,
new LinkedBlockingQueue<Runnable>(),
tf);
super.serviceInit(conf);
}
@Override
protected void serviceStop() throws Exception {
numStops++;
ContainerAllocatorStopped = numStops;
super.serviceStop();
}
@Override
public void serviceInit(final Configuration conf) throws Exception {
this.state = DAGAppMasterState.INITED;
this.amConf = conf;
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
downloadTokensAndSetupUGI(conf);
context = new RunningAppContext(conf);
clientHandler = new DAGClientHandler();
dispatcher = createDispatcher();
addIfService(dispatcher, false);
clientRpcServer = new DAGClientServer(clientHandler);
addIfService(clientRpcServer, true);
taskHeartbeatHandler = createTaskHeartbeatHandler(context, conf);
addIfService(taskHeartbeatHandler, true);
containerHeartbeatHandler = createContainerHeartbeatHandler(context, conf);
addIfService(containerHeartbeatHandler, true);
// service to handle requests to TaskUmbilicalProtocol
taskAttemptListener =
createTaskAttemptListener(context, taskHeartbeatHandler, containerHeartbeatHandler);
addIfService(taskAttemptListener, true);
containers = new AMContainerMap(containerHeartbeatHandler, taskAttemptListener, context);
addIfService(containers, true);
dispatcher.register(AMContainerEventType.class, containers);
nodes = new AMNodeMap(dispatcher.getEventHandler(), context);
addIfService(nodes, true);
dispatcher.register(AMNodeEventType.class, nodes);
// service to do the task cleanup
taskCleaner = createTaskCleaner(context);
addIfService(taskCleaner, true);
this.dagEventDispatcher = new DagEventDispatcher();
this.vertexEventDispatcher = new VertexEventDispatcher();
// register the event dispatchers
dispatcher.register(DAGAppMasterEventType.class, new DAGAppMasterEventHandler());
dispatcher.register(DAGEventType.class, dagEventDispatcher);
dispatcher.register(VertexEventType.class, vertexEventDispatcher);
dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher());
dispatcher.register(TaskCleaner.EventType.class, taskCleaner);
taskSchedulerEventHandler =
new TaskSchedulerEventHandler(context, clientRpcServer, dispatcher.getEventHandler());
addIfService(taskSchedulerEventHandler, true);
dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
addIfServiceDependency(taskSchedulerEventHandler, clientRpcServer);
containerLauncher = createContainerLauncher(context);
addIfService(containerLauncher, true);
dispatcher.register(NMCommunicatorEventType.class, containerLauncher);
historyEventHandler = new HistoryEventHandler(context);
addIfService(historyEventHandler, true);
dispatcher.register(HistoryEventType.class, historyEventHandler);
this.sessionTimeoutInterval =
1000
* amConf.getInt(
TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT);
initServices(conf);
super.serviceInit(conf);
}