@Override public void serviceInit(Configuration conf) throws Exception { Constructor<?> cons; try { if (conf instanceof TajoConf) { this.conf = (TajoConf) conf; } else { throw new TajoInternalError("conf must be a TajoConf instance"); } Class<?> storeClass = this.conf.getClass(CatalogConstants.STORE_CLASS, DerbyStore.class); LOG.info("Catalog Store Class: " + storeClass.getCanonicalName()); cons = storeClass.getConstructor(new Class[] {Configuration.class}); this.store = (CatalogStore) cons.newInstance(this.conf); initBuiltinFunctions(builtingFuncs); } catch (Throwable t) { LOG.error("CatalogServer initialization failed", t); throw new TajoInternalError(t); } super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { this.exitOnDispatchException = conf.getBoolean( Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, Dispatcher.DEFAULT_DISPATCHER_EXIT_ON_ERROR); super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); rmPollInterval = conf.getInt( MRJobConfig.MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS, MRJobConfig.DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS); }
@Override protected void serviceInit(Configuration conf) throws Exception { clientServiceBindAddress = RMADDRESS; /* clientServiceBindAddress = conf.get( YarnConfiguration.APPSMANAGER_ADDRESS, YarnConfiguration.DEFAULT_APPSMANAGER_BIND_ADDRESS); */ clientBindAddress = NetUtils.createSocketAddr(clientServiceBindAddress); super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { this.conf = conf; int serialNumberLowDigits = 3; serialNumberFormat = ("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d"); long maxFSWaitTime = conf.getLong( JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME); createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime); this.aclsMgr = new JobACLsManager(conf); maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE); jobListCache = createJobListCache(); serialNumberIndex = new SerialNumberIndex( conf.getInt( JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE, JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE)); int numMoveThreads = conf.getInt( JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT, JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT); ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build(); moveToDoneExecutor = new ThreadPoolExecutor( numMoveThreads, numMoveThreads, 1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf); super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat("DeletionService #%d").build(); if (conf != null) { sched = new ScheduledThreadPoolExecutor( conf.getInt( YarnConfiguration.NM_DELETE_THREAD_COUNT, YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT), tf); debugDelay = conf.getInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0); } else { sched = new ScheduledThreadPoolExecutor(YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT, tf); } sched.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); sched.setKeepAliveTime(60L, SECONDS); if (stateStore.canRecover()) { recover(stateStore.loadDeletionServiceState()); } super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { this.hasProxyUserPrivileges = conf.getBoolean( YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, YarnConfiguration.DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED); this.tokenKeepAliveEnabled = conf.getBoolean( YarnConfiguration.LOG_AGGREGATION_ENABLED, YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED); this.tokenRemovalDelayMs = conf.getInt( YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS); this.credentialsValidTimeRemaining = conf.getLong( RM_SYSTEM_CREDENTIALS_VALID_TIME_REMAINING, DEFAULT_RM_SYSTEM_CREDENTIALS_VALID_TIME_REMAINING); setLocalSecretManagerAndServiceAddr(); renewerService = createNewThreadPoolService(conf); pendingEventQueue = new LinkedBlockingQueue<DelegationTokenRenewerEvent>(); renewalTimer = new Timer(true); super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { String auth = conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION); if (auth == null || "simple".equals(auth)) { isSecurityEnabled = false; } else if ("kerberos".equals(auth)) { isSecurityEnabled = true; } else { LOG.warn( "Unrecongized attribute value for " + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION + " of " + auth); } String proxy = YarnConfiguration.getProxyHostAndPort(conf); String[] proxyParts = proxy.split(":"); proxyHost = proxyParts[0]; fetcher = new AppReportFetcher(conf); bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); if (bindAddress == null || bindAddress.isEmpty()) { throw new YarnRuntimeException( YarnConfiguration.PROXY_ADDRESS + " is not set so the proxy will not run."); } LOG.info("Instantiating Proxy at " + bindAddress); String[] parts = StringUtils.split(bindAddress, ':'); port = 0; if (parts.length == 2) { bindAddress = parts[0]; port = Integer.parseInt(parts[1]); } acl = new AccessControlList( conf.get(YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { this.conf = conf; int serialNumberLowDigits = 3; serialNumberFormat = ("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d"); String doneDirPrefix = null; doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf); try { doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix)); doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf); doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK); mkdir( doneDirFc, doneDirPrefixPath, new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION)); } catch (IOException e) { throw new YarnRuntimeException( "Error creating done directory: [" + doneDirPrefixPath + "]", e); } String intermediateDoneDirPrefix = null; intermediateDoneDirPrefix = JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf); try { intermediateDoneDirPath = FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix)); intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf); mkdir( intermediateDoneDirFc, intermediateDoneDirPath, new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort())); } catch (IOException e) { LOG.info("error creating done directory on dfs " + e); throw new YarnRuntimeException( "Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e); } this.aclsMgr = new JobACLsManager(conf); maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE); jobListCache = new JobListCache( conf.getInt( JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE, JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE), maxHistoryAge); serialNumberIndex = new SerialNumberIndex( conf.getInt( JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE, JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE)); int numMoveThreads = conf.getInt( JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT, JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT); ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build(); moveToDoneExecutor = new ThreadPoolExecutor( numMoveThreads, numMoveThreads, 1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf); super.serviceInit(conf); }
@Override public void serviceInit(final Configuration conf) throws Exception { this.state = DAGAppMasterState.INITED; this.amConf = conf; conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); downloadTokensAndSetupUGI(conf); context = new RunningAppContext(conf); clientHandler = new DAGClientHandler(); dispatcher = createDispatcher(); addIfService(dispatcher, false); clientRpcServer = new DAGClientServer(clientHandler); addIfService(clientRpcServer, true); taskHeartbeatHandler = createTaskHeartbeatHandler(context, conf); addIfService(taskHeartbeatHandler, true); containerHeartbeatHandler = createContainerHeartbeatHandler(context, conf); addIfService(containerHeartbeatHandler, true); // service to handle requests to TaskUmbilicalProtocol taskAttemptListener = createTaskAttemptListener(context, taskHeartbeatHandler, containerHeartbeatHandler); addIfService(taskAttemptListener, true); containers = new AMContainerMap(containerHeartbeatHandler, taskAttemptListener, context); addIfService(containers, true); dispatcher.register(AMContainerEventType.class, containers); nodes = new AMNodeMap(dispatcher.getEventHandler(), context); addIfService(nodes, true); dispatcher.register(AMNodeEventType.class, nodes); // service to do the task cleanup taskCleaner = createTaskCleaner(context); addIfService(taskCleaner, true); this.dagEventDispatcher = new DagEventDispatcher(); this.vertexEventDispatcher = new VertexEventDispatcher(); // register the event dispatchers dispatcher.register(DAGAppMasterEventType.class, new DAGAppMasterEventHandler()); dispatcher.register(DAGEventType.class, dagEventDispatcher); dispatcher.register(VertexEventType.class, vertexEventDispatcher); dispatcher.register(TaskEventType.class, new TaskEventDispatcher()); dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher()); dispatcher.register(TaskCleaner.EventType.class, taskCleaner); taskSchedulerEventHandler = new TaskSchedulerEventHandler(context, clientRpcServer, dispatcher.getEventHandler()); addIfService(taskSchedulerEventHandler, true); dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler); addIfServiceDependency(taskSchedulerEventHandler, clientRpcServer); containerLauncher = createContainerLauncher(context); addIfService(containerLauncher, true); dispatcher.register(NMCommunicatorEventType.class, containerLauncher); historyEventHandler = new HistoryEventHandler(context); addIfService(historyEventHandler, true); dispatcher.register(HistoryEventType.class, historyEventHandler); this.sessionTimeoutInterval = 1000 * amConf.getInt( TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT); initServices(conf); super.serviceInit(conf); }