Beispiel #1
0
  @Override
  public void serviceInit(Configuration conf) throws Exception {

    Constructor<?> cons;
    try {
      if (conf instanceof TajoConf) {
        this.conf = (TajoConf) conf;
      } else {
        throw new TajoInternalError("conf must be a TajoConf instance");
      }

      Class<?> storeClass = this.conf.getClass(CatalogConstants.STORE_CLASS, DerbyStore.class);

      LOG.info("Catalog Store Class: " + storeClass.getCanonicalName());
      cons = storeClass.getConstructor(new Class[] {Configuration.class});

      this.store = (CatalogStore) cons.newInstance(this.conf);

      initBuiltinFunctions(builtingFuncs);
    } catch (Throwable t) {
      LOG.error("CatalogServer initialization failed", t);
      throw new TajoInternalError(t);
    }

    super.serviceInit(conf);
  }
 @Override
 protected void serviceInit(Configuration conf) throws Exception {
   this.exitOnDispatchException =
       conf.getBoolean(
           Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, Dispatcher.DEFAULT_DISPATCHER_EXIT_ON_ERROR);
   super.serviceInit(conf);
 }
Beispiel #3
0
 @Override
 protected void serviceInit(Configuration conf) throws Exception {
   super.serviceInit(conf);
   rmPollInterval =
       conf.getInt(
           MRJobConfig.MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS,
           MRJobConfig.DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS);
 }
 @Override
 protected void serviceInit(Configuration conf) throws Exception {
   clientServiceBindAddress = RMADDRESS;
   /*
   clientServiceBindAddress = conf.get(
       YarnConfiguration.APPSMANAGER_ADDRESS,
       YarnConfiguration.DEFAULT_APPSMANAGER_BIND_ADDRESS);
       */
   clientBindAddress = NetUtils.createSocketAddr(clientServiceBindAddress);
   super.serviceInit(conf);
 }
  @Override
  protected void serviceInit(Configuration conf) throws Exception {
    this.conf = conf;

    int serialNumberLowDigits = 3;
    serialNumberFormat =
        ("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");

    long maxFSWaitTime =
        conf.getLong(
            JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
            JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
    createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);

    this.aclsMgr = new JobACLsManager(conf);

    maxHistoryAge =
        conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);

    jobListCache = createJobListCache();

    serialNumberIndex =
        new SerialNumberIndex(
            conf.getInt(
                JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
                JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

    int numMoveThreads =
        conf.getInt(
            JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
            JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
    ThreadFactory tf =
        new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
    moveToDoneExecutor =
        new ThreadPoolExecutor(
            numMoveThreads,
            numMoveThreads,
            1,
            TimeUnit.HOURS,
            new LinkedBlockingQueue<Runnable>(),
            tf);

    super.serviceInit(conf);
  }
 @Override
 protected void serviceInit(Configuration conf) throws Exception {
   ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat("DeletionService #%d").build();
   if (conf != null) {
     sched =
         new ScheduledThreadPoolExecutor(
             conf.getInt(
                 YarnConfiguration.NM_DELETE_THREAD_COUNT,
                 YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT),
             tf);
     debugDelay = conf.getInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0);
   } else {
     sched = new ScheduledThreadPoolExecutor(YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT, tf);
   }
   sched.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
   sched.setKeepAliveTime(60L, SECONDS);
   if (stateStore.canRecover()) {
     recover(stateStore.loadDeletionServiceState());
   }
   super.serviceInit(conf);
 }
 @Override
 protected void serviceInit(Configuration conf) throws Exception {
   this.hasProxyUserPrivileges =
       conf.getBoolean(
           YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED,
           YarnConfiguration.DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED);
   this.tokenKeepAliveEnabled =
       conf.getBoolean(
           YarnConfiguration.LOG_AGGREGATION_ENABLED,
           YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED);
   this.tokenRemovalDelayMs =
       conf.getInt(
           YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
           YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS);
   this.credentialsValidTimeRemaining =
       conf.getLong(
           RM_SYSTEM_CREDENTIALS_VALID_TIME_REMAINING,
           DEFAULT_RM_SYSTEM_CREDENTIALS_VALID_TIME_REMAINING);
   setLocalSecretManagerAndServiceAddr();
   renewerService = createNewThreadPoolService(conf);
   pendingEventQueue = new LinkedBlockingQueue<DelegationTokenRenewerEvent>();
   renewalTimer = new Timer(true);
   super.serviceInit(conf);
 }
Beispiel #8
0
  @Override
  protected void serviceInit(Configuration conf) throws Exception {
    String auth = conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
    if (auth == null || "simple".equals(auth)) {
      isSecurityEnabled = false;
    } else if ("kerberos".equals(auth)) {
      isSecurityEnabled = true;
    } else {
      LOG.warn(
          "Unrecongized attribute value for "
              + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION
              + " of "
              + auth);
    }
    String proxy = YarnConfiguration.getProxyHostAndPort(conf);
    String[] proxyParts = proxy.split(":");
    proxyHost = proxyParts[0];

    fetcher = new AppReportFetcher(conf);
    bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
    if (bindAddress == null || bindAddress.isEmpty()) {
      throw new YarnRuntimeException(
          YarnConfiguration.PROXY_ADDRESS + " is not set so the proxy will not run.");
    }
    LOG.info("Instantiating Proxy at " + bindAddress);
    String[] parts = StringUtils.split(bindAddress, ':');
    port = 0;
    if (parts.length == 2) {
      bindAddress = parts[0];
      port = Integer.parseInt(parts[1]);
    }
    acl =
        new AccessControlList(
            conf.get(YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
    super.serviceInit(conf);
  }
  @Override
  protected void serviceInit(Configuration conf) throws Exception {
    this.conf = conf;

    int serialNumberLowDigits = 3;
    serialNumberFormat =
        ("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");

    String doneDirPrefix = null;
    doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    try {
      doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
      doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
      doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
      mkdir(
          doneDirFc,
          doneDirPrefixPath,
          new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
    } catch (IOException e) {
      throw new YarnRuntimeException(
          "Error creating done directory: [" + doneDirPrefixPath + "]", e);
    }

    String intermediateDoneDirPrefix = null;
    intermediateDoneDirPrefix = JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
    try {
      intermediateDoneDirPath =
          FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
      intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
      mkdir(
          intermediateDoneDirFc,
          intermediateDoneDirPath,
          new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
    } catch (IOException e) {
      LOG.info("error creating done directory on dfs " + e);
      throw new YarnRuntimeException(
          "Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
    }

    this.aclsMgr = new JobACLsManager(conf);

    maxHistoryAge =
        conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);

    jobListCache =
        new JobListCache(
            conf.getInt(
                JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,
                JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE),
            maxHistoryAge);

    serialNumberIndex =
        new SerialNumberIndex(
            conf.getInt(
                JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
                JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

    int numMoveThreads =
        conf.getInt(
            JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
            JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
    ThreadFactory tf =
        new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
    moveToDoneExecutor =
        new ThreadPoolExecutor(
            numMoveThreads,
            numMoveThreads,
            1,
            TimeUnit.HOURS,
            new LinkedBlockingQueue<Runnable>(),
            tf);

    super.serviceInit(conf);
  }
  @Override
  public void serviceInit(final Configuration conf) throws Exception {

    this.state = DAGAppMasterState.INITED;

    this.amConf = conf;
    conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);

    downloadTokensAndSetupUGI(conf);

    context = new RunningAppContext(conf);

    clientHandler = new DAGClientHandler();

    dispatcher = createDispatcher();
    addIfService(dispatcher, false);

    clientRpcServer = new DAGClientServer(clientHandler);
    addIfService(clientRpcServer, true);

    taskHeartbeatHandler = createTaskHeartbeatHandler(context, conf);
    addIfService(taskHeartbeatHandler, true);

    containerHeartbeatHandler = createContainerHeartbeatHandler(context, conf);
    addIfService(containerHeartbeatHandler, true);

    // service to handle requests to TaskUmbilicalProtocol
    taskAttemptListener =
        createTaskAttemptListener(context, taskHeartbeatHandler, containerHeartbeatHandler);
    addIfService(taskAttemptListener, true);

    containers = new AMContainerMap(containerHeartbeatHandler, taskAttemptListener, context);
    addIfService(containers, true);
    dispatcher.register(AMContainerEventType.class, containers);

    nodes = new AMNodeMap(dispatcher.getEventHandler(), context);
    addIfService(nodes, true);
    dispatcher.register(AMNodeEventType.class, nodes);

    // service to do the task cleanup
    taskCleaner = createTaskCleaner(context);
    addIfService(taskCleaner, true);

    this.dagEventDispatcher = new DagEventDispatcher();
    this.vertexEventDispatcher = new VertexEventDispatcher();

    // register the event dispatchers
    dispatcher.register(DAGAppMasterEventType.class, new DAGAppMasterEventHandler());
    dispatcher.register(DAGEventType.class, dagEventDispatcher);
    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
    dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
    dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher());
    dispatcher.register(TaskCleaner.EventType.class, taskCleaner);

    taskSchedulerEventHandler =
        new TaskSchedulerEventHandler(context, clientRpcServer, dispatcher.getEventHandler());
    addIfService(taskSchedulerEventHandler, true);
    dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
    addIfServiceDependency(taskSchedulerEventHandler, clientRpcServer);

    containerLauncher = createContainerLauncher(context);
    addIfService(containerLauncher, true);
    dispatcher.register(NMCommunicatorEventType.class, containerLauncher);

    historyEventHandler = new HistoryEventHandler(context);
    addIfService(historyEventHandler, true);
    dispatcher.register(HistoryEventType.class, historyEventHandler);

    this.sessionTimeoutInterval =
        1000
            * amConf.getInt(
                TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
                TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT);

    initServices(conf);
    super.serviceInit(conf);
  }