コード例 #1
0
  @Override
  protected void serviceInit(Configuration conf) throws Exception {
    this.conf = conf;

    int serialNumberLowDigits = 3;
    serialNumberFormat =
        ("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");

    String doneDirPrefix = null;
    doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    try {
      doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
      doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
      doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
      mkdir(
          doneDirFc,
          doneDirPrefixPath,
          new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
    } catch (IOException e) {
      throw new YarnRuntimeException(
          "Error creating done directory: [" + doneDirPrefixPath + "]", e);
    }

    String intermediateDoneDirPrefix = null;
    intermediateDoneDirPrefix = JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
    try {
      intermediateDoneDirPath =
          FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
      intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
      mkdir(
          intermediateDoneDirFc,
          intermediateDoneDirPath,
          new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
    } catch (IOException e) {
      LOG.info("error creating done directory on dfs " + e);
      throw new YarnRuntimeException(
          "Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
    }

    this.aclsMgr = new JobACLsManager(conf);

    maxHistoryAge =
        conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);

    jobListCache =
        new JobListCache(
            conf.getInt(
                JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,
                JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE),
            maxHistoryAge);

    serialNumberIndex =
        new SerialNumberIndex(
            conf.getInt(
                JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
                JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

    int numMoveThreads =
        conf.getInt(
            JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
            JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
    ThreadFactory tf =
        new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
    moveToDoneExecutor =
        new ThreadPoolExecutor(
            numMoveThreads,
            numMoveThreads,
            1,
            TimeUnit.HOURS,
            new LinkedBlockingQueue<Runnable>(),
            tf);

    super.serviceInit(conf);
  }
コード例 #2
0
 /**
  * Returns TRUE if the history dirs were created, FALSE if they could not be created because the
  * FileSystem is not reachable or in safe mode and throws and exception otherwise.
  */
 @VisibleForTesting
 boolean tryCreatingHistoryDirs(boolean logWait) throws IOException {
   boolean succeeded = true;
   String doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
   try {
     doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
     doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
     doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
     mkdir(
         doneDirFc,
         doneDirPrefixPath,
         new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
   } catch (ConnectException ex) {
     if (logWait) {
       /* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be available") */
       LOG.waiting_for_filesystem_available(
               String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
           .tag("methodCall")
           .info();
     }
     succeeded = false;
   } catch (IOException e) {
     if (isBecauseSafeMode(e)) {
       succeeded = false;
       if (logWait) {
         /* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be out of safe mode") */
         LOG.waiting_for_filesystem_out_safe_mode(
                 String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
             .tag("methodCall")
             .info();
       }
     } else {
       throw new YarnRuntimeException(
           "Error creating done directory: [" + doneDirPrefixPath + "]", e);
     }
   }
   if (succeeded) {
     String intermediateDoneDirPrefix =
         JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
     try {
       intermediateDoneDirPath =
           FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
       intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
       mkdir(
           intermediateDoneDirFc,
           intermediateDoneDirPath,
           new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
     } catch (ConnectException ex) {
       succeeded = false;
       if (logWait) {
         /* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be available") */
         LOG.waiting_for_filesystem_available(
                 String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
             .tag("methodCall")
             .info();
       }
     } catch (IOException e) {
       if (isBecauseSafeMode(e)) {
         succeeded = false;
         if (logWait) {
           /* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be out of safe mode") */
           LOG.waiting_for_filesystem_out_safe_mode(
                   String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
               .tag("methodCall")
               .info();
         }
       } else {
         throw new YarnRuntimeException(
             "Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
       }
     }
   }
   return succeeded;
 }