private void addDirectoryToJobListCache(Path path) throws IOException {
   if (LogGlobal.isDebugEnabled()) {
     /* LOG.debug("Adding "+path+" to job list cache.") */
     LOG.adding_job_list_cache(path.toString()).debug();
   }
   List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path, doneDirFc);
   for (FileStatus fs : historyFileList) {
     if (LogGlobal.isDebugEnabled()) {
       /* LOG.debug("Adding in history for "+fs.getPath()) */
       LOG.adding_history_for(String.valueOf(fs.getPath())).tag("methodCall").debug();
     }
     JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
     String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
     String summaryFileName =
         JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
     HistoryFileInfo fileInfo =
         new HistoryFileInfo(
             fs.getPath(),
             new Path(fs.getPath().getParent(), confFileName),
             new Path(fs.getPath().getParent(), summaryFileName),
             jobIndexInfo,
             true);
     jobListCache.addIfAbsent(fileInfo);
   }
 }
  /**
   * Scans the specified path and populates the intermediate cache.
   *
   * @param absPath
   * @throws IOException
   */
  private void scanIntermediateDirectory(final Path absPath) throws IOException {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Scanning intermediate dir " + absPath);
    }
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(absPath, intermediateDoneDirFc);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Found " + fileStatusList.size() + " files");
    }
    for (FileStatus fs : fileStatusList) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("scanning file: " + fs.getPath());
      }
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
      String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName =
          JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo =
          new HistoryFileInfo(
              fs.getPath(),
              new Path(fs.getPath().getParent(), confFileName),
              new Path(fs.getPath().getParent(), summaryFileName),
              jobIndexInfo,
              false);

      final HistoryFileInfo old = jobListCache.addIfAbsent(fileInfo);
      if (old == null || old.didMoveFail()) {
        final HistoryFileInfo found = (old == null) ? fileInfo : old;
        long cutoff = System.currentTimeMillis() - maxHistoryAge;
        if (found.getJobIndexInfo().getFinishTime() <= cutoff) {
          try {
            found.delete();
          } catch (IOException e) {
            LOG.warn("Error cleaning up a HistoryFile that is out of date.", e);
          }
        } else {
          if (LOG.isDebugEnabled()) {
            LOG.debug("Scheduling move to done of " + found);
          }
          moveToDoneExecutor.execute(
              new Runnable() {
                @Override
                public void run() {
                  try {
                    found.moveToDone();
                  } catch (IOException e) {
                    LOG.info("Failed to process fileInfo for job: " + found.getJobId(), e);
                  }
                }
              });
        }
      } else if (old != null && !old.isMovePending()) {
        // This is a duplicate so just delete it
        if (LOG.isDebugEnabled()) {
          LOG.debug("Duplicate: deleting");
        }
        fileInfo.delete();
      }
    }
  }
  /**
   * Clean up older history files.
   *
   * @throws IOException on any error trying to remove the entries.
   */
  @SuppressWarnings("unchecked")
  void clean() throws IOException {
    // TODO this should be replaced by something that knows about the directory
    // structure and will put less of a load on HDFS.
    long cutoff = System.currentTimeMillis() - maxHistoryAge;
    boolean halted = false;
    // TODO Delete YYYY/MM/DD directories.
    List<FileStatus> serialDirList = findTimestampedDirectories();
    // Sort in ascending order. Relies on YYYY/MM/DD/Serial
    Collections.sort(serialDirList);
    for (FileStatus serialDir : serialDirList) {
      List<FileStatus> historyFileList =
          scanDirectoryForHistoryFiles(serialDir.getPath(), doneDirFc);
      for (FileStatus historyFile : historyFileList) {
        JobIndexInfo jobIndexInfo =
            FileNameIndexUtils.getIndexInfo(historyFile.getPath().getName());
        long effectiveTimestamp = getEffectiveTimestamp(jobIndexInfo.getFinishTime(), historyFile);
        if (effectiveTimestamp <= cutoff) {
          HistoryFileInfo fileInfo = this.jobListCache.get(jobIndexInfo.getJobId());
          if (fileInfo == null) {
            String confFileName =
                JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());

            fileInfo =
                new HistoryFileInfo(
                    historyFile.getPath(),
                    new Path(historyFile.getPath().getParent(), confFileName),
                    null,
                    jobIndexInfo,
                    true);
          }
          deleteJobFromDone(fileInfo);
        } else {
          halted = true;
          break;
        }
      }
      if (!halted) {
        doneDirFc.delete(doneDirFc.makeQualified(serialDir.getPath()), true);
        removeDirectoryFromSerialNumberIndex(serialDir.getPath());
        existingDoneSubdirs.remove(serialDir.getPath());
      } else {
        break; // Don't scan any more directories.
      }
    }
  }
  /**
   * Clean up older history files.
   *
   * @throws IOException on any error trying to remove the entries.
   */
  @SuppressWarnings("unchecked")
  void clean() throws IOException {
    long cutoff = System.currentTimeMillis() - maxHistoryAge;
    boolean halted = false;
    List<FileStatus> serialDirList = getHistoryDirsForCleaning(cutoff);
    // Sort in ascending order. Relies on YYYY/MM/DD/Serial
    Collections.sort(serialDirList);
    for (FileStatus serialDir : serialDirList) {
      List<FileStatus> historyFileList =
          scanDirectoryForHistoryFiles(serialDir.getPath(), doneDirFc);
      for (FileStatus historyFile : historyFileList) {
        JobIndexInfo jobIndexInfo =
            FileNameIndexUtils.getIndexInfo(historyFile.getPath().getName());
        long effectiveTimestamp = getEffectiveTimestamp(jobIndexInfo.getFinishTime(), historyFile);
        if (effectiveTimestamp <= cutoff) {
          HistoryFileInfo fileInfo = this.jobListCache.get(jobIndexInfo.getJobId());
          if (fileInfo == null) {
            String confFileName =
                JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());

            fileInfo =
                new HistoryFileInfo(
                    historyFile.getPath(),
                    new Path(historyFile.getPath().getParent(), confFileName),
                    null,
                    jobIndexInfo,
                    true);
          }
          deleteJobFromDone(fileInfo);
        } else {
          halted = true;
          break;
        }
      }
      if (!halted) {
        deleteDir(serialDir);
        removeDirectoryFromSerialNumberIndex(serialDir.getPath());
        existingDoneSubdirs.remove(serialDir.getPath());
      } else {
        break; // Don't scan any more directories.
      }
    }
  }
 /**
  * Searches the job history file FileStatus list for the specified JobId.
  *
  * @param fileStatusList fileStatus list of Job History Files.
  * @param jobId The JobId to find.
  * @return A FileInfo object for the jobId, null if not found.
  * @throws IOException
  */
 private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList, JobId jobId)
     throws IOException {
   for (FileStatus fs : fileStatusList) {
     JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
     if (jobIndexInfo.getJobId().equals(jobId)) {
       String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
       String summaryFileName =
           JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
       HistoryFileInfo fileInfo =
           new HistoryFileInfo(
               fs.getPath(),
               new Path(fs.getPath().getParent(), confFileName),
               new Path(fs.getPath().getParent(), summaryFileName),
               jobIndexInfo,
               true);
       return fileInfo;
     }
   }
   return null;
 }
 public JobId getJobId() {
   return jobIndexInfo.getJobId();
 }
 /**
  * Parse a job from the JobHistoryFile, if the underlying file is not going to be deleted.
  *
  * @return the Job or null if the underlying file was deleted.
  * @throws IOException if there is an error trying to read the file.
  */
 public synchronized Job loadJob() throws IOException {
   return new CompletedJob(
       conf, jobIndexInfo.getJobId(), historyFile, false, jobIndexInfo.getUser(), this, aclsMgr);
 }
    private synchronized void moveToDone() throws IOException {
      if (LOG.isDebugEnabled()) {
        LOG.debug("moveToDone: " + historyFile);
      }
      if (!isMovePending()) {
        // It was either deleted or is already in done. Either way do nothing
        if (LOG.isDebugEnabled()) {
          LOG.debug("Move no longer pending");
        }
        return;
      }
      try {
        long completeTime = jobIndexInfo.getFinishTime();
        if (completeTime == 0) {
          completeTime = System.currentTimeMillis();
        }
        JobId jobId = jobIndexInfo.getJobId();

        List<Path> paths = new ArrayList<Path>(2);
        if (historyFile == null) {
          LOG.info("No file for job-history with " + jobId + " found in cache!");
        } else {
          paths.add(historyFile);
        }

        if (confFile == null) {
          LOG.info("No file for jobConf with " + jobId + " found in cache!");
        } else {
          paths.add(confFile);
        }

        if (summaryFile == null) {
          LOG.info("No summary file for job: " + jobId);
        } else {
          String jobSummaryString = getJobSummary(intermediateDoneDirFc, summaryFile);
          SUMMARY_LOG.info(jobSummaryString);
          LOG.info("Deleting JobSummary file: [" + summaryFile + "]");
          intermediateDoneDirFc.delete(summaryFile, false);
          summaryFile = null;
        }

        Path targetDir = canonicalHistoryLogPath(jobId, completeTime);
        addDirectoryToSerialNumberIndex(targetDir);
        makeDoneSubdir(targetDir);
        if (historyFile != null) {
          Path toPath = doneDirFc.makeQualified(new Path(targetDir, historyFile.getName()));
          if (!toPath.equals(historyFile)) {
            moveToDoneNow(historyFile, toPath);
            historyFile = toPath;
          }
        }
        if (confFile != null) {
          Path toPath = doneDirFc.makeQualified(new Path(targetDir, confFile.getName()));
          if (!toPath.equals(confFile)) {
            moveToDoneNow(confFile, toPath);
            confFile = toPath;
          }
        }
        state = HistoryInfoState.IN_DONE;
      } catch (Throwable t) {
        LOG.error("Error while trying to move a job to done", t);
        this.state = HistoryInfoState.MOVE_FAILED;
      }
    }