/**
   * Scans the specified path and populates the intermediate cache.
   *
   * @param absPath
   * @throws IOException
   */
  private void scanIntermediateDirectory(final Path absPath) throws IOException {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Scanning intermediate dir " + absPath);
    }
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(absPath, intermediateDoneDirFc);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Found " + fileStatusList.size() + " files");
    }
    for (FileStatus fs : fileStatusList) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("scanning file: " + fs.getPath());
      }
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
      String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName =
          JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo =
          new HistoryFileInfo(
              fs.getPath(),
              new Path(fs.getPath().getParent(), confFileName),
              new Path(fs.getPath().getParent(), summaryFileName),
              jobIndexInfo,
              false);

      final HistoryFileInfo old = jobListCache.addIfAbsent(fileInfo);
      if (old == null || old.didMoveFail()) {
        final HistoryFileInfo found = (old == null) ? fileInfo : old;
        long cutoff = System.currentTimeMillis() - maxHistoryAge;
        if (found.getJobIndexInfo().getFinishTime() <= cutoff) {
          try {
            found.delete();
          } catch (IOException e) {
            LOG.warn("Error cleaning up a HistoryFile that is out of date.", e);
          }
        } else {
          if (LOG.isDebugEnabled()) {
            LOG.debug("Scheduling move to done of " + found);
          }
          moveToDoneExecutor.execute(
              new Runnable() {
                @Override
                public void run() {
                  try {
                    found.moveToDone();
                  } catch (IOException e) {
                    LOG.info("Failed to process fileInfo for job: " + found.getJobId(), e);
                  }
                }
              });
        }
      } else if (old != null && !old.isMovePending()) {
        // This is a duplicate so just delete it
        if (LOG.isDebugEnabled()) {
          LOG.debug("Duplicate: deleting");
        }
        fileInfo.delete();
      }
    }
  }
 public HistoryFileInfo addIfAbsent(HistoryFileInfo fileInfo) {
   JobId jobId = fileInfo.getJobId();
   if (LOG.isDebugEnabled()) {
     LOG.debug("Adding " + jobId + " to job list cache with " + fileInfo.getJobIndexInfo());
   }
   HistoryFileInfo old = cache.putIfAbsent(jobId, fileInfo);
   if (cache.size() > maxSize) {
     // There is a race here, where more then one thread could be trying to
     // remove entries.  This could result in too many entries being removed
     // from the cache.  This is considered OK as the size of the cache
     // should be rather large, and we would rather have performance over
     // keeping the cache size exactly at the maximum.
     Iterator<JobId> keys = cache.navigableKeySet().iterator();
     long cutoff = System.currentTimeMillis() - maxAge;
     while (cache.size() > maxSize && keys.hasNext()) {
       JobId key = keys.next();
       HistoryFileInfo firstValue = cache.get(key);
       if (firstValue != null) {
         synchronized (firstValue) {
           if (firstValue.isMovePending()) {
             if (firstValue.didMoveFail() && firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
               cache.remove(key);
               // Now lets try to delete it
               try {
                 firstValue.delete();
               } catch (IOException e) {
                 LOG.error(
                     "Error while trying to delete history files"
                         + " that could not be moved to done.",
                     e);
               }
             } else {
               LOG.warn(
                   "Waiting to remove "
                       + key
                       + " from JobListCache because it is not in done yet.");
             }
           } else {
             cache.remove(key);
           }
         }
       }
     }
   }
   return old;
 }
 private void deleteJobFromDone(HistoryFileInfo fileInfo) throws IOException {
   jobListCache.delete(fileInfo);
   fileInfo.delete();
 }