Ejemplo n.º 1
0
/**
 * A {@link GSet} is set, which supports the {@link #get(Object)} operation. The {@link
 * #get(Object)} operation uses a key to lookup an element.
 *
 * <p>Null element is not supported.
 *
 * @param <K> The type of the keys.
 * @param <E> The type of the elements, which must be a subclass of the keys.
 */
@InterfaceAudience.Private
public interface GSet<K, E extends K> extends Iterable<E> {
  static final /* LogLOG=LogFactory.getLog(GSet.class); */ UtilNamespace LOG =
      LoggerFactory.getLogger(UtilNamespace.class, new SimpleLogger());

  /** @return The size of this set. */
  int size();

  /**
   * Does this set contain an element corresponding to the given key?
   *
   * @param key The given key.
   * @return true if the given key equals to a stored element. Otherwise, return false.
   * @throws NullPointerException if key == null.
   */
  boolean contains(K key);

  /**
   * Return the stored element which is equal to the given key. This operation is similar to {@link
   * java.util.Map#get(Object)}.
   *
   * @param key The given key.
   * @return The stored element if it exists. Otherwise, return null.
   * @throws NullPointerException if key == null.
   */
  E get(K key);

  /**
   * Add/replace an element. If the element does not exist, add it to the set. Otherwise, replace
   * the existing element.
   *
   * <p>Note that this operation is similar to {@link java.util.Map#put(Object, Object)} but is
   * different from {@link java.util.Set#add(Object)} which does not replace the existing element if
   * there is any.
   *
   * @param element The element being put.
   * @return the previous stored element if there is any. Otherwise, return null.
   * @throws NullPointerException if element == null.
   */
  E put(E element);

  /**
   * Remove the element corresponding to the given key. This operation is similar to {@link
   * java.util.Map#remove(Object)}.
   *
   * @param key The key of the element being removed.
   * @return If such element exists, return it. Otherwise, return null.
   * @throws NullPointerException if key == null.
   */
  E remove(K key);

  void clear();
}
Ejemplo n.º 2
0
/** Facilitates hooking process termination for tests and debugging. */
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Unstable
public final class ExitUtil {
  private static final /* LogLOG=LogFactory.getLog(ExitUtil.class.getName()); */ UtilNamespace LOG =
      LoggerFactory.getLogger(UtilNamespace.class, new SimpleLogger());
  private static volatile boolean systemExitDisabled = false;
  private static volatile boolean systemHaltDisabled = false;
  private static volatile ExitException firstExitException;
  private static volatile HaltException firstHaltException;

  public static class ExitException extends RuntimeException {
    private static final long serialVersionUID = 1L;
    public final int status;

    public ExitException(int status, String msg) {
      super(msg);
      this.status = status;
    }
  }

  public static class HaltException extends RuntimeException {
    private static final long serialVersionUID = 1L;
    public final int status;

    public HaltException(int status, String msg) {
      super(msg);
      this.status = status;
    }
  }

  /** Disable the use of System.exit for testing. */
  public static void disableSystemExit() {
    systemExitDisabled = true;
  }

  /** Disable the use of {@code Runtime.getRuntime().halt() } for testing. */
  public static void disableSystemHalt() {
    systemHaltDisabled = true;
  }

  /** @return true if terminate has been called */
  public static boolean terminateCalled() {
    // Either we set this member or we actually called System#exit
    return firstExitException != null;
  }

  /** @return true if halt has been called */
  public static boolean haltCalled() {
    return firstHaltException != null;
  }

  /** @return the first ExitException thrown, null if none thrown yet */
  public static ExitException getFirstExitException() {
    return firstExitException;
  }

  /** @return the first {@code HaltException} thrown, null if none thrown yet */
  public static HaltException getFirstHaltException() {
    return firstHaltException;
  }

  /**
   * Reset the tracking of process termination. This is for use in unit tests where one test in the
   * suite expects an exit but others do not.
   */
  public static void resetFirstExitException() {
    firstExitException = null;
  }

  public static void resetFirstHaltException() {
    firstHaltException = null;
  }

  /**
   * Terminate the current process. Note that terminate is the *only* method that should be used to
   * terminate the daemon processes.
   *
   * @param status exit code
   * @param msg message used to create the {@code ExitException}
   * @throws ExitException if System.exit is disabled for test purposes
   */
  public static void terminate(int status, String msg) throws ExitException {
    /* LOG.info("Exiting with status "+status) */
    LOG.exiting_with_status(status).info();
    if (systemExitDisabled) {
      ExitException ee = new ExitException(status, msg);
      /* LOG.fatal("Terminate called",ee) */
      LOG.terminate_called(ee.toString()).fatal();
      if (null == firstExitException) {
        firstExitException = ee;
      }
      throw ee;
    }
    System.exit(status);
  }

  /**
   * Forcibly terminates the currently running Java virtual machine.
   *
   * @param status exit code
   * @param msg message used to create the {@code HaltException}
   * @throws HaltException if Runtime.getRuntime().halt() is disabled for test purposes
   */
  public static void halt(int status, String msg) throws HaltException {
    /* LOG.info("Halt with status "+status+" Message: "+msg) */
    LOG.halt_with_status_message(status, msg).info();
    if (systemHaltDisabled) {
      HaltException ee = new HaltException(status, msg);
      /* LOG.fatal("Halt called",ee) */
      LOG.halt_called(ee.toString()).fatal();
      if (null == firstHaltException) {
        firstHaltException = ee;
      }
      throw ee;
    }
    Runtime.getRuntime().halt(status);
  }

  /**
   * Like {@link terminate(int, String)} but uses the given throwable to initialize the
   * ExitException.
   *
   * @param status
   * @param t throwable used to create the ExitException
   * @throws ExitException if System.exit is disabled for test purposes
   */
  public static void terminate(int status, Throwable t) throws ExitException {
    terminate(status, StringUtils.stringifyException(t));
  }

  /**
   * Forcibly terminates the currently running Java virtual machine.
   *
   * @param status
   * @param t
   * @throws ExitException
   */
  public static void halt(int status, Throwable t) throws HaltException {
    halt(status, StringUtils.stringifyException(t));
  }

  /**
   * Like {@link terminate(int, String)} without a message.
   *
   * @param status
   * @throws ExitException if System.exit is disabled for test purposes
   */
  public static void terminate(int status) throws ExitException {
    terminate(status, "ExitException");
  }

  /**
   * Forcibly terminates the currently running Java virtual machine.
   *
   * @param status
   * @throws ExitException
   */
  public static void halt(int status) throws HaltException {
    halt(status, "HaltException");
  }
}
/**
 * A base class for file-based {@link InputFormat}s.
 *
 * <p><code>FileInputFormat</code> is the base class for all file-based <code>InputFormat</code>s.
 * This provides a generic implementation of {@link #getSplits(JobContext)}. Subclasses of <code>
 * FileInputFormat</code> can also override the {@link #isSplitable(JobContext, Path)} method to
 * ensure input-files are not split-up and are processed as a whole by {@link Mapper}s.
 */
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
  public static final String INPUT_DIR = "mapreduce.input.fileinputformat.inputdir";
  public static final String SPLIT_MAXSIZE = "mapreduce.input.fileinputformat.split.maxsize";
  public static final String SPLIT_MINSIZE = "mapreduce.input.fileinputformat.split.minsize";
  public static final String PATHFILTER_CLASS = "mapreduce.input.pathFilter.class";
  public static final String NUM_INPUT_FILES = "mapreduce.input.fileinputformat.numinputfiles";
  public static final String INPUT_DIR_RECURSIVE =
      "mapreduce.input.fileinputformat.input.dir.recursive";
  public static final String LIST_STATUS_NUM_THREADS =
      "mapreduce.input.fileinputformat.list-status.num-threads";
  public static final int DEFAULT_LIST_STATUS_NUM_THREADS = 1;

  private static final /* LogLOG=LogFactory.getLog(FileInputFormat.class); */ MapreduceNamespace
      LOG = LoggerFactory.getLogger(MapreduceNamespace.class, new SimpleLogger());

  private static final double SPLIT_SLOP = 1.1; // 10% slop

  @Deprecated
  public static enum Counter {
    BYTES_READ
  }

  private static final PathFilter hiddenFileFilter =
      new PathFilter() {
        public boolean accept(Path p) {
          String name = p.getName();
          return !name.startsWith("_") && !name.startsWith(".");
        }
      };

  /**
   * Proxy PathFilter that accepts a path only if all filters given in the constructor do. Used by
   * the listPaths() to apply the built-in hiddenFileFilter together with a user provided one (if
   * any).
   */
  private static class MultiPathFilter implements PathFilter {
    private List<PathFilter> filters;

    public MultiPathFilter(List<PathFilter> filters) {
      this.filters = filters;
    }

    public boolean accept(Path path) {
      for (PathFilter filter : filters) {
        if (!filter.accept(path)) {
          return false;
        }
      }
      return true;
    }
  }

  /**
   * @param job the job to modify
   * @param inputDirRecursive
   */
  public static void setInputDirRecursive(Job job, boolean inputDirRecursive) {
    job.getConfiguration().setBoolean(INPUT_DIR_RECURSIVE, inputDirRecursive);
  }

  /**
   * @param job the job to look at.
   * @return should the files to be read recursively?
   */
  public static boolean getInputDirRecursive(JobContext job) {
    return job.getConfiguration().getBoolean(INPUT_DIR_RECURSIVE, false);
  }

  /**
   * Get the lower bound on split size imposed by the format.
   *
   * @return the number of bytes of the minimal split for this format
   */
  protected long getFormatMinSplitSize() {
    return 1;
  }

  /**
   * Is the given filename splitable? Usually, true, but if the file is stream compressed, it will
   * not be. <code>FileInputFormat</code> implementations can override this and return <code>false
   * </code> to ensure that individual input files are never split-up so that {@link Mapper}s
   * process entire files.
   *
   * @param context the job context
   * @param filename the file name to check
   * @return is this file splitable?
   */
  protected boolean isSplitable(JobContext context, Path filename) {
    return true;
  }

  /**
   * Set a PathFilter to be applied to the input paths for the map-reduce job.
   *
   * @param job the job to modify
   * @param filter the PathFilter class use for filtering the input paths.
   */
  public static void setInputPathFilter(Job job, Class<? extends PathFilter> filter) {
    job.getConfiguration().setClass(PATHFILTER_CLASS, filter, PathFilter.class);
  }

  /**
   * Set the minimum input split size
   *
   * @param job the job to modify
   * @param size the minimum size
   */
  public static void setMinInputSplitSize(Job job, long size) {
    job.getConfiguration().setLong(SPLIT_MINSIZE, size);
  }

  /**
   * Get the minimum split size
   *
   * @param job the job
   * @return the minimum number of bytes that can be in a split
   */
  public static long getMinSplitSize(JobContext job) {
    return job.getConfiguration().getLong(SPLIT_MINSIZE, 1L);
  }

  /**
   * Set the maximum split size
   *
   * @param job the job to modify
   * @param size the maximum split size
   */
  public static void setMaxInputSplitSize(Job job, long size) {
    job.getConfiguration().setLong(SPLIT_MAXSIZE, size);
  }

  /**
   * Get the maximum split size.
   *
   * @param context the job to look at.
   * @return the maximum number of bytes a split can include
   */
  public static long getMaxSplitSize(JobContext context) {
    return context.getConfiguration().getLong(SPLIT_MAXSIZE, Long.MAX_VALUE);
  }

  /**
   * Get a PathFilter instance of the filter set for the input paths.
   *
   * @return the PathFilter instance set for the job, NULL if none has been set.
   */
  public static PathFilter getInputPathFilter(JobContext context) {
    Configuration conf = context.getConfiguration();
    Class<?> filterClass = conf.getClass(PATHFILTER_CLASS, null, PathFilter.class);
    return (filterClass != null)
        ? (PathFilter) ReflectionUtils.newInstance(filterClass, conf)
        : null;
  }

  /**
   * List input directories. Subclasses may override to, e.g., select only files matching a regular
   * expression.
   *
   * @param job the job to list input paths for
   * @return array of FileStatus objects
   * @throws IOException if zero items.
   */
  protected List<FileStatus> listStatus(JobContext job) throws IOException {
    Path[] dirs = getInputPaths(job);
    if (dirs.length == 0) {
      throw new IOException("No input paths specified in job");
    }

    // get tokens for all the required FileSystems..
    TokenCache.obtainTokensForNamenodes(job.getCredentials(), dirs, job.getConfiguration());

    // Whether we need to recursive look into the directory structure
    boolean recursive = getInputDirRecursive(job);

    // creates a MultiPathFilter with the hiddenFileFilter and the
    // user provided one (if any).
    List<PathFilter> filters = new ArrayList<PathFilter>();
    filters.add(hiddenFileFilter);
    PathFilter jobFilter = getInputPathFilter(job);
    if (jobFilter != null) {
      filters.add(jobFilter);
    }
    PathFilter inputFilter = new MultiPathFilter(filters);

    List<FileStatus> result = null;

    int numThreads =
        job.getConfiguration().getInt(LIST_STATUS_NUM_THREADS, DEFAULT_LIST_STATUS_NUM_THREADS);
    Stopwatch sw = new Stopwatch().start();
    if (numThreads == 1) {
      result = singleThreadedListStatus(job, dirs, inputFilter, recursive);
    } else {
      Iterable<FileStatus> locatedFiles = null;
      try {
        LocatedFileStatusFetcher locatedFileStatusFetcher =
            new LocatedFileStatusFetcher(
                job.getConfiguration(), dirs, recursive, inputFilter, true);
        locatedFiles = locatedFileStatusFetcher.getFileStatuses();
      } catch (InterruptedException e) {
        throw new IOException("Interrupted while getting file statuses");
      }
      result = Lists.newArrayList(locatedFiles);
    }

    sw.stop();
    if (LogGlobal.isDebugEnabled()) {
      /* LOG.debug("Time taken to get FileStatuses: "+sw.elapsedMillis()) */
      LOG.time_taken_get_filestatuses(String.valueOf(sw.elapsedMillis())).tag("methodCall").debug();
    }
    /* LOG.info("Total input paths to process : "+result.size()) */
    LOG.total_input_paths_process(String.valueOf(result.size())).tag("methodCall").info();
    return result;
  }

  private List<FileStatus> singleThreadedListStatus(
      JobContext job, Path[] dirs, PathFilter inputFilter, boolean recursive) throws IOException {
    List<FileStatus> result = new ArrayList<FileStatus>();
    List<IOException> errors = new ArrayList<IOException>();
    for (int i = 0; i < dirs.length; ++i) {
      Path p = dirs[i];
      FileSystem fs = p.getFileSystem(job.getConfiguration());
      FileStatus[] matches = fs.globStatus(p, inputFilter);
      if (matches == null) {
        errors.add(new IOException("Input path does not exist: " + p));
      } else if (matches.length == 0) {
        errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
      } else {
        for (FileStatus globStat : matches) {
          if (globStat.isDirectory()) {
            RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(globStat.getPath());
            while (iter.hasNext()) {
              LocatedFileStatus stat = iter.next();
              if (inputFilter.accept(stat.getPath())) {
                if (recursive && stat.isDirectory()) {
                  addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
                } else {
                  result.add(stat);
                }
              }
            }
          } else {
            result.add(globStat);
          }
        }
      }
    }

    if (!errors.isEmpty()) {
      throw new InvalidInputException(errors);
    }
    return result;
  }

  /**
   * Add files in the input path recursively into the results.
   *
   * @param result The List to store all files.
   * @param fs The FileSystem.
   * @param path The input path.
   * @param inputFilter The input filter that can be used to filter files/dirs.
   * @throws IOException
   */
  protected void addInputPathRecursively(
      List<FileStatus> result, FileSystem fs, Path path, PathFilter inputFilter)
      throws IOException {
    RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
    while (iter.hasNext()) {
      LocatedFileStatus stat = iter.next();
      if (inputFilter.accept(stat.getPath())) {
        if (stat.isDirectory()) {
          addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
        } else {
          result.add(stat);
        }
      }
    }
  }

  /**
   * A factory that makes the split for this class. It can be overridden by sub-classes to make
   * sub-types
   */
  protected FileSplit makeSplit(Path file, long start, long length, String[] hosts) {
    return new FileSplit(file, start, length, hosts);
  }

  /**
   * Generate the list of files and make them into FileSplits.
   *
   * @param job the job context
   * @throws IOException
   */
  public List<InputSplit> getSplits(JobContext job) throws IOException {
    Stopwatch sw = new Stopwatch().start();
    long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
    long maxSize = getMaxSplitSize(job);

    // generate splits
    List<InputSplit> splits = new ArrayList<InputSplit>();
    List<FileStatus> files = listStatus(job);
    for (FileStatus file : files) {
      Path path = file.getPath();
      long length = file.getLen();
      if (length != 0) {
        BlockLocation[] blkLocations;
        if (file instanceof LocatedFileStatus) {
          blkLocations = ((LocatedFileStatus) file).getBlockLocations();
        } else {
          FileSystem fs = path.getFileSystem(job.getConfiguration());
          blkLocations = fs.getFileBlockLocations(file, 0, length);
        }
        if (isSplitable(job, path)) {
          long blockSize = file.getBlockSize();
          long splitSize = computeSplitSize(blockSize, minSize, maxSize);

          long bytesRemaining = length;
          while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) {
            int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
            splits.add(
                makeSplit(
                    path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts()));
            bytesRemaining -= splitSize;
          }

          if (bytesRemaining != 0) {
            int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
            splits.add(
                makeSplit(
                    path,
                    length - bytesRemaining,
                    bytesRemaining,
                    blkLocations[blkIndex].getHosts()));
          }
        } else { // not splitable
          splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts()));
        }
      } else {
        // Create empty hosts array for zero length files
        splits.add(makeSplit(path, 0, length, new String[0]));
      }
    }
    // Save the number of input files for metrics/loadgen
    job.getConfiguration().setLong(NUM_INPUT_FILES, files.size());
    sw.stop();
    if (LogGlobal.isDebugEnabled()) {
      /* LOG.debug("Total # of splits generated by getSplits: "+splits.size()+", TimeTaken: "+sw.elapsedMillis()) */
      LOG.total_splits_generated_getsplits_timetak(
              String.valueOf(splits.size()), String.valueOf(sw.elapsedMillis()))
          .tag("methodCall")
          .debug();
    }
    return splits;
  }

  protected long computeSplitSize(long blockSize, long minSize, long maxSize) {
    return Math.max(minSize, Math.min(maxSize, blockSize));
  }

  protected int getBlockIndex(BlockLocation[] blkLocations, long offset) {
    for (int i = 0; i < blkLocations.length; i++) {
      // is the offset inside this block?
      if ((blkLocations[i].getOffset() <= offset)
          && (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())) {
        return i;
      }
    }
    BlockLocation last = blkLocations[blkLocations.length - 1];
    long fileLength = last.getOffset() + last.getLength() - 1;
    throw new IllegalArgumentException(
        "Offset " + offset + " is outside of file (0.." + fileLength + ")");
  }

  /**
   * Sets the given comma separated paths as the list of inputs for the map-reduce job.
   *
   * @param job the job
   * @param commaSeparatedPaths Comma separated paths to be set as the list of inputs for the
   *     map-reduce job.
   */
  public static void setInputPaths(Job job, String commaSeparatedPaths) throws IOException {
    setInputPaths(job, StringUtils.stringToPath(getPathStrings(commaSeparatedPaths)));
  }

  /**
   * Add the given comma separated paths to the list of inputs for the map-reduce job.
   *
   * @param job The job to modify
   * @param commaSeparatedPaths Comma separated paths to be added to the list of inputs for the
   *     map-reduce job.
   */
  public static void addInputPaths(Job job, String commaSeparatedPaths) throws IOException {
    for (String str : getPathStrings(commaSeparatedPaths)) {
      addInputPath(job, new Path(str));
    }
  }

  /**
   * Set the array of {@link Path}s as the list of inputs for the map-reduce job.
   *
   * @param job The job to modify
   * @param inputPaths the {@link Path}s of the input directories/files for the map-reduce job.
   */
  public static void setInputPaths(Job job, Path... inputPaths) throws IOException {
    Configuration conf = job.getConfiguration();
    Path path = inputPaths[0].getFileSystem(conf).makeQualified(inputPaths[0]);
    StringBuffer str = new StringBuffer(StringUtils.escapeString(path.toString()));
    for (int i = 1; i < inputPaths.length; i++) {
      str.append(StringUtils.COMMA_STR);
      path = inputPaths[i].getFileSystem(conf).makeQualified(inputPaths[i]);
      str.append(StringUtils.escapeString(path.toString()));
    }
    conf.set(INPUT_DIR, str.toString());
  }

  /**
   * Add a {@link Path} to the list of inputs for the map-reduce job.
   *
   * @param job The {@link Job} to modify
   * @param path {@link Path} to be added to the list of inputs for the map-reduce job.
   */
  public static void addInputPath(Job job, Path path) throws IOException {
    Configuration conf = job.getConfiguration();
    path = path.getFileSystem(conf).makeQualified(path);
    String dirStr = StringUtils.escapeString(path.toString());
    String dirs = conf.get(INPUT_DIR);
    conf.set(INPUT_DIR, dirs == null ? dirStr : dirs + "," + dirStr);
  }

  // This method escapes commas in the glob pattern of the given paths.
  private static String[] getPathStrings(String commaSeparatedPaths) {
    int length = commaSeparatedPaths.length();
    int curlyOpen = 0;
    int pathStart = 0;
    boolean globPattern = false;
    List<String> pathStrings = new ArrayList<String>();

    for (int i = 0; i < length; i++) {
      char ch = commaSeparatedPaths.charAt(i);
      switch (ch) {
        case '{':
          {
            curlyOpen++;
            if (!globPattern) {
              globPattern = true;
            }
            break;
          }
        case '}':
          {
            curlyOpen--;
            if (curlyOpen == 0 && globPattern) {
              globPattern = false;
            }
            break;
          }
        case ',':
          {
            if (!globPattern) {
              pathStrings.add(commaSeparatedPaths.substring(pathStart, i));
              pathStart = i + 1;
            }
            break;
          }
        default:
          continue; // nothing special to do for this character
      }
    }
    pathStrings.add(commaSeparatedPaths.substring(pathStart, length));

    return pathStrings.toArray(new String[0]);
  }

  /**
   * Get the list of input {@link Path}s for the map-reduce job.
   *
   * @param context The job
   * @return the list of input {@link Path}s for the map-reduce job.
   */
  public static Path[] getInputPaths(JobContext context) {
    String dirs = context.getConfiguration().get(INPUT_DIR, "");
    String[] list = StringUtils.split(dirs);
    Path[] result = new Path[list.length];
    for (int i = 0; i < list.length; i++) {
      result[i] = new Path(StringUtils.unEscapeString(list[i]));
    }
    return result;
  }
}
Ejemplo n.º 4
0
/** This class returns build information about Hadoop components. */
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class VersionInfo {
  private static final /* LogLOG=LogFactory.getLog(VersionInfo.class); */ UtilNamespace LOG =
      LoggerFactory.getLogger(UtilNamespace.class, new SimpleLogger());

  private Properties info;

  protected VersionInfo(String component) {
    info = new Properties();
    String versionInfoFile = component + "-version-info.properties";
    InputStream is = null;
    try {
      is = Thread.currentThread().getContextClassLoader().getResourceAsStream(versionInfoFile);
      if (is == null) {
        throw new IOException("Resource not found");
      }
      info.load(is);
    } catch (IOException ex) {
      LogFactory.getLog(getClass())
          .warn("Could not read '" + versionInfoFile + "', " + ex.toString(), ex);
    } finally {
      IOUtils.closeStream(is);
    }
  }

  protected String _getVersion() {
    return info.getProperty("version", "Unknown");
  }

  protected String _getRevision() {
    return info.getProperty("revision", "Unknown");
  }

  protected String _getBranch() {
    return info.getProperty("branch", "Unknown");
  }

  protected String _getDate() {
    return info.getProperty("date", "Unknown");
  }

  protected String _getUser() {
    return info.getProperty("user", "Unknown");
  }

  protected String _getUrl() {
    return info.getProperty("url", "Unknown");
  }

  protected String _getSrcChecksum() {
    return info.getProperty("srcChecksum", "Unknown");
  }

  protected String _getBuildVersion() {
    return getVersion()
        + " from "
        + _getRevision()
        + " by "
        + _getUser()
        + " source checksum "
        + _getSrcChecksum();
  }

  protected String _getProtocVersion() {
    return info.getProperty("protocVersion", "Unknown");
  }

  private static VersionInfo COMMON_VERSION_INFO = new VersionInfo("common");
  /**
   * Get the Hadoop version.
   *
   * @return the Hadoop version string, eg. "0.6.3-dev"
   */
  public static String getVersion() {
    return COMMON_VERSION_INFO._getVersion();
  }

  /**
   * Get the subversion revision number for the root directory
   *
   * @return the revision number, eg. "451451"
   */
  public static String getRevision() {
    return COMMON_VERSION_INFO._getRevision();
  }

  /**
   * Get the branch on which this originated.
   *
   * @return The branch name, e.g. "trunk" or "branches/branch-0.20"
   */
  public static String getBranch() {
    return COMMON_VERSION_INFO._getBranch();
  }

  /**
   * The date that Hadoop was compiled.
   *
   * @return the compilation date in unix date format
   */
  public static String getDate() {
    return COMMON_VERSION_INFO._getDate();
  }

  /**
   * The user that compiled Hadoop.
   *
   * @return the username of the user
   */
  public static String getUser() {
    return COMMON_VERSION_INFO._getUser();
  }

  /** Get the subversion URL for the root Hadoop directory. */
  public static String getUrl() {
    return COMMON_VERSION_INFO._getUrl();
  }

  /** Get the checksum of the source files from which Hadoop was built. */
  public static String getSrcChecksum() {
    return COMMON_VERSION_INFO._getSrcChecksum();
  }

  /** Returns the buildVersion which includes version, revision, user and date. */
  public static String getBuildVersion() {
    return COMMON_VERSION_INFO._getBuildVersion();
  }

  /** Returns the protoc version used for the build. */
  public static String getProtocVersion() {
    return COMMON_VERSION_INFO._getProtocVersion();
  }

  public static void main(String[] args) {
    /* LOG.debug("version: "+getVersion()) */
    LOG.version(getVersion()).tag("methodCall").debug();
    System.out.println("Hadoop " + getVersion());
    System.out.println("Subversion " + getUrl() + " -r " + getRevision());
    System.out.println("Compiled by " + getUser() + " on " + getDate());
    System.out.println("Compiled with protoc " + getProtocVersion());
    System.out.println("From source with checksum " + getSrcChecksum());
    System.out.println(
        "This command was run using " + ClassUtil.findContainingJar(VersionInfo.class));
  }
}
Ejemplo n.º 5
0
/** A class for file/directory permissions. */
@InterfaceAudience.Public
@InterfaceStability.Stable
public class FsPermission implements Writable {
  private static final /* LogLOG=LogFactory.getLog(FsPermission.class); */ FsNamespace LOG =
      LoggerFactory.getLogger(FsNamespace.class, new SimpleLogger());

  static final WritableFactory FACTORY =
      new WritableFactory() {
        @Override
        public Writable newInstance() {
          return new FsPermission();
        }
      };

  static { // register a ctor
    WritableFactories.setFactory(FsPermission.class, FACTORY);
    WritableFactories.setFactory(ImmutableFsPermission.class, FACTORY);
  }

  /** Maximum acceptable length of a permission string to parse */
  public static final int MAX_PERMISSION_LENGTH = 10;

  /** Create an immutable {@link FsPermission} object. */
  public static FsPermission createImmutable(short permission) {
    return new ImmutableFsPermission(permission);
  }

  // POSIX permission style
  private FsAction useraction = null;
  private FsAction groupaction = null;
  private FsAction otheraction = null;
  private boolean stickyBit = false;

  private FsPermission() {}

  /**
   * Construct by the given {@link FsAction}.
   *
   * @param u user action
   * @param g group action
   * @param o other action
   */
  public FsPermission(FsAction u, FsAction g, FsAction o) {
    this(u, g, o, false);
  }

  public FsPermission(FsAction u, FsAction g, FsAction o, boolean sb) {
    set(u, g, o, sb);
  }

  /**
   * Construct by the given mode.
   *
   * @param mode
   * @see #toShort()
   */
  public FsPermission(short mode) {
    fromShort(mode);
  }

  /**
   * Copy constructor
   *
   * @param other other permission
   */
  public FsPermission(FsPermission other) {
    this.useraction = other.useraction;
    this.groupaction = other.groupaction;
    this.otheraction = other.otheraction;
    this.stickyBit = other.stickyBit;
  }

  /**
   * Construct by given mode, either in octal or symbolic format.
   *
   * @param mode mode as a string, either in octal or symbolic format
   * @throws IllegalArgumentException if <code>mode</code> is invalid
   */
  public FsPermission(String mode) {
    this(new UmaskParser(mode).getUMask());
  }

  /** Return user {@link FsAction}. */
  public FsAction getUserAction() {
    return useraction;
  }

  /** Return group {@link FsAction}. */
  public FsAction getGroupAction() {
    return groupaction;
  }

  /** Return other {@link FsAction}. */
  public FsAction getOtherAction() {
    return otheraction;
  }

  private void set(FsAction u, FsAction g, FsAction o, boolean sb) {
    useraction = u;
    groupaction = g;
    otheraction = o;
    stickyBit = sb;
  }

  public void fromShort(short n) {
    FsAction[] v = FSACTION_VALUES;
    set(v[(n >>> 6) & 7], v[(n >>> 3) & 7], v[n & 7], (((n >>> 9) & 1) == 1));
  }

  @Override
  public void write(DataOutput out) throws IOException {
    out.writeShort(toShort());
  }

  @Override
  public void readFields(DataInput in) throws IOException {
    fromShort(in.readShort());
  }

  /** Create and initialize a {@link FsPermission} from {@link DataInput}. */
  public static FsPermission read(DataInput in) throws IOException {
    FsPermission p = new FsPermission();
    p.readFields(in);
    return p;
  }

  /** Encode the object to a short. */
  public short toShort() {
    int s =
        (stickyBit ? 1 << 9 : 0)
            | (useraction.ordinal() << 6)
            | (groupaction.ordinal() << 3)
            | otheraction.ordinal();

    return (short) s;
  }

  @Override
  public boolean equals(Object obj) {
    if (obj instanceof FsPermission) {
      FsPermission that = (FsPermission) obj;
      return this.useraction == that.useraction
          && this.groupaction == that.groupaction
          && this.otheraction == that.otheraction
          && this.stickyBit == that.stickyBit;
    }
    return false;
  }

  @Override
  public int hashCode() {
    return toShort();
  }

  @Override
  public String toString() {
    String str = useraction.SYMBOL + groupaction.SYMBOL + otheraction.SYMBOL;
    if (stickyBit) {
      StringBuilder str2 = new StringBuilder(str);
      str2.replace(
          str2.length() - 1, str2.length(), otheraction.implies(FsAction.EXECUTE) ? "t" : "T");
      str = str2.toString();
    }

    return str;
  }

  /**
   * Apply a umask to this permission and return a new one.
   *
   * <p>The umask is used by create, mkdir, and other Hadoop filesystem operations. The mode
   * argument for these operations is modified by removing the bits which are set in the umask.
   * Thus, the umask limits the permissions which newly created files and directories get.
   *
   * @param umask The umask to use
   * @return The effective permission
   */
  public FsPermission applyUMask(FsPermission umask) {
    return new FsPermission(
        useraction.and(umask.useraction.not()),
        groupaction.and(umask.groupaction.not()),
        otheraction.and(umask.otheraction.not()));
  }

  /**
   * umask property label deprecated key and code in getUMask method to accommodate it may be
   * removed in version .23
   */
  public static final String DEPRECATED_UMASK_LABEL = "dfs.umask";

  public static final String UMASK_LABEL = CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY;
  public static final int DEFAULT_UMASK = CommonConfigurationKeys.FS_PERMISSIONS_UMASK_DEFAULT;

  private static final FsAction[] FSACTION_VALUES = FsAction.values();

  /**
   * Get the user file creation mask (umask)
   *
   * <p>{@code UMASK_LABEL} config param has umask value that is either symbolic or octal.
   *
   * <p>Symbolic umask is applied relative to file mode creation mask; the permission op characters
   * '+' clears the corresponding bit in the mask, '-' sets bits in the mask.
   *
   * <p>Octal umask, the specified bits are set in the file mode creation mask.
   *
   * <p>{@code DEPRECATED_UMASK_LABEL} config param has umask value set to decimal.
   */
  public static FsPermission getUMask(Configuration conf) {
    int umask = DEFAULT_UMASK;

    // To ensure backward compatibility first use the deprecated key.
    // If the deprecated key is not present then check for the new key
    if (conf != null) {
      String confUmask = conf.get(UMASK_LABEL);
      int oldUmask = conf.getInt(DEPRECATED_UMASK_LABEL, Integer.MIN_VALUE);
      try {
        if (confUmask != null) {
          umask = new UmaskParser(confUmask).getUMask();
        }
      } catch (IllegalArgumentException iae) {
        // Provide more explanation for user-facing message
        String type = iae instanceof NumberFormatException ? "decimal" : "octal or symbolic";
        String error =
            "Unable to parse configuration "
                + UMASK_LABEL
                + " with value "
                + confUmask
                + " as "
                + type
                + " umask.";
        /* LOG.warn(error) */
        LOG.error(error).warn();

        // If oldUmask is not set, then throw the exception
        if (oldUmask == Integer.MIN_VALUE) {
          throw new IllegalArgumentException(error);
        }
      }

      if (oldUmask != Integer.MIN_VALUE) { // Property was set with old key
        if (umask != oldUmask) {
          /* LOG.warn(DEPRECATED_UMASK_LABEL+" configuration key is deprecated. "+"Convert to "+UMASK_LABEL+", using octal or symbolic umask "+"specifications.") */
          LOG.configuration_key_deprecated_convert_usi(DEPRECATED_UMASK_LABEL, UMASK_LABEL).warn();
          // Old and new umask values do not match - Use old umask
          umask = oldUmask;
        }
      }
    }

    return new FsPermission((short) umask);
  }

  public boolean getStickyBit() {
    return stickyBit;
  }

  /** Set the user file creation mask (umask) */
  public static void setUMask(Configuration conf, FsPermission umask) {
    conf.set(UMASK_LABEL, String.format("%1$03o", umask.toShort()));
    conf.setInt(DEPRECATED_UMASK_LABEL, umask.toShort());
  }

  /**
   * Get the default permission for directory and symlink. In previous versions, this default
   * permission was also used to create files, so files created end up with ugo+x permission. See
   * HADOOP-9155 for detail. Two new methods are added to solve this, please use {@link
   * FsPermission#getDirDefault()} for directory, and use {@link FsPermission#getFileDefault()} for
   * file. This method is kept for compatibility.
   */
  public static FsPermission getDefault() {
    return new FsPermission((short) 00777);
  }

  /** Get the default permission for directory. */
  public static FsPermission getDirDefault() {
    return new FsPermission((short) 00777);
  }

  /** Get the default permission for file. */
  public static FsPermission getFileDefault() {
    return new FsPermission((short) 00666);
  }

  /** Get the default permission for cache pools. */
  public static FsPermission getCachePoolDefault() {
    return new FsPermission((short) 00755);
  }

  /**
   * Create a FsPermission from a Unix symbolic permission string
   *
   * @param unixSymbolicPermission e.g. "-rw-rw-rw-"
   */
  public static FsPermission valueOf(String unixSymbolicPermission) {
    if (unixSymbolicPermission == null) {
      return null;
    } else if (unixSymbolicPermission.length() != MAX_PERMISSION_LENGTH) {
      throw new IllegalArgumentException(
          String.format(
              "length != %d(unixSymbolicPermission=%s)",
              MAX_PERMISSION_LENGTH, unixSymbolicPermission));
    }

    int n = 0;
    for (int i = 1; i < unixSymbolicPermission.length(); i++) {
      n = n << 1;
      char c = unixSymbolicPermission.charAt(i);
      n += (c == '-' || c == 'T' || c == 'S') ? 0 : 1;
    }

    // Add sticky bit value if set
    if (unixSymbolicPermission.charAt(9) == 't' || unixSymbolicPermission.charAt(9) == 'T')
      n += 01000;

    return new FsPermission((short) n);
  }

  private static class ImmutableFsPermission extends FsPermission {
    public ImmutableFsPermission(short permission) {
      super(permission);
    }

    @Override
    public FsPermission applyUMask(FsPermission umask) {
      throw new UnsupportedOperationException();
    }

    @Override
    public void readFields(DataInput in) throws IOException {
      throw new UnsupportedOperationException();
    }
  }
}
/** This class provides a way to interact with history files in a thread safe manor. */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class HistoryFileManager extends AbstractService {
  //  private static final /* LogLOG=LogFactory.getLog(HistoryFileManager.class); */
  //			MapreduceNamespace LOG = LoggerFactory.getLogger(MapreduceNamespace.class, new
  // SimpleLogger());
  private static final /* LogSUMMARY_LOG=LogFactory.getLog(JobSummary.class); */ MapreduceNamespace
      LOG = LoggerFactory.getLogger(MapreduceNamespace.class, new SimpleLogger());

  private static enum HistoryInfoState {
    IN_INTERMEDIATE,
    IN_DONE,
    DELETED,
    MOVE_FAILED
  };

  private static String DONE_BEFORE_SERIAL_TAIL = JobHistoryUtils.doneSubdirsBeforeSerialTail();

  /**
   * Maps between a serial number (generated based on jobId) and the timestamp component(s) to which
   * it belongs. Facilitates jobId based searches. If a jobId is not found in this list - it will
   * not be found.
   */
  private static class SerialNumberIndex {
    private SortedMap<String, Set<String>> cache;
    private int maxSize;

    public SerialNumberIndex(int maxSize) {
      this.cache = new TreeMap<String, Set<String>>();
      this.maxSize = maxSize;
    }

    public synchronized void add(String serialPart, String timestampPart) {
      if (!cache.containsKey(serialPart)) {
        cache.put(serialPart, new HashSet<String>());
        if (cache.size() > maxSize) {
          String key = cache.firstKey();
          /* LOG.error("Dropping "+key+" from the SerialNumberIndex. We will no "+"longer be able to see jobs that are in that serial index for "+cache.get(key)) */
          LOG.dropping_from_serialnumberindex_will_lon(key, String.valueOf(cache.get(key)))
              .tag("methodCall")
              .error();
          cache.remove(key);
        }
      }
      Set<String> datePartSet = cache.get(serialPart);
      datePartSet.add(timestampPart);
    }

    public synchronized void remove(String serialPart, String timeStampPart) {
      if (cache.containsKey(serialPart)) {
        Set<String> set = cache.get(serialPart);
        set.remove(timeStampPart);
        if (set.isEmpty()) {
          cache.remove(serialPart);
        }
      }
    }

    public synchronized Set<String> get(String serialPart) {
      Set<String> found = cache.get(serialPart);
      if (found != null) {
        return new HashSet<String>(found);
      }
      return null;
    }
  }

  /**
   * Wrapper around {@link ConcurrentSkipListMap} that maintains size along side for O(1) size()
   * implementation for use in JobListCache.
   *
   * <p>Note: The size is not updated atomically with changes additions/removals. This race can lead
   * to size() returning an incorrect size at times.
   */
  static class JobIdHistoryFileInfoMap {
    private ConcurrentSkipListMap<JobId, HistoryFileInfo> cache;
    private AtomicInteger mapSize;

    JobIdHistoryFileInfoMap() {
      cache = new ConcurrentSkipListMap<JobId, HistoryFileInfo>();
      mapSize = new AtomicInteger();
    }

    public HistoryFileInfo putIfAbsent(JobId key, HistoryFileInfo value) {
      HistoryFileInfo ret = cache.putIfAbsent(key, value);
      if (ret == null) {
        mapSize.incrementAndGet();
      }
      return ret;
    }

    public HistoryFileInfo remove(JobId key) {
      HistoryFileInfo ret = cache.remove(key);
      if (ret != null) {
        mapSize.decrementAndGet();
      }
      return ret;
    }

    /**
     * Returns the recorded size of the internal map. Note that this could be out of sync with the
     * actual size of the map
     *
     * @return "recorded" size
     */
    public int size() {
      return mapSize.get();
    }

    public HistoryFileInfo get(JobId key) {
      return cache.get(key);
    }

    public NavigableSet<JobId> navigableKeySet() {
      return cache.navigableKeySet();
    }

    public Collection<HistoryFileInfo> values() {
      return cache.values();
    }
  }

  static class JobListCache {
    private JobIdHistoryFileInfoMap cache;
    private int maxSize;
    private long maxAge;

    public JobListCache(int maxSize, long maxAge) {
      this.maxSize = maxSize;
      this.maxAge = maxAge;
      this.cache = new JobIdHistoryFileInfoMap();
    }

    public HistoryFileInfo addIfAbsent(HistoryFileInfo fileInfo) {
      JobId jobId = fileInfo.getJobId();
      if (LogGlobal.isDebugEnabled()) {
        /* LOG.debug("Adding "+jobId+" to job list cache with "+fileInfo.getJobIndexInfo()) */
        LOG.adding_job_list_cache_with(jobId.toString(), String.valueOf(fileInfo.getJobIndexInfo()))
            .tag("methodCall")
            .debug();
      }
      HistoryFileInfo old = cache.putIfAbsent(jobId, fileInfo);
      if (cache.size() > maxSize) {
        // There is a race here, where more then one thread could be trying to
        // remove entries.  This could result in too many entries being removed
        // from the cache.  This is considered OK as the size of the cache
        // should be rather large, and we would rather have performance over
        // keeping the cache size exactly at the maximum.
        Iterator<JobId> keys = cache.navigableKeySet().iterator();
        long cutoff = System.currentTimeMillis() - maxAge;
        while (cache.size() > maxSize && keys.hasNext()) {
          JobId key = keys.next();
          HistoryFileInfo firstValue = cache.get(key);
          if (firstValue != null) {
            synchronized (firstValue) {
              if (firstValue.isMovePending()) {
                if (firstValue.didMoveFail() && firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
                  cache.remove(key);
                  // Now lets try to delete it
                  try {
                    firstValue.delete();
                  } catch (IOException e) {
                    /* LOG.error("Error while trying to delete history files"+" that could not be moved to done.",e) */
                    LOG.error_while_trying_delete_history_files_(e.toString()).error();
                  }
                } else {
                  /* LOG.warn("Waiting to remove "+key+" from JobListCache because it is not in done yet.") */
                  LOG.waiting_remove_from_joblistcache_because(key.toString()).warn();
                }
              } else {
                cache.remove(key);
              }
            }
          }
        }
      }
      return old;
    }

    public void delete(HistoryFileInfo fileInfo) {
      if (LogGlobal.isDebugEnabled()) {
        /* LOG.debug("Removing from cache "+fileInfo) */
        LOG.removing_from_cache(fileInfo.toString()).debug();
      }
      cache.remove(fileInfo.getJobId());
    }

    public Collection<HistoryFileInfo> values() {
      return new ArrayList<HistoryFileInfo>(cache.values());
    }

    public HistoryFileInfo get(JobId jobId) {
      return cache.get(jobId);
    }
  }

  /**
   * This class represents a user dir in the intermediate done directory. This is mostly for locking
   * purposes.
   */
  private class UserLogDir {
    long modTime = 0;

    public synchronized void scanIfNeeded(FileStatus fs) {
      long newModTime = fs.getModificationTime();
      if (modTime != newModTime) {
        Path p = fs.getPath();
        try {
          scanIntermediateDirectory(p);
          // If scanning fails, we will scan again.  We assume the failure is
          // temporary.
          modTime = newModTime;
        } catch (IOException e) {
          /* LOG.error("Error while trying to scan the directory "+p,e) */
          LOG.error_while_trying_scan_directory(p.toString(), e.toString()).error();
        }
      } else {
        if (LogGlobal.isDebugEnabled()) {
          /* LOG.debug("Scan not needed of "+fs.getPath()) */
          LOG.scan_not_needed(String.valueOf(fs.getPath())).tag("methodCall").debug();
        }
      }
    }
  }

  public class HistoryFileInfo {
    private Path historyFile;
    private Path confFile;
    private Path summaryFile;
    private JobIndexInfo jobIndexInfo;
    private HistoryInfoState state;

    private HistoryFileInfo(
        Path historyFile,
        Path confFile,
        Path summaryFile,
        JobIndexInfo jobIndexInfo,
        boolean isInDone) {
      this.historyFile = historyFile;
      this.confFile = confFile;
      this.summaryFile = summaryFile;
      this.jobIndexInfo = jobIndexInfo;
      state = isInDone ? HistoryInfoState.IN_DONE : HistoryInfoState.IN_INTERMEDIATE;
    }

    @VisibleForTesting
    synchronized boolean isMovePending() {
      return state == HistoryInfoState.IN_INTERMEDIATE || state == HistoryInfoState.MOVE_FAILED;
    }

    @VisibleForTesting
    synchronized boolean didMoveFail() {
      return state == HistoryInfoState.MOVE_FAILED;
    }

    /** @return true if the files backed by this were deleted. */
    public synchronized boolean isDeleted() {
      return state == HistoryInfoState.DELETED;
    }

    @Override
    public String toString() {
      return "HistoryFileInfo jobID " + getJobId() + " historyFile = " + historyFile;
    }

    private synchronized void moveToDone() throws IOException {
      if (LogGlobal.isDebugEnabled()) {
        /* LOG.debug("moveToDone: "+historyFile) */
        LOG.movetodone(historyFile.toString()).debug();
      }
      if (!isMovePending()) {
        // It was either deleted or is already in done. Either way do nothing
        if (LogGlobal.isDebugEnabled()) {
          /* LOG.debug("Move no longer pending") */
          LOG.move_longer_pending().debug();
        }
        return;
      }
      try {
        long completeTime = jobIndexInfo.getFinishTime();
        if (completeTime == 0) {
          completeTime = System.currentTimeMillis();
        }
        JobId jobId = jobIndexInfo.getJobId();

        List<Path> paths = new ArrayList<Path>(2);
        if (historyFile == null) {
          /* LOG.info("No file for job-history with "+jobId+" found in cache!") */
          LOG.file_for_job_history_with_found(jobId.toString()).info();
        } else {
          paths.add(historyFile);
        }

        if (confFile == null) {
          /* LOG.info("No file for jobConf with "+jobId+" found in cache!") */
          LOG.file_for_jobconf_with_found_cache(jobId.toString()).info();
        } else {
          paths.add(confFile);
        }

        if (summaryFile == null) {
          /* LOG.info("No summary file for job: "+jobId) */
          LOG.summary_file_for_job(jobId.toString()).info();
        } else {
          String jobSummaryString = getJobSummary(intermediateDoneDirFc, summaryFile);
          LOG.summary_file_for_job(jobSummaryString);
          /* LOG.info("Deleting JobSummary file: ["+summaryFile+"]") */
          LOG.deleting_jobsummary_file(summaryFile.toString()).info();
          intermediateDoneDirFc.delete(summaryFile, false);
          summaryFile = null;
        }

        Path targetDir = canonicalHistoryLogPath(jobId, completeTime);
        addDirectoryToSerialNumberIndex(targetDir);
        makeDoneSubdir(targetDir);
        if (historyFile != null) {
          Path toPath = doneDirFc.makeQualified(new Path(targetDir, historyFile.getName()));
          if (!toPath.equals(historyFile)) {
            moveToDoneNow(historyFile, toPath);
            historyFile = toPath;
          }
        }
        if (confFile != null) {
          Path toPath = doneDirFc.makeQualified(new Path(targetDir, confFile.getName()));
          if (!toPath.equals(confFile)) {
            moveToDoneNow(confFile, toPath);
            confFile = toPath;
          }
        }
        state = HistoryInfoState.IN_DONE;
      } catch (Throwable t) {
        /* LOG.error("Error while trying to move a job to done",t) */
        LOG.error_while_trying_move_job_done(t.toString()).error();
        this.state = HistoryInfoState.MOVE_FAILED;
      }
    }

    /**
     * Parse a job from the JobHistoryFile, if the underlying file is not going to be deleted.
     *
     * @return the Job or null if the underlying file was deleted.
     * @throws IOException if there is an error trying to read the file.
     */
    public synchronized Job loadJob() throws IOException {
      return new CompletedJob(
          conf, jobIndexInfo.getJobId(), historyFile, false, jobIndexInfo.getUser(), this, aclsMgr);
    }

    /**
     * Return the history file. This should only be used for testing.
     *
     * @return the history file.
     */
    synchronized Path getHistoryFile() {
      return historyFile;
    }

    protected synchronized void delete() throws IOException {
      if (LogGlobal.isDebugEnabled()) {
        /* LOG.debug("deleting "+historyFile+" and "+confFile) */
        LOG.deleting_and(historyFile.toString(), confFile.toString()).debug();
      }
      state = HistoryInfoState.DELETED;
      doneDirFc.delete(doneDirFc.makeQualified(historyFile), false);
      doneDirFc.delete(doneDirFc.makeQualified(confFile), false);
    }

    public JobIndexInfo getJobIndexInfo() {
      return jobIndexInfo;
    }

    public JobId getJobId() {
      return jobIndexInfo.getJobId();
    }

    public synchronized Path getConfFile() {
      return confFile;
    }

    public synchronized Configuration loadConfFile() throws IOException {
      FileContext fc = FileContext.getFileContext(confFile.toUri(), conf);
      Configuration jobConf = new Configuration(false);
      jobConf.addResource(fc.open(confFile), confFile.toString());
      return jobConf;
    }
  }

  private SerialNumberIndex serialNumberIndex = null;
  protected JobListCache jobListCache = null;

  // Maintains a list of known done subdirectories.
  private final Set<Path> existingDoneSubdirs = Collections.synchronizedSet(new HashSet<Path>());

  /**
   * Maintains a mapping between intermediate user directories and the last known modification time.
   */
  private ConcurrentMap<String, UserLogDir> userDirModificationTimeMap =
      new ConcurrentHashMap<String, UserLogDir>();

  private JobACLsManager aclsMgr;

  @VisibleForTesting Configuration conf;

  private String serialNumberFormat;

  private Path doneDirPrefixPath = null; // folder for completed jobs
  private FileContext doneDirFc; // done Dir FileContext

  private Path intermediateDoneDirPath = null; // Intermediate Done Dir Path
  private FileContext intermediateDoneDirFc; // Intermediate Done Dir
  // FileContext
  @VisibleForTesting protected ThreadPoolExecutor moveToDoneExecutor = null;
  private long maxHistoryAge = 0;

  public HistoryFileManager() {
    super(HistoryFileManager.class.getName());
  }

  @Override
  protected void serviceInit(Configuration conf) throws Exception {
    this.conf = conf;

    int serialNumberLowDigits = 3;
    serialNumberFormat =
        ("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");

    long maxFSWaitTime =
        conf.getLong(
            JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
            JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
    createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);

    this.aclsMgr = new JobACLsManager(conf);

    maxHistoryAge =
        conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);

    jobListCache = createJobListCache();

    serialNumberIndex =
        new SerialNumberIndex(
            conf.getInt(
                JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
                JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

    int numMoveThreads =
        conf.getInt(
            JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
            JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
    ThreadFactory tf =
        new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
    moveToDoneExecutor =
        new ThreadPoolExecutor(
            numMoveThreads,
            numMoveThreads,
            1,
            TimeUnit.HOURS,
            new LinkedBlockingQueue<Runnable>(),
            tf);

    super.serviceInit(conf);
  }

  @VisibleForTesting
  void createHistoryDirs(Clock clock, long intervalCheckMillis, long timeOutMillis)
      throws IOException {
    long start = clock.getTime();
    boolean done = false;
    int counter = 0;
    while (!done && ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
      done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
      try {
        Thread.sleep(intervalCheckMillis);
      } catch (InterruptedException ex) {
        throw new YarnRuntimeException(ex);
      }
    }
    if (!done) {
      throw new YarnRuntimeException(
          "Timed out '" + timeOutMillis + "ms' waiting for FileSystem to become available");
    }
  }

  /**
   * DistributedFileSystem returns a RemoteException with a message stating SafeModeException in it.
   * So this is only way to check it is because of being in safe mode.
   */
  private boolean isBecauseSafeMode(Throwable ex) {
    return ex.toString().contains("SafeModeException");
  }

  /**
   * Returns TRUE if the history dirs were created, FALSE if they could not be created because the
   * FileSystem is not reachable or in safe mode and throws and exception otherwise.
   */
  @VisibleForTesting
  boolean tryCreatingHistoryDirs(boolean logWait) throws IOException {
    boolean succeeded = true;
    String doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    try {
      doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
      doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
      doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
      mkdir(
          doneDirFc,
          doneDirPrefixPath,
          new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
    } catch (ConnectException ex) {
      if (logWait) {
        /* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be available") */
        LOG.waiting_for_filesystem_available(
                String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
            .tag("methodCall")
            .info();
      }
      succeeded = false;
    } catch (IOException e) {
      if (isBecauseSafeMode(e)) {
        succeeded = false;
        if (logWait) {
          /* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be out of safe mode") */
          LOG.waiting_for_filesystem_out_safe_mode(
                  String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
              .tag("methodCall")
              .info();
        }
      } else {
        throw new YarnRuntimeException(
            "Error creating done directory: [" + doneDirPrefixPath + "]", e);
      }
    }
    if (succeeded) {
      String intermediateDoneDirPrefix =
          JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
      try {
        intermediateDoneDirPath =
            FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
        intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
        mkdir(
            intermediateDoneDirFc,
            intermediateDoneDirPath,
            new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
      } catch (ConnectException ex) {
        succeeded = false;
        if (logWait) {
          /* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be available") */
          LOG.waiting_for_filesystem_available(
                  String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
              .tag("methodCall")
              .info();
        }
      } catch (IOException e) {
        if (isBecauseSafeMode(e)) {
          succeeded = false;
          if (logWait) {
            /* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be out of safe mode") */
            LOG.waiting_for_filesystem_out_safe_mode(
                    String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
                .tag("methodCall")
                .info();
          }
        } else {
          throw new YarnRuntimeException(
              "Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
        }
      }
    }
    return succeeded;
  }

  @Override
  public void serviceStop() throws Exception {
    ShutdownThreadsHelper.shutdownExecutorService(moveToDoneExecutor);
    super.serviceStop();
  }

  protected JobListCache createJobListCache() {
    return new JobListCache(
        conf.getInt(
            JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,
            JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE),
        maxHistoryAge);
  }

  private void mkdir(FileContext fc, Path path, FsPermission fsp) throws IOException {
    if (!fc.util().exists(path)) {
      try {
        fc.mkdir(path, fsp, true);

        FileStatus fsStatus = fc.getFileStatus(path);
        /* LOG.info("Perms after creating "+fsStatus.getPermission().toShort()+", Expected: "+fsp.toShort()) */
        LOG.perms_after_creating_expected(
                String.valueOf(fsStatus.getPermission().toShort()), String.valueOf(fsp.toShort()))
            .tag("methodCall")
            .info();
        if (fsStatus.getPermission().toShort() != fsp.toShort()) {
          /* LOG.info("Explicitly setting permissions to : "+fsp.toShort()+", "+fsp) */
          LOG.explicitly_setting_permissions(String.valueOf(fsp.toShort()), fsp.toString())
              .tag("methodCall")
              .info();
          fc.setPermission(path, fsp);
        }
      } catch (FileAlreadyExistsException e) {
        /* LOG.info("Directory: ["+path+"] already exists.") */
        LOG.directory_already_exists(path.toString()).info();
      }
    }
  }

  /** Populates index data structures. Should only be called at initialization times. */
  @SuppressWarnings("unchecked")
  void initExisting() throws IOException {
    /* LOG.info("Initializing Existing Jobs...") */
    LOG.initializing_existing_jobs().info();
    List<FileStatus> timestampedDirList = findTimestampedDirectories();
    // Sort first just so insertion is in a consistent order
    Collections.sort(timestampedDirList);
    for (FileStatus fs : timestampedDirList) {
      // TODO Could verify the correct format for these directories.
      addDirectoryToSerialNumberIndex(fs.getPath());
      addDirectoryToJobListCache(fs.getPath());
    }
  }

  private void removeDirectoryFromSerialNumberIndex(Path serialDirPath) {
    String serialPart = serialDirPath.getName();
    String timeStampPart = JobHistoryUtils.getTimestampPartFromPath(serialDirPath.toString());
    if (timeStampPart == null) {
      /* LOG.warn("Could not find timestamp portion from path: "+serialDirPath.toString()+". Continuing with next") */
      LOG.could_not_find_timestamp_portion_from_co(String.valueOf(serialDirPath.toString()))
          .tag("methodCall")
          .warn();
      return;
    }
    if (serialPart == null) {
      /* LOG.warn("Could not find serial portion from path: "+serialDirPath.toString()+". Continuing with next") */
      LOG.could_not_find_serial_portion_from_conti(String.valueOf(serialDirPath.toString()))
          .tag("methodCall")
          .warn();
      return;
    }
    serialNumberIndex.remove(serialPart, timeStampPart);
  }

  private void addDirectoryToSerialNumberIndex(Path serialDirPath) {
    if (LogGlobal.isDebugEnabled()) {
      /* LOG.debug("Adding "+serialDirPath+" to serial index") */
      LOG.adding_serial_index(serialDirPath.toString()).debug();
    }
    String serialPart = serialDirPath.getName();
    String timestampPart = JobHistoryUtils.getTimestampPartFromPath(serialDirPath.toString());
    if (timestampPart == null) {
      /* LOG.warn("Could not find timestamp portion from path: "+serialDirPath+". Continuing with next") */
      LOG.could_not_find_timestamp_portion_from_co(serialDirPath.toString()).warn();
      return;
    }
    if (serialPart == null) {
      /* LOG.warn("Could not find serial portion from path: "+serialDirPath.toString()+". Continuing with next") */
      LOG.could_not_find_serial_portion_from_conti(String.valueOf(serialDirPath.toString()))
          .tag("methodCall")
          .warn();
    } else {
      serialNumberIndex.add(serialPart, timestampPart);
    }
  }

  private void addDirectoryToJobListCache(Path path) throws IOException {
    if (LogGlobal.isDebugEnabled()) {
      /* LOG.debug("Adding "+path+" to job list cache.") */
      LOG.adding_job_list_cache(path.toString()).debug();
    }
    List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path, doneDirFc);
    for (FileStatus fs : historyFileList) {
      if (LogGlobal.isDebugEnabled()) {
        /* LOG.debug("Adding in history for "+fs.getPath()) */
        LOG.adding_history_for(String.valueOf(fs.getPath())).tag("methodCall").debug();
      }
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
      String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName =
          JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo =
          new HistoryFileInfo(
              fs.getPath(),
              new Path(fs.getPath().getParent(), confFileName),
              new Path(fs.getPath().getParent(), summaryFileName),
              jobIndexInfo,
              true);
      jobListCache.addIfAbsent(fileInfo);
    }
  }

  private static List<FileStatus> scanDirectory(Path path, FileContext fc, PathFilter pathFilter)
      throws IOException {
    path = fc.makeQualified(path);
    List<FileStatus> jhStatusList = new ArrayList<FileStatus>();
    RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path);
    while (fileStatusIter.hasNext()) {
      FileStatus fileStatus = fileStatusIter.next();
      Path filePath = fileStatus.getPath();
      if (fileStatus.isFile() && pathFilter.accept(filePath)) {
        jhStatusList.add(fileStatus);
      }
    }
    return jhStatusList;
  }

  protected List<FileStatus> scanDirectoryForHistoryFiles(Path path, FileContext fc)
      throws IOException {
    return scanDirectory(path, fc, JobHistoryUtils.getHistoryFileFilter());
  }

  /**
   * Finds all history directories with a timestamp component by scanning the filesystem. Used when
   * the JobHistory server is started.
   *
   * @return list of history directories
   */
  protected List<FileStatus> findTimestampedDirectories() throws IOException {
    List<FileStatus> fsList =
        JobHistoryUtils.localGlobber(doneDirFc, doneDirPrefixPath, DONE_BEFORE_SERIAL_TAIL);
    return fsList;
  }

  /**
   * Scans the intermediate directory to find user directories. Scans these for history files if the
   * modification time for the directory has changed. Once it finds history files it starts the
   * process of moving them to the done directory.
   *
   * @throws IOException if there was a error while scanning
   */
  void scanIntermediateDirectory() throws IOException {
    // TODO it would be great to limit how often this happens, except in the
    // case where we are looking for a particular job.
    List<FileStatus> userDirList =
        JobHistoryUtils.localGlobber(intermediateDoneDirFc, intermediateDoneDirPath, "");
    /* LOG.debug("Scanning intermediate dirs") */
    LOG.scanning_intermediate_dirs().debug();
    for (FileStatus userDir : userDirList) {
      String name = userDir.getPath().getName();
      UserLogDir dir = userDirModificationTimeMap.get(name);
      if (dir == null) {
        dir = new UserLogDir();
        UserLogDir old = userDirModificationTimeMap.putIfAbsent(name, dir);
        if (old != null) {
          dir = old;
        }
      }
      dir.scanIfNeeded(userDir);
    }
  }

  /**
   * Scans the specified path and populates the intermediate cache.
   *
   * @param absPath
   * @throws IOException
   */
  private void scanIntermediateDirectory(final Path absPath) throws IOException {
    if (LogGlobal.isDebugEnabled()) {
      /* LOG.debug("Scanning intermediate dir "+absPath) */
      LOG.scanning_intermediate_dir(absPath.toString()).debug();
    }
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(absPath, intermediateDoneDirFc);
    if (LogGlobal.isDebugEnabled()) {
      /* LOG.debug("Found "+fileStatusList.size()+" files") */
      LOG.found_files(String.valueOf(fileStatusList.size())).tag("methodCall").debug();
    }
    for (FileStatus fs : fileStatusList) {
      if (LogGlobal.isDebugEnabled()) {
        /* LOG.debug("scanning file: "+fs.getPath()) */
        LOG.scanning_file(String.valueOf(fs.getPath())).tag("methodCall").debug();
      }
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
      String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName =
          JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo =
          new HistoryFileInfo(
              fs.getPath(),
              new Path(fs.getPath().getParent(), confFileName),
              new Path(fs.getPath().getParent(), summaryFileName),
              jobIndexInfo,
              false);

      final HistoryFileInfo old = jobListCache.addIfAbsent(fileInfo);
      if (old == null || old.didMoveFail()) {
        final HistoryFileInfo found = (old == null) ? fileInfo : old;
        long cutoff = System.currentTimeMillis() - maxHistoryAge;
        if (found.getJobIndexInfo().getFinishTime() <= cutoff) {
          try {
            found.delete();
          } catch (IOException e) {
            /* LOG.warn("Error cleaning up a HistoryFile that is out of date.",e) */
            LOG.error_cleaning_historyfile_that_out_date(e.toString()).warn();
          }
        } else {
          if (LogGlobal.isDebugEnabled()) {
            /* LOG.debug("Scheduling move to done of "+found) */
            LOG.scheduling_move_done(found.toString()).debug();
          }
          moveToDoneExecutor.execute(
              new Runnable() {
                @Override
                public void run() {
                  try {
                    found.moveToDone();
                  } catch (IOException e) {
                    /* LOG.info("Failed to process fileInfo for job: "+found.getJobId(),e) */
                    LOG.failed_process_fileinfo_for_job(
                            String.valueOf(found.getJobId()), e.toString())
                        .tag("methodCall")
                        .info();
                  }
                }
              });
        }
      } else if (old != null && !old.isMovePending()) {
        // This is a duplicate so just delete it
        if (LogGlobal.isDebugEnabled()) {
          /* LOG.debug("Duplicate: deleting") */
          LOG.duplicate_deleting().debug();
        }
        fileInfo.delete();
      }
    }
  }

  /**
   * Searches the job history file FileStatus list for the specified JobId.
   *
   * @param fileStatusList fileStatus list of Job History Files.
   * @param jobId The JobId to find.
   * @return A FileInfo object for the jobId, null if not found.
   * @throws IOException
   */
  private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList, JobId jobId)
      throws IOException {
    for (FileStatus fs : fileStatusList) {
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
      if (jobIndexInfo.getJobId().equals(jobId)) {
        String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
        String summaryFileName =
            JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
        HistoryFileInfo fileInfo =
            new HistoryFileInfo(
                fs.getPath(),
                new Path(fs.getPath().getParent(), confFileName),
                new Path(fs.getPath().getParent(), summaryFileName),
                jobIndexInfo,
                true);
        return fileInfo;
      }
    }
    return null;
  }

  /**
   * Scans old directories known by the idToDateString map for the specified jobId. If the number of
   * directories is higher than the supported size of the idToDateString cache, the jobId will not
   * be found.
   *
   * @param jobId the jobId.
   * @return
   * @throws IOException
   */
  private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
    String boxedSerialNumber =
        JobHistoryUtils.serialNumberDirectoryComponent(jobId, serialNumberFormat);
    Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
    if (dateStringSet == null) {
      return null;
    }
    for (String timestampPart : dateStringSet) {
      Path logDir = canonicalHistoryLogPath(jobId, timestampPart);
      List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(logDir, doneDirFc);
      HistoryFileInfo fileInfo = getJobFileInfo(fileStatusList, jobId);
      if (fileInfo != null) {
        return fileInfo;
      }
    }
    return null;
  }

  public Collection<HistoryFileInfo> getAllFileInfo() throws IOException {
    scanIntermediateDirectory();
    return jobListCache.values();
  }

  public HistoryFileInfo getFileInfo(JobId jobId) throws IOException {
    // FileInfo available in cache.
    HistoryFileInfo fileInfo = jobListCache.get(jobId);
    if (fileInfo != null) {
      return fileInfo;
    }
    // OK so scan the intermediate to be sure we did not lose it that way
    scanIntermediateDirectory();
    fileInfo = jobListCache.get(jobId);
    if (fileInfo != null) {
      return fileInfo;
    }

    // Intermediate directory does not contain job. Search through older ones.
    fileInfo = scanOldDirsForJob(jobId);
    if (fileInfo != null) {
      return fileInfo;
    }
    return null;
  }

  private void moveToDoneNow(final Path src, final Path target) throws IOException {
    /* LOG.info("Moving "+src.toString()+" to "+target.toString()) */
    LOG.moving(String.valueOf(src.toString()), String.valueOf(target.toString()))
        .tag("methodCall")
        .info();
    intermediateDoneDirFc.rename(src, target, Options.Rename.NONE);
  }

  private String getJobSummary(FileContext fc, Path path) throws IOException {
    Path qPath = fc.makeQualified(path);
    FSDataInputStream in = fc.open(qPath);
    String jobSummaryString = in.readUTF();
    in.close();
    return jobSummaryString;
  }

  private void makeDoneSubdir(Path path) throws IOException {
    try {
      doneDirFc.getFileStatus(path);
      existingDoneSubdirs.add(path);
    } catch (FileNotFoundException fnfE) {
      try {
        FsPermission fsp = new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION);
        doneDirFc.mkdir(path, fsp, true);
        FileStatus fsStatus = doneDirFc.getFileStatus(path);
        /* LOG.info("Perms after creating "+fsStatus.getPermission().toShort()+", Expected: "+fsp.toShort()) */
        LOG.perms_after_creating_expected(
                String.valueOf(fsStatus.getPermission().toShort()), String.valueOf(fsp.toShort()))
            .tag("methodCall")
            .info();
        if (fsStatus.getPermission().toShort() != fsp.toShort()) {
          /* LOG.info("Explicitly setting permissions to : "+fsp.toShort()+", "+fsp) */
          LOG.explicitly_setting_permissions(String.valueOf(fsp.toShort()), fsp.toString())
              .tag("methodCall")
              .info();
          doneDirFc.setPermission(path, fsp);
        }
        existingDoneSubdirs.add(path);
      } catch (FileAlreadyExistsException faeE) { // Nothing to do.
      }
    }
  }

  private Path canonicalHistoryLogPath(JobId id, String timestampComponent) {
    return new Path(
        doneDirPrefixPath,
        JobHistoryUtils.historyLogSubdirectory(id, timestampComponent, serialNumberFormat));
  }

  private Path canonicalHistoryLogPath(JobId id, long millisecondTime) {
    String timestampComponent = JobHistoryUtils.timestampDirectoryComponent(millisecondTime);
    return new Path(
        doneDirPrefixPath,
        JobHistoryUtils.historyLogSubdirectory(id, timestampComponent, serialNumberFormat));
  }

  private long getEffectiveTimestamp(long finishTime, FileStatus fileStatus) {
    if (finishTime == 0) {
      return fileStatus.getModificationTime();
    }
    return finishTime;
  }

  private void deleteJobFromDone(HistoryFileInfo fileInfo) throws IOException {
    jobListCache.delete(fileInfo);
    fileInfo.delete();
  }

  List<FileStatus> getHistoryDirsForCleaning(long cutoff) throws IOException {
    return JobHistoryUtils.getHistoryDirsForCleaning(doneDirFc, doneDirPrefixPath, cutoff);
  }

  /**
   * Clean up older history files.
   *
   * @throws IOException on any error trying to remove the entries.
   */
  @SuppressWarnings("unchecked")
  void clean() throws IOException {
    long cutoff = System.currentTimeMillis() - maxHistoryAge;
    boolean halted = false;
    List<FileStatus> serialDirList = getHistoryDirsForCleaning(cutoff);
    // Sort in ascending order. Relies on YYYY/MM/DD/Serial
    Collections.sort(serialDirList);
    for (FileStatus serialDir : serialDirList) {
      List<FileStatus> historyFileList =
          scanDirectoryForHistoryFiles(serialDir.getPath(), doneDirFc);
      for (FileStatus historyFile : historyFileList) {
        JobIndexInfo jobIndexInfo =
            FileNameIndexUtils.getIndexInfo(historyFile.getPath().getName());
        long effectiveTimestamp = getEffectiveTimestamp(jobIndexInfo.getFinishTime(), historyFile);
        if (effectiveTimestamp <= cutoff) {
          HistoryFileInfo fileInfo = this.jobListCache.get(jobIndexInfo.getJobId());
          if (fileInfo == null) {
            String confFileName =
                JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());

            fileInfo =
                new HistoryFileInfo(
                    historyFile.getPath(),
                    new Path(historyFile.getPath().getParent(), confFileName),
                    null,
                    jobIndexInfo,
                    true);
          }
          deleteJobFromDone(fileInfo);
        } else {
          halted = true;
          break;
        }
      }
      if (!halted) {
        deleteDir(serialDir);
        removeDirectoryFromSerialNumberIndex(serialDir.getPath());
        existingDoneSubdirs.remove(serialDir.getPath());
      } else {
        break; // Don't scan any more directories.
      }
    }
  }

  protected boolean deleteDir(FileStatus serialDir)
      throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException,
          IOException {
    return doneDirFc.delete(doneDirFc.makeQualified(serialDir.getPath()), true);
  }

  // for test
  @VisibleForTesting
  void setMaxHistoryAge(long newValue) {
    maxHistoryAge = newValue;
  }
}
Ejemplo n.º 7
0
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class DefaultCodec implements Configurable, CompressionCodec, DirectDecompressionCodec {
  private static final /* LogLOG=LogFactory.getLog(DefaultCodec.class); */ IoNamespace LOG =
      LoggerFactory.getLogger(IoNamespace.class, new SimpleLogger());

  Configuration conf;

  @Override
  public void setConf(Configuration conf) {
    this.conf = conf;
  }

  @Override
  public Configuration getConf() {
    return conf;
  }

  @Override
  public CompressionOutputStream createOutputStream(OutputStream out) throws IOException {
    // This may leak memory if called in a loop. The createCompressor() call
    // may cause allocation of an untracked direct-backed buffer if native
    // libs are being used (even if you close the stream).  A Compressor
    // object should be reused between successive calls.
    /* LOG.warn("DefaultCodec.createOutputStream() may leak memory. "+"Create a compressor first.") */
    LOG.defaultcodec_createoutputstream_may_leak().warn();
    return new CompressorStream(
        out, createCompressor(), conf.getInt("io.file.buffer.size", 4 * 1024));
  }

  @Override
  public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor)
      throws IOException {
    return new CompressorStream(out, compressor, conf.getInt("io.file.buffer.size", 4 * 1024));
  }

  @Override
  public Class<? extends Compressor> getCompressorType() {
    return ZlibFactory.getZlibCompressorType(conf);
  }

  @Override
  public Compressor createCompressor() {
    return ZlibFactory.getZlibCompressor(conf);
  }

  @Override
  public CompressionInputStream createInputStream(InputStream in) throws IOException {
    return new DecompressorStream(
        in, createDecompressor(), conf.getInt("io.file.buffer.size", 4 * 1024));
  }

  @Override
  public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor)
      throws IOException {
    return new DecompressorStream(in, decompressor, conf.getInt("io.file.buffer.size", 4 * 1024));
  }

  @Override
  public Class<? extends Decompressor> getDecompressorType() {
    return ZlibFactory.getZlibDecompressorType(conf);
  }

  @Override
  public Decompressor createDecompressor() {
    return ZlibFactory.getZlibDecompressor(conf);
  }

  /** {@inheritDoc} */
  @Override
  public DirectDecompressor createDirectDecompressor() {
    return ZlibFactory.getZlibDirectDecompressor(conf);
  }

  @Override
  public String getDefaultExtension() {
    return ".deflate";
  }
}
Ejemplo n.º 8
0
/**
 * This class parses the configured list of fencing methods, and is responsible for trying each one
 * in turn while logging informative output.
 *
 * <p>The fencing methods are configured as a carriage-return separated list. Each line in the list
 * is of the form:
 *
 * <p><code>com.example.foo.MyMethod(arg string)</code> or <code>com.example.foo.MyMethod</code> The
 * class provided must implement the {@link FenceMethod} interface. The fencing methods that ship
 * with Hadoop may also be referred to by shortened names:
 *
 * <p>
 *
 * <ul>
 *   <li><code>shell(/path/to/some/script.sh args...)</code>
 *   <li><code>sshfence(...)</code> (see {@link SshFenceByTcpPort})
 * </ul>
 */
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class NodeFencer {
  private static final String CLASS_RE = "([a-zA-Z0-9\\.\\$]+)";
  private static final Pattern CLASS_WITH_ARGUMENT = Pattern.compile(CLASS_RE + "\\((.+?)\\)");
  private static final Pattern CLASS_WITHOUT_ARGUMENT = Pattern.compile(CLASS_RE);
  private static final Pattern HASH_COMMENT_RE = Pattern.compile("#.*$");

  private static final /* LogLOG=LogFactory.getLog(NodeFencer.class); */ HaNamespace LOG =
      LoggerFactory.getLogger(HaNamespace.class, new SimpleLogger());

  /** Standard fencing methods included with Hadoop. */
  private static final Map<String, Class<? extends FenceMethod>> STANDARD_METHODS =
      ImmutableMap.<String, Class<? extends FenceMethod>>of(
          "shell", ShellCommandFencer.class,
          "sshfence", SshFenceByTcpPort.class);

  private final List<FenceMethodWithArg> methods;

  NodeFencer(Configuration conf, String spec) throws BadFencingConfigurationException {
    this.methods = parseMethods(conf, spec);
  }

  public static NodeFencer create(Configuration conf, String confKey)
      throws BadFencingConfigurationException {
    String confStr = conf.get(confKey);
    if (confStr == null) {
      return null;
    }
    return new NodeFencer(conf, confStr);
  }

  public boolean fence(HAServiceTarget fromSvc) {
    /* LOG.info("====== Beginning Service Fencing Process... ======") */
    LOG.beginning_service_fencing_process().info();
    int i = 0;
    for (FenceMethodWithArg method : methods) {
      /* LOG.info("Trying method "+(++i)+"/"+methods.size()+": "+method) */
      LOG.trying_method((++i), String.valueOf(methods.size()), method.toString())
          .tag("mathExp")
          .tag("methodCall")
          .info();

      try {
        if (method.method.tryFence(fromSvc, method.arg)) {
          /* LOG.info("====== Fencing successful by method "+method+" ======") */
          LOG.fencing_successful_method(method.toString()).info();
          return true;
        }
      } catch (BadFencingConfigurationException e) {
        /* LOG.error("Fencing method "+method+" misconfigured",e) */
        LOG.fencing_method_misconfigured(method.toString(), e.toString()).error();
        continue;
      } catch (Throwable t) {
        /* LOG.error("Fencing method "+method+" failed with an unexpected error.",t) */
        LOG.fencing_method_failed_with_unexpected_er(method.toString(), t.toString()).error();
        continue;
      }
      /* LOG.warn("Fencing method "+method+" was unsuccessful.") */
      LOG.fencing_method_was_unsuccessful(method.toString()).warn();
    }

    /* LOG.error("Unable to fence service by any configured method.") */
    LOG.unable_fence_service_any_configured_meth().error();
    return false;
  }

  private static List<FenceMethodWithArg> parseMethods(Configuration conf, String spec)
      throws BadFencingConfigurationException {
    String[] lines = spec.split("\\s*\n\\s*");

    List<FenceMethodWithArg> methods = Lists.newArrayList();
    for (String line : lines) {
      line = HASH_COMMENT_RE.matcher(line).replaceAll("");
      line = line.trim();
      if (!line.isEmpty()) {
        methods.add(parseMethod(conf, line));
      }
    }

    return methods;
  }

  private static FenceMethodWithArg parseMethod(Configuration conf, String line)
      throws BadFencingConfigurationException {
    Matcher m;
    if ((m = CLASS_WITH_ARGUMENT.matcher(line)).matches()) {
      String className = m.group(1);
      String arg = m.group(2);
      return createFenceMethod(conf, className, arg);
    } else if ((m = CLASS_WITHOUT_ARGUMENT.matcher(line)).matches()) {
      String className = m.group(1);
      return createFenceMethod(conf, className, null);
    } else {
      throw new BadFencingConfigurationException("Unable to parse line: '" + line + "'");
    }
  }

  private static FenceMethodWithArg createFenceMethod(
      Configuration conf, String clazzName, String arg) throws BadFencingConfigurationException {

    Class<?> clazz;
    try {
      // See if it's a short name for one of the built-in methods
      clazz = STANDARD_METHODS.get(clazzName);
      if (clazz == null) {
        // Try to instantiate the user's custom method
        clazz = Class.forName(clazzName);
      }
    } catch (Exception e) {
      throw new BadFencingConfigurationException(
          "Could not find configured fencing method " + clazzName, e);
    }

    // Check that it implements the right interface
    if (!FenceMethod.class.isAssignableFrom(clazz)) {
      throw new BadFencingConfigurationException(
          "Class " + clazzName + " does not implement FenceMethod");
    }

    FenceMethod method = (FenceMethod) ReflectionUtils.newInstance(clazz, conf);
    method.checkArgs(arg);
    return new FenceMethodWithArg(method, arg);
  }

  private static class FenceMethodWithArg {
    private final FenceMethod method;
    private final String arg;

    private FenceMethodWithArg(FenceMethod method, String arg) {
      this.method = method;
      this.arg = arg;
    }

    @Override
    public String toString() {
      return method.getClass().getCanonicalName() + "(" + arg + ")";
    }
  }
}