/**
* A {@link GSet} is set, which supports the {@link #get(Object)} operation. The {@link
* #get(Object)} operation uses a key to lookup an element.
*
* <p>Null element is not supported.
*
* @param <K> The type of the keys.
* @param <E> The type of the elements, which must be a subclass of the keys.
*/
@InterfaceAudience.Private
public interface GSet<K, E extends K> extends Iterable<E> {
static final /* LogLOG=LogFactory.getLog(GSet.class); */ UtilNamespace LOG =
LoggerFactory.getLogger(UtilNamespace.class, new SimpleLogger());
/** @return The size of this set. */
int size();
/**
* Does this set contain an element corresponding to the given key?
*
* @param key The given key.
* @return true if the given key equals to a stored element. Otherwise, return false.
* @throws NullPointerException if key == null.
*/
boolean contains(K key);
/**
* Return the stored element which is equal to the given key. This operation is similar to {@link
* java.util.Map#get(Object)}.
*
* @param key The given key.
* @return The stored element if it exists. Otherwise, return null.
* @throws NullPointerException if key == null.
*/
E get(K key);
/**
* Add/replace an element. If the element does not exist, add it to the set. Otherwise, replace
* the existing element.
*
* <p>Note that this operation is similar to {@link java.util.Map#put(Object, Object)} but is
* different from {@link java.util.Set#add(Object)} which does not replace the existing element if
* there is any.
*
* @param element The element being put.
* @return the previous stored element if there is any. Otherwise, return null.
* @throws NullPointerException if element == null.
*/
E put(E element);
/**
* Remove the element corresponding to the given key. This operation is similar to {@link
* java.util.Map#remove(Object)}.
*
* @param key The key of the element being removed.
* @return If such element exists, return it. Otherwise, return null.
* @throws NullPointerException if key == null.
*/
E remove(K key);
void clear();
}
/** Facilitates hooking process termination for tests and debugging. */
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Unstable
public final class ExitUtil {
private static final /* LogLOG=LogFactory.getLog(ExitUtil.class.getName()); */ UtilNamespace LOG =
LoggerFactory.getLogger(UtilNamespace.class, new SimpleLogger());
private static volatile boolean systemExitDisabled = false;
private static volatile boolean systemHaltDisabled = false;
private static volatile ExitException firstExitException;
private static volatile HaltException firstHaltException;
public static class ExitException extends RuntimeException {
private static final long serialVersionUID = 1L;
public final int status;
public ExitException(int status, String msg) {
super(msg);
this.status = status;
}
}
public static class HaltException extends RuntimeException {
private static final long serialVersionUID = 1L;
public final int status;
public HaltException(int status, String msg) {
super(msg);
this.status = status;
}
}
/** Disable the use of System.exit for testing. */
public static void disableSystemExit() {
systemExitDisabled = true;
}
/** Disable the use of {@code Runtime.getRuntime().halt() } for testing. */
public static void disableSystemHalt() {
systemHaltDisabled = true;
}
/** @return true if terminate has been called */
public static boolean terminateCalled() {
// Either we set this member or we actually called System#exit
return firstExitException != null;
}
/** @return true if halt has been called */
public static boolean haltCalled() {
return firstHaltException != null;
}
/** @return the first ExitException thrown, null if none thrown yet */
public static ExitException getFirstExitException() {
return firstExitException;
}
/** @return the first {@code HaltException} thrown, null if none thrown yet */
public static HaltException getFirstHaltException() {
return firstHaltException;
}
/**
* Reset the tracking of process termination. This is for use in unit tests where one test in the
* suite expects an exit but others do not.
*/
public static void resetFirstExitException() {
firstExitException = null;
}
public static void resetFirstHaltException() {
firstHaltException = null;
}
/**
* Terminate the current process. Note that terminate is the *only* method that should be used to
* terminate the daemon processes.
*
* @param status exit code
* @param msg message used to create the {@code ExitException}
* @throws ExitException if System.exit is disabled for test purposes
*/
public static void terminate(int status, String msg) throws ExitException {
/* LOG.info("Exiting with status "+status) */
LOG.exiting_with_status(status).info();
if (systemExitDisabled) {
ExitException ee = new ExitException(status, msg);
/* LOG.fatal("Terminate called",ee) */
LOG.terminate_called(ee.toString()).fatal();
if (null == firstExitException) {
firstExitException = ee;
}
throw ee;
}
System.exit(status);
}
/**
* Forcibly terminates the currently running Java virtual machine.
*
* @param status exit code
* @param msg message used to create the {@code HaltException}
* @throws HaltException if Runtime.getRuntime().halt() is disabled for test purposes
*/
public static void halt(int status, String msg) throws HaltException {
/* LOG.info("Halt with status "+status+" Message: "+msg) */
LOG.halt_with_status_message(status, msg).info();
if (systemHaltDisabled) {
HaltException ee = new HaltException(status, msg);
/* LOG.fatal("Halt called",ee) */
LOG.halt_called(ee.toString()).fatal();
if (null == firstHaltException) {
firstHaltException = ee;
}
throw ee;
}
Runtime.getRuntime().halt(status);
}
/**
* Like {@link terminate(int, String)} but uses the given throwable to initialize the
* ExitException.
*
* @param status
* @param t throwable used to create the ExitException
* @throws ExitException if System.exit is disabled for test purposes
*/
public static void terminate(int status, Throwable t) throws ExitException {
terminate(status, StringUtils.stringifyException(t));
}
/**
* Forcibly terminates the currently running Java virtual machine.
*
* @param status
* @param t
* @throws ExitException
*/
public static void halt(int status, Throwable t) throws HaltException {
halt(status, StringUtils.stringifyException(t));
}
/**
* Like {@link terminate(int, String)} without a message.
*
* @param status
* @throws ExitException if System.exit is disabled for test purposes
*/
public static void terminate(int status) throws ExitException {
terminate(status, "ExitException");
}
/**
* Forcibly terminates the currently running Java virtual machine.
*
* @param status
* @throws ExitException
*/
public static void halt(int status) throws HaltException {
halt(status, "HaltException");
}
}
/**
* A base class for file-based {@link InputFormat}s.
*
* <p><code>FileInputFormat</code> is the base class for all file-based <code>InputFormat</code>s.
* This provides a generic implementation of {@link #getSplits(JobContext)}. Subclasses of <code>
* FileInputFormat</code> can also override the {@link #isSplitable(JobContext, Path)} method to
* ensure input-files are not split-up and are processed as a whole by {@link Mapper}s.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
public static final String INPUT_DIR = "mapreduce.input.fileinputformat.inputdir";
public static final String SPLIT_MAXSIZE = "mapreduce.input.fileinputformat.split.maxsize";
public static final String SPLIT_MINSIZE = "mapreduce.input.fileinputformat.split.minsize";
public static final String PATHFILTER_CLASS = "mapreduce.input.pathFilter.class";
public static final String NUM_INPUT_FILES = "mapreduce.input.fileinputformat.numinputfiles";
public static final String INPUT_DIR_RECURSIVE =
"mapreduce.input.fileinputformat.input.dir.recursive";
public static final String LIST_STATUS_NUM_THREADS =
"mapreduce.input.fileinputformat.list-status.num-threads";
public static final int DEFAULT_LIST_STATUS_NUM_THREADS = 1;
private static final /* LogLOG=LogFactory.getLog(FileInputFormat.class); */ MapreduceNamespace
LOG = LoggerFactory.getLogger(MapreduceNamespace.class, new SimpleLogger());
private static final double SPLIT_SLOP = 1.1; // 10% slop
@Deprecated
public static enum Counter {
BYTES_READ
}
private static final PathFilter hiddenFileFilter =
new PathFilter() {
public boolean accept(Path p) {
String name = p.getName();
return !name.startsWith("_") && !name.startsWith(".");
}
};
/**
* Proxy PathFilter that accepts a path only if all filters given in the constructor do. Used by
* the listPaths() to apply the built-in hiddenFileFilter together with a user provided one (if
* any).
*/
private static class MultiPathFilter implements PathFilter {
private List<PathFilter> filters;
public MultiPathFilter(List<PathFilter> filters) {
this.filters = filters;
}
public boolean accept(Path path) {
for (PathFilter filter : filters) {
if (!filter.accept(path)) {
return false;
}
}
return true;
}
}
/**
* @param job the job to modify
* @param inputDirRecursive
*/
public static void setInputDirRecursive(Job job, boolean inputDirRecursive) {
job.getConfiguration().setBoolean(INPUT_DIR_RECURSIVE, inputDirRecursive);
}
/**
* @param job the job to look at.
* @return should the files to be read recursively?
*/
public static boolean getInputDirRecursive(JobContext job) {
return job.getConfiguration().getBoolean(INPUT_DIR_RECURSIVE, false);
}
/**
* Get the lower bound on split size imposed by the format.
*
* @return the number of bytes of the minimal split for this format
*/
protected long getFormatMinSplitSize() {
return 1;
}
/**
* Is the given filename splitable? Usually, true, but if the file is stream compressed, it will
* not be. <code>FileInputFormat</code> implementations can override this and return <code>false
* </code> to ensure that individual input files are never split-up so that {@link Mapper}s
* process entire files.
*
* @param context the job context
* @param filename the file name to check
* @return is this file splitable?
*/
protected boolean isSplitable(JobContext context, Path filename) {
return true;
}
/**
* Set a PathFilter to be applied to the input paths for the map-reduce job.
*
* @param job the job to modify
* @param filter the PathFilter class use for filtering the input paths.
*/
public static void setInputPathFilter(Job job, Class<? extends PathFilter> filter) {
job.getConfiguration().setClass(PATHFILTER_CLASS, filter, PathFilter.class);
}
/**
* Set the minimum input split size
*
* @param job the job to modify
* @param size the minimum size
*/
public static void setMinInputSplitSize(Job job, long size) {
job.getConfiguration().setLong(SPLIT_MINSIZE, size);
}
/**
* Get the minimum split size
*
* @param job the job
* @return the minimum number of bytes that can be in a split
*/
public static long getMinSplitSize(JobContext job) {
return job.getConfiguration().getLong(SPLIT_MINSIZE, 1L);
}
/**
* Set the maximum split size
*
* @param job the job to modify
* @param size the maximum split size
*/
public static void setMaxInputSplitSize(Job job, long size) {
job.getConfiguration().setLong(SPLIT_MAXSIZE, size);
}
/**
* Get the maximum split size.
*
* @param context the job to look at.
* @return the maximum number of bytes a split can include
*/
public static long getMaxSplitSize(JobContext context) {
return context.getConfiguration().getLong(SPLIT_MAXSIZE, Long.MAX_VALUE);
}
/**
* Get a PathFilter instance of the filter set for the input paths.
*
* @return the PathFilter instance set for the job, NULL if none has been set.
*/
public static PathFilter getInputPathFilter(JobContext context) {
Configuration conf = context.getConfiguration();
Class<?> filterClass = conf.getClass(PATHFILTER_CLASS, null, PathFilter.class);
return (filterClass != null)
? (PathFilter) ReflectionUtils.newInstance(filterClass, conf)
: null;
}
/**
* List input directories. Subclasses may override to, e.g., select only files matching a regular
* expression.
*
* @param job the job to list input paths for
* @return array of FileStatus objects
* @throws IOException if zero items.
*/
protected List<FileStatus> listStatus(JobContext job) throws IOException {
Path[] dirs = getInputPaths(job);
if (dirs.length == 0) {
throw new IOException("No input paths specified in job");
}
// get tokens for all the required FileSystems..
TokenCache.obtainTokensForNamenodes(job.getCredentials(), dirs, job.getConfiguration());
// Whether we need to recursive look into the directory structure
boolean recursive = getInputDirRecursive(job);
// creates a MultiPathFilter with the hiddenFileFilter and the
// user provided one (if any).
List<PathFilter> filters = new ArrayList<PathFilter>();
filters.add(hiddenFileFilter);
PathFilter jobFilter = getInputPathFilter(job);
if (jobFilter != null) {
filters.add(jobFilter);
}
PathFilter inputFilter = new MultiPathFilter(filters);
List<FileStatus> result = null;
int numThreads =
job.getConfiguration().getInt(LIST_STATUS_NUM_THREADS, DEFAULT_LIST_STATUS_NUM_THREADS);
Stopwatch sw = new Stopwatch().start();
if (numThreads == 1) {
result = singleThreadedListStatus(job, dirs, inputFilter, recursive);
} else {
Iterable<FileStatus> locatedFiles = null;
try {
LocatedFileStatusFetcher locatedFileStatusFetcher =
new LocatedFileStatusFetcher(
job.getConfiguration(), dirs, recursive, inputFilter, true);
locatedFiles = locatedFileStatusFetcher.getFileStatuses();
} catch (InterruptedException e) {
throw new IOException("Interrupted while getting file statuses");
}
result = Lists.newArrayList(locatedFiles);
}
sw.stop();
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Time taken to get FileStatuses: "+sw.elapsedMillis()) */
LOG.time_taken_get_filestatuses(String.valueOf(sw.elapsedMillis())).tag("methodCall").debug();
}
/* LOG.info("Total input paths to process : "+result.size()) */
LOG.total_input_paths_process(String.valueOf(result.size())).tag("methodCall").info();
return result;
}
private List<FileStatus> singleThreadedListStatus(
JobContext job, Path[] dirs, PathFilter inputFilter, boolean recursive) throws IOException {
List<FileStatus> result = new ArrayList<FileStatus>();
List<IOException> errors = new ArrayList<IOException>();
for (int i = 0; i < dirs.length; ++i) {
Path p = dirs[i];
FileSystem fs = p.getFileSystem(job.getConfiguration());
FileStatus[] matches = fs.globStatus(p, inputFilter);
if (matches == null) {
errors.add(new IOException("Input path does not exist: " + p));
} else if (matches.length == 0) {
errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
} else {
for (FileStatus globStat : matches) {
if (globStat.isDirectory()) {
RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(globStat.getPath());
while (iter.hasNext()) {
LocatedFileStatus stat = iter.next();
if (inputFilter.accept(stat.getPath())) {
if (recursive && stat.isDirectory()) {
addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
} else {
result.add(stat);
}
}
}
} else {
result.add(globStat);
}
}
}
}
if (!errors.isEmpty()) {
throw new InvalidInputException(errors);
}
return result;
}
/**
* Add files in the input path recursively into the results.
*
* @param result The List to store all files.
* @param fs The FileSystem.
* @param path The input path.
* @param inputFilter The input filter that can be used to filter files/dirs.
* @throws IOException
*/
protected void addInputPathRecursively(
List<FileStatus> result, FileSystem fs, Path path, PathFilter inputFilter)
throws IOException {
RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
while (iter.hasNext()) {
LocatedFileStatus stat = iter.next();
if (inputFilter.accept(stat.getPath())) {
if (stat.isDirectory()) {
addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
} else {
result.add(stat);
}
}
}
}
/**
* A factory that makes the split for this class. It can be overridden by sub-classes to make
* sub-types
*/
protected FileSplit makeSplit(Path file, long start, long length, String[] hosts) {
return new FileSplit(file, start, length, hosts);
}
/**
* Generate the list of files and make them into FileSplits.
*
* @param job the job context
* @throws IOException
*/
public List<InputSplit> getSplits(JobContext job) throws IOException {
Stopwatch sw = new Stopwatch().start();
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
// generate splits
List<InputSplit> splits = new ArrayList<InputSplit>();
List<FileStatus> files = listStatus(job);
for (FileStatus file : files) {
Path path = file.getPath();
long length = file.getLen();
if (length != 0) {
BlockLocation[] blkLocations;
if (file instanceof LocatedFileStatus) {
blkLocations = ((LocatedFileStatus) file).getBlockLocations();
} else {
FileSystem fs = path.getFileSystem(job.getConfiguration());
blkLocations = fs.getFileBlockLocations(file, 0, length);
}
if (isSplitable(job, path)) {
long blockSize = file.getBlockSize();
long splitSize = computeSplitSize(blockSize, minSize, maxSize);
long bytesRemaining = length;
while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) {
int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
splits.add(
makeSplit(
path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts()));
bytesRemaining -= splitSize;
}
if (bytesRemaining != 0) {
int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
splits.add(
makeSplit(
path,
length - bytesRemaining,
bytesRemaining,
blkLocations[blkIndex].getHosts()));
}
} else { // not splitable
splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts()));
}
} else {
// Create empty hosts array for zero length files
splits.add(makeSplit(path, 0, length, new String[0]));
}
}
// Save the number of input files for metrics/loadgen
job.getConfiguration().setLong(NUM_INPUT_FILES, files.size());
sw.stop();
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Total # of splits generated by getSplits: "+splits.size()+", TimeTaken: "+sw.elapsedMillis()) */
LOG.total_splits_generated_getsplits_timetak(
String.valueOf(splits.size()), String.valueOf(sw.elapsedMillis()))
.tag("methodCall")
.debug();
}
return splits;
}
protected long computeSplitSize(long blockSize, long minSize, long maxSize) {
return Math.max(minSize, Math.min(maxSize, blockSize));
}
protected int getBlockIndex(BlockLocation[] blkLocations, long offset) {
for (int i = 0; i < blkLocations.length; i++) {
// is the offset inside this block?
if ((blkLocations[i].getOffset() <= offset)
&& (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())) {
return i;
}
}
BlockLocation last = blkLocations[blkLocations.length - 1];
long fileLength = last.getOffset() + last.getLength() - 1;
throw new IllegalArgumentException(
"Offset " + offset + " is outside of file (0.." + fileLength + ")");
}
/**
* Sets the given comma separated paths as the list of inputs for the map-reduce job.
*
* @param job the job
* @param commaSeparatedPaths Comma separated paths to be set as the list of inputs for the
* map-reduce job.
*/
public static void setInputPaths(Job job, String commaSeparatedPaths) throws IOException {
setInputPaths(job, StringUtils.stringToPath(getPathStrings(commaSeparatedPaths)));
}
/**
* Add the given comma separated paths to the list of inputs for the map-reduce job.
*
* @param job The job to modify
* @param commaSeparatedPaths Comma separated paths to be added to the list of inputs for the
* map-reduce job.
*/
public static void addInputPaths(Job job, String commaSeparatedPaths) throws IOException {
for (String str : getPathStrings(commaSeparatedPaths)) {
addInputPath(job, new Path(str));
}
}
/**
* Set the array of {@link Path}s as the list of inputs for the map-reduce job.
*
* @param job The job to modify
* @param inputPaths the {@link Path}s of the input directories/files for the map-reduce job.
*/
public static void setInputPaths(Job job, Path... inputPaths) throws IOException {
Configuration conf = job.getConfiguration();
Path path = inputPaths[0].getFileSystem(conf).makeQualified(inputPaths[0]);
StringBuffer str = new StringBuffer(StringUtils.escapeString(path.toString()));
for (int i = 1; i < inputPaths.length; i++) {
str.append(StringUtils.COMMA_STR);
path = inputPaths[i].getFileSystem(conf).makeQualified(inputPaths[i]);
str.append(StringUtils.escapeString(path.toString()));
}
conf.set(INPUT_DIR, str.toString());
}
/**
* Add a {@link Path} to the list of inputs for the map-reduce job.
*
* @param job The {@link Job} to modify
* @param path {@link Path} to be added to the list of inputs for the map-reduce job.
*/
public static void addInputPath(Job job, Path path) throws IOException {
Configuration conf = job.getConfiguration();
path = path.getFileSystem(conf).makeQualified(path);
String dirStr = StringUtils.escapeString(path.toString());
String dirs = conf.get(INPUT_DIR);
conf.set(INPUT_DIR, dirs == null ? dirStr : dirs + "," + dirStr);
}
// This method escapes commas in the glob pattern of the given paths.
private static String[] getPathStrings(String commaSeparatedPaths) {
int length = commaSeparatedPaths.length();
int curlyOpen = 0;
int pathStart = 0;
boolean globPattern = false;
List<String> pathStrings = new ArrayList<String>();
for (int i = 0; i < length; i++) {
char ch = commaSeparatedPaths.charAt(i);
switch (ch) {
case '{':
{
curlyOpen++;
if (!globPattern) {
globPattern = true;
}
break;
}
case '}':
{
curlyOpen--;
if (curlyOpen == 0 && globPattern) {
globPattern = false;
}
break;
}
case ',':
{
if (!globPattern) {
pathStrings.add(commaSeparatedPaths.substring(pathStart, i));
pathStart = i + 1;
}
break;
}
default:
continue; // nothing special to do for this character
}
}
pathStrings.add(commaSeparatedPaths.substring(pathStart, length));
return pathStrings.toArray(new String[0]);
}
/**
* Get the list of input {@link Path}s for the map-reduce job.
*
* @param context The job
* @return the list of input {@link Path}s for the map-reduce job.
*/
public static Path[] getInputPaths(JobContext context) {
String dirs = context.getConfiguration().get(INPUT_DIR, "");
String[] list = StringUtils.split(dirs);
Path[] result = new Path[list.length];
for (int i = 0; i < list.length; i++) {
result[i] = new Path(StringUtils.unEscapeString(list[i]));
}
return result;
}
}
/** This class returns build information about Hadoop components. */
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class VersionInfo {
private static final /* LogLOG=LogFactory.getLog(VersionInfo.class); */ UtilNamespace LOG =
LoggerFactory.getLogger(UtilNamespace.class, new SimpleLogger());
private Properties info;
protected VersionInfo(String component) {
info = new Properties();
String versionInfoFile = component + "-version-info.properties";
InputStream is = null;
try {
is = Thread.currentThread().getContextClassLoader().getResourceAsStream(versionInfoFile);
if (is == null) {
throw new IOException("Resource not found");
}
info.load(is);
} catch (IOException ex) {
LogFactory.getLog(getClass())
.warn("Could not read '" + versionInfoFile + "', " + ex.toString(), ex);
} finally {
IOUtils.closeStream(is);
}
}
protected String _getVersion() {
return info.getProperty("version", "Unknown");
}
protected String _getRevision() {
return info.getProperty("revision", "Unknown");
}
protected String _getBranch() {
return info.getProperty("branch", "Unknown");
}
protected String _getDate() {
return info.getProperty("date", "Unknown");
}
protected String _getUser() {
return info.getProperty("user", "Unknown");
}
protected String _getUrl() {
return info.getProperty("url", "Unknown");
}
protected String _getSrcChecksum() {
return info.getProperty("srcChecksum", "Unknown");
}
protected String _getBuildVersion() {
return getVersion()
+ " from "
+ _getRevision()
+ " by "
+ _getUser()
+ " source checksum "
+ _getSrcChecksum();
}
protected String _getProtocVersion() {
return info.getProperty("protocVersion", "Unknown");
}
private static VersionInfo COMMON_VERSION_INFO = new VersionInfo("common");
/**
* Get the Hadoop version.
*
* @return the Hadoop version string, eg. "0.6.3-dev"
*/
public static String getVersion() {
return COMMON_VERSION_INFO._getVersion();
}
/**
* Get the subversion revision number for the root directory
*
* @return the revision number, eg. "451451"
*/
public static String getRevision() {
return COMMON_VERSION_INFO._getRevision();
}
/**
* Get the branch on which this originated.
*
* @return The branch name, e.g. "trunk" or "branches/branch-0.20"
*/
public static String getBranch() {
return COMMON_VERSION_INFO._getBranch();
}
/**
* The date that Hadoop was compiled.
*
* @return the compilation date in unix date format
*/
public static String getDate() {
return COMMON_VERSION_INFO._getDate();
}
/**
* The user that compiled Hadoop.
*
* @return the username of the user
*/
public static String getUser() {
return COMMON_VERSION_INFO._getUser();
}
/** Get the subversion URL for the root Hadoop directory. */
public static String getUrl() {
return COMMON_VERSION_INFO._getUrl();
}
/** Get the checksum of the source files from which Hadoop was built. */
public static String getSrcChecksum() {
return COMMON_VERSION_INFO._getSrcChecksum();
}
/** Returns the buildVersion which includes version, revision, user and date. */
public static String getBuildVersion() {
return COMMON_VERSION_INFO._getBuildVersion();
}
/** Returns the protoc version used for the build. */
public static String getProtocVersion() {
return COMMON_VERSION_INFO._getProtocVersion();
}
public static void main(String[] args) {
/* LOG.debug("version: "+getVersion()) */
LOG.version(getVersion()).tag("methodCall").debug();
System.out.println("Hadoop " + getVersion());
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
System.out.println("Compiled by " + getUser() + " on " + getDate());
System.out.println("Compiled with protoc " + getProtocVersion());
System.out.println("From source with checksum " + getSrcChecksum());
System.out.println(
"This command was run using " + ClassUtil.findContainingJar(VersionInfo.class));
}
}
/** A class for file/directory permissions. */
@InterfaceAudience.Public
@InterfaceStability.Stable
public class FsPermission implements Writable {
private static final /* LogLOG=LogFactory.getLog(FsPermission.class); */ FsNamespace LOG =
LoggerFactory.getLogger(FsNamespace.class, new SimpleLogger());
static final WritableFactory FACTORY =
new WritableFactory() {
@Override
public Writable newInstance() {
return new FsPermission();
}
};
static { // register a ctor
WritableFactories.setFactory(FsPermission.class, FACTORY);
WritableFactories.setFactory(ImmutableFsPermission.class, FACTORY);
}
/** Maximum acceptable length of a permission string to parse */
public static final int MAX_PERMISSION_LENGTH = 10;
/** Create an immutable {@link FsPermission} object. */
public static FsPermission createImmutable(short permission) {
return new ImmutableFsPermission(permission);
}
// POSIX permission style
private FsAction useraction = null;
private FsAction groupaction = null;
private FsAction otheraction = null;
private boolean stickyBit = false;
private FsPermission() {}
/**
* Construct by the given {@link FsAction}.
*
* @param u user action
* @param g group action
* @param o other action
*/
public FsPermission(FsAction u, FsAction g, FsAction o) {
this(u, g, o, false);
}
public FsPermission(FsAction u, FsAction g, FsAction o, boolean sb) {
set(u, g, o, sb);
}
/**
* Construct by the given mode.
*
* @param mode
* @see #toShort()
*/
public FsPermission(short mode) {
fromShort(mode);
}
/**
* Copy constructor
*
* @param other other permission
*/
public FsPermission(FsPermission other) {
this.useraction = other.useraction;
this.groupaction = other.groupaction;
this.otheraction = other.otheraction;
this.stickyBit = other.stickyBit;
}
/**
* Construct by given mode, either in octal or symbolic format.
*
* @param mode mode as a string, either in octal or symbolic format
* @throws IllegalArgumentException if <code>mode</code> is invalid
*/
public FsPermission(String mode) {
this(new UmaskParser(mode).getUMask());
}
/** Return user {@link FsAction}. */
public FsAction getUserAction() {
return useraction;
}
/** Return group {@link FsAction}. */
public FsAction getGroupAction() {
return groupaction;
}
/** Return other {@link FsAction}. */
public FsAction getOtherAction() {
return otheraction;
}
private void set(FsAction u, FsAction g, FsAction o, boolean sb) {
useraction = u;
groupaction = g;
otheraction = o;
stickyBit = sb;
}
public void fromShort(short n) {
FsAction[] v = FSACTION_VALUES;
set(v[(n >>> 6) & 7], v[(n >>> 3) & 7], v[n & 7], (((n >>> 9) & 1) == 1));
}
@Override
public void write(DataOutput out) throws IOException {
out.writeShort(toShort());
}
@Override
public void readFields(DataInput in) throws IOException {
fromShort(in.readShort());
}
/** Create and initialize a {@link FsPermission} from {@link DataInput}. */
public static FsPermission read(DataInput in) throws IOException {
FsPermission p = new FsPermission();
p.readFields(in);
return p;
}
/** Encode the object to a short. */
public short toShort() {
int s =
(stickyBit ? 1 << 9 : 0)
| (useraction.ordinal() << 6)
| (groupaction.ordinal() << 3)
| otheraction.ordinal();
return (short) s;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof FsPermission) {
FsPermission that = (FsPermission) obj;
return this.useraction == that.useraction
&& this.groupaction == that.groupaction
&& this.otheraction == that.otheraction
&& this.stickyBit == that.stickyBit;
}
return false;
}
@Override
public int hashCode() {
return toShort();
}
@Override
public String toString() {
String str = useraction.SYMBOL + groupaction.SYMBOL + otheraction.SYMBOL;
if (stickyBit) {
StringBuilder str2 = new StringBuilder(str);
str2.replace(
str2.length() - 1, str2.length(), otheraction.implies(FsAction.EXECUTE) ? "t" : "T");
str = str2.toString();
}
return str;
}
/**
* Apply a umask to this permission and return a new one.
*
* <p>The umask is used by create, mkdir, and other Hadoop filesystem operations. The mode
* argument for these operations is modified by removing the bits which are set in the umask.
* Thus, the umask limits the permissions which newly created files and directories get.
*
* @param umask The umask to use
* @return The effective permission
*/
public FsPermission applyUMask(FsPermission umask) {
return new FsPermission(
useraction.and(umask.useraction.not()),
groupaction.and(umask.groupaction.not()),
otheraction.and(umask.otheraction.not()));
}
/**
* umask property label deprecated key and code in getUMask method to accommodate it may be
* removed in version .23
*/
public static final String DEPRECATED_UMASK_LABEL = "dfs.umask";
public static final String UMASK_LABEL = CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY;
public static final int DEFAULT_UMASK = CommonConfigurationKeys.FS_PERMISSIONS_UMASK_DEFAULT;
private static final FsAction[] FSACTION_VALUES = FsAction.values();
/**
* Get the user file creation mask (umask)
*
* <p>{@code UMASK_LABEL} config param has umask value that is either symbolic or octal.
*
* <p>Symbolic umask is applied relative to file mode creation mask; the permission op characters
* '+' clears the corresponding bit in the mask, '-' sets bits in the mask.
*
* <p>Octal umask, the specified bits are set in the file mode creation mask.
*
* <p>{@code DEPRECATED_UMASK_LABEL} config param has umask value set to decimal.
*/
public static FsPermission getUMask(Configuration conf) {
int umask = DEFAULT_UMASK;
// To ensure backward compatibility first use the deprecated key.
// If the deprecated key is not present then check for the new key
if (conf != null) {
String confUmask = conf.get(UMASK_LABEL);
int oldUmask = conf.getInt(DEPRECATED_UMASK_LABEL, Integer.MIN_VALUE);
try {
if (confUmask != null) {
umask = new UmaskParser(confUmask).getUMask();
}
} catch (IllegalArgumentException iae) {
// Provide more explanation for user-facing message
String type = iae instanceof NumberFormatException ? "decimal" : "octal or symbolic";
String error =
"Unable to parse configuration "
+ UMASK_LABEL
+ " with value "
+ confUmask
+ " as "
+ type
+ " umask.";
/* LOG.warn(error) */
LOG.error(error).warn();
// If oldUmask is not set, then throw the exception
if (oldUmask == Integer.MIN_VALUE) {
throw new IllegalArgumentException(error);
}
}
if (oldUmask != Integer.MIN_VALUE) { // Property was set with old key
if (umask != oldUmask) {
/* LOG.warn(DEPRECATED_UMASK_LABEL+" configuration key is deprecated. "+"Convert to "+UMASK_LABEL+", using octal or symbolic umask "+"specifications.") */
LOG.configuration_key_deprecated_convert_usi(DEPRECATED_UMASK_LABEL, UMASK_LABEL).warn();
// Old and new umask values do not match - Use old umask
umask = oldUmask;
}
}
}
return new FsPermission((short) umask);
}
public boolean getStickyBit() {
return stickyBit;
}
/** Set the user file creation mask (umask) */
public static void setUMask(Configuration conf, FsPermission umask) {
conf.set(UMASK_LABEL, String.format("%1$03o", umask.toShort()));
conf.setInt(DEPRECATED_UMASK_LABEL, umask.toShort());
}
/**
* Get the default permission for directory and symlink. In previous versions, this default
* permission was also used to create files, so files created end up with ugo+x permission. See
* HADOOP-9155 for detail. Two new methods are added to solve this, please use {@link
* FsPermission#getDirDefault()} for directory, and use {@link FsPermission#getFileDefault()} for
* file. This method is kept for compatibility.
*/
public static FsPermission getDefault() {
return new FsPermission((short) 00777);
}
/** Get the default permission for directory. */
public static FsPermission getDirDefault() {
return new FsPermission((short) 00777);
}
/** Get the default permission for file. */
public static FsPermission getFileDefault() {
return new FsPermission((short) 00666);
}
/** Get the default permission for cache pools. */
public static FsPermission getCachePoolDefault() {
return new FsPermission((short) 00755);
}
/**
* Create a FsPermission from a Unix symbolic permission string
*
* @param unixSymbolicPermission e.g. "-rw-rw-rw-"
*/
public static FsPermission valueOf(String unixSymbolicPermission) {
if (unixSymbolicPermission == null) {
return null;
} else if (unixSymbolicPermission.length() != MAX_PERMISSION_LENGTH) {
throw new IllegalArgumentException(
String.format(
"length != %d(unixSymbolicPermission=%s)",
MAX_PERMISSION_LENGTH, unixSymbolicPermission));
}
int n = 0;
for (int i = 1; i < unixSymbolicPermission.length(); i++) {
n = n << 1;
char c = unixSymbolicPermission.charAt(i);
n += (c == '-' || c == 'T' || c == 'S') ? 0 : 1;
}
// Add sticky bit value if set
if (unixSymbolicPermission.charAt(9) == 't' || unixSymbolicPermission.charAt(9) == 'T')
n += 01000;
return new FsPermission((short) n);
}
private static class ImmutableFsPermission extends FsPermission {
public ImmutableFsPermission(short permission) {
super(permission);
}
@Override
public FsPermission applyUMask(FsPermission umask) {
throw new UnsupportedOperationException();
}
@Override
public void readFields(DataInput in) throws IOException {
throw new UnsupportedOperationException();
}
}
}
/** This class provides a way to interact with history files in a thread safe manor. */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class HistoryFileManager extends AbstractService {
// private static final /* LogLOG=LogFactory.getLog(HistoryFileManager.class); */
// MapreduceNamespace LOG = LoggerFactory.getLogger(MapreduceNamespace.class, new
// SimpleLogger());
private static final /* LogSUMMARY_LOG=LogFactory.getLog(JobSummary.class); */ MapreduceNamespace
LOG = LoggerFactory.getLogger(MapreduceNamespace.class, new SimpleLogger());
private static enum HistoryInfoState {
IN_INTERMEDIATE,
IN_DONE,
DELETED,
MOVE_FAILED
};
private static String DONE_BEFORE_SERIAL_TAIL = JobHistoryUtils.doneSubdirsBeforeSerialTail();
/**
* Maps between a serial number (generated based on jobId) and the timestamp component(s) to which
* it belongs. Facilitates jobId based searches. If a jobId is not found in this list - it will
* not be found.
*/
private static class SerialNumberIndex {
private SortedMap<String, Set<String>> cache;
private int maxSize;
public SerialNumberIndex(int maxSize) {
this.cache = new TreeMap<String, Set<String>>();
this.maxSize = maxSize;
}
public synchronized void add(String serialPart, String timestampPart) {
if (!cache.containsKey(serialPart)) {
cache.put(serialPart, new HashSet<String>());
if (cache.size() > maxSize) {
String key = cache.firstKey();
/* LOG.error("Dropping "+key+" from the SerialNumberIndex. We will no "+"longer be able to see jobs that are in that serial index for "+cache.get(key)) */
LOG.dropping_from_serialnumberindex_will_lon(key, String.valueOf(cache.get(key)))
.tag("methodCall")
.error();
cache.remove(key);
}
}
Set<String> datePartSet = cache.get(serialPart);
datePartSet.add(timestampPart);
}
public synchronized void remove(String serialPart, String timeStampPart) {
if (cache.containsKey(serialPart)) {
Set<String> set = cache.get(serialPart);
set.remove(timeStampPart);
if (set.isEmpty()) {
cache.remove(serialPart);
}
}
}
public synchronized Set<String> get(String serialPart) {
Set<String> found = cache.get(serialPart);
if (found != null) {
return new HashSet<String>(found);
}
return null;
}
}
/**
* Wrapper around {@link ConcurrentSkipListMap} that maintains size along side for O(1) size()
* implementation for use in JobListCache.
*
* <p>Note: The size is not updated atomically with changes additions/removals. This race can lead
* to size() returning an incorrect size at times.
*/
static class JobIdHistoryFileInfoMap {
private ConcurrentSkipListMap<JobId, HistoryFileInfo> cache;
private AtomicInteger mapSize;
JobIdHistoryFileInfoMap() {
cache = new ConcurrentSkipListMap<JobId, HistoryFileInfo>();
mapSize = new AtomicInteger();
}
public HistoryFileInfo putIfAbsent(JobId key, HistoryFileInfo value) {
HistoryFileInfo ret = cache.putIfAbsent(key, value);
if (ret == null) {
mapSize.incrementAndGet();
}
return ret;
}
public HistoryFileInfo remove(JobId key) {
HistoryFileInfo ret = cache.remove(key);
if (ret != null) {
mapSize.decrementAndGet();
}
return ret;
}
/**
* Returns the recorded size of the internal map. Note that this could be out of sync with the
* actual size of the map
*
* @return "recorded" size
*/
public int size() {
return mapSize.get();
}
public HistoryFileInfo get(JobId key) {
return cache.get(key);
}
public NavigableSet<JobId> navigableKeySet() {
return cache.navigableKeySet();
}
public Collection<HistoryFileInfo> values() {
return cache.values();
}
}
static class JobListCache {
private JobIdHistoryFileInfoMap cache;
private int maxSize;
private long maxAge;
public JobListCache(int maxSize, long maxAge) {
this.maxSize = maxSize;
this.maxAge = maxAge;
this.cache = new JobIdHistoryFileInfoMap();
}
public HistoryFileInfo addIfAbsent(HistoryFileInfo fileInfo) {
JobId jobId = fileInfo.getJobId();
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Adding "+jobId+" to job list cache with "+fileInfo.getJobIndexInfo()) */
LOG.adding_job_list_cache_with(jobId.toString(), String.valueOf(fileInfo.getJobIndexInfo()))
.tag("methodCall")
.debug();
}
HistoryFileInfo old = cache.putIfAbsent(jobId, fileInfo);
if (cache.size() > maxSize) {
// There is a race here, where more then one thread could be trying to
// remove entries. This could result in too many entries being removed
// from the cache. This is considered OK as the size of the cache
// should be rather large, and we would rather have performance over
// keeping the cache size exactly at the maximum.
Iterator<JobId> keys = cache.navigableKeySet().iterator();
long cutoff = System.currentTimeMillis() - maxAge;
while (cache.size() > maxSize && keys.hasNext()) {
JobId key = keys.next();
HistoryFileInfo firstValue = cache.get(key);
if (firstValue != null) {
synchronized (firstValue) {
if (firstValue.isMovePending()) {
if (firstValue.didMoveFail() && firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
cache.remove(key);
// Now lets try to delete it
try {
firstValue.delete();
} catch (IOException e) {
/* LOG.error("Error while trying to delete history files"+" that could not be moved to done.",e) */
LOG.error_while_trying_delete_history_files_(e.toString()).error();
}
} else {
/* LOG.warn("Waiting to remove "+key+" from JobListCache because it is not in done yet.") */
LOG.waiting_remove_from_joblistcache_because(key.toString()).warn();
}
} else {
cache.remove(key);
}
}
}
}
}
return old;
}
public void delete(HistoryFileInfo fileInfo) {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Removing from cache "+fileInfo) */
LOG.removing_from_cache(fileInfo.toString()).debug();
}
cache.remove(fileInfo.getJobId());
}
public Collection<HistoryFileInfo> values() {
return new ArrayList<HistoryFileInfo>(cache.values());
}
public HistoryFileInfo get(JobId jobId) {
return cache.get(jobId);
}
}
/**
* This class represents a user dir in the intermediate done directory. This is mostly for locking
* purposes.
*/
private class UserLogDir {
long modTime = 0;
public synchronized void scanIfNeeded(FileStatus fs) {
long newModTime = fs.getModificationTime();
if (modTime != newModTime) {
Path p = fs.getPath();
try {
scanIntermediateDirectory(p);
// If scanning fails, we will scan again. We assume the failure is
// temporary.
modTime = newModTime;
} catch (IOException e) {
/* LOG.error("Error while trying to scan the directory "+p,e) */
LOG.error_while_trying_scan_directory(p.toString(), e.toString()).error();
}
} else {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Scan not needed of "+fs.getPath()) */
LOG.scan_not_needed(String.valueOf(fs.getPath())).tag("methodCall").debug();
}
}
}
}
public class HistoryFileInfo {
private Path historyFile;
private Path confFile;
private Path summaryFile;
private JobIndexInfo jobIndexInfo;
private HistoryInfoState state;
private HistoryFileInfo(
Path historyFile,
Path confFile,
Path summaryFile,
JobIndexInfo jobIndexInfo,
boolean isInDone) {
this.historyFile = historyFile;
this.confFile = confFile;
this.summaryFile = summaryFile;
this.jobIndexInfo = jobIndexInfo;
state = isInDone ? HistoryInfoState.IN_DONE : HistoryInfoState.IN_INTERMEDIATE;
}
@VisibleForTesting
synchronized boolean isMovePending() {
return state == HistoryInfoState.IN_INTERMEDIATE || state == HistoryInfoState.MOVE_FAILED;
}
@VisibleForTesting
synchronized boolean didMoveFail() {
return state == HistoryInfoState.MOVE_FAILED;
}
/** @return true if the files backed by this were deleted. */
public synchronized boolean isDeleted() {
return state == HistoryInfoState.DELETED;
}
@Override
public String toString() {
return "HistoryFileInfo jobID " + getJobId() + " historyFile = " + historyFile;
}
private synchronized void moveToDone() throws IOException {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("moveToDone: "+historyFile) */
LOG.movetodone(historyFile.toString()).debug();
}
if (!isMovePending()) {
// It was either deleted or is already in done. Either way do nothing
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Move no longer pending") */
LOG.move_longer_pending().debug();
}
return;
}
try {
long completeTime = jobIndexInfo.getFinishTime();
if (completeTime == 0) {
completeTime = System.currentTimeMillis();
}
JobId jobId = jobIndexInfo.getJobId();
List<Path> paths = new ArrayList<Path>(2);
if (historyFile == null) {
/* LOG.info("No file for job-history with "+jobId+" found in cache!") */
LOG.file_for_job_history_with_found(jobId.toString()).info();
} else {
paths.add(historyFile);
}
if (confFile == null) {
/* LOG.info("No file for jobConf with "+jobId+" found in cache!") */
LOG.file_for_jobconf_with_found_cache(jobId.toString()).info();
} else {
paths.add(confFile);
}
if (summaryFile == null) {
/* LOG.info("No summary file for job: "+jobId) */
LOG.summary_file_for_job(jobId.toString()).info();
} else {
String jobSummaryString = getJobSummary(intermediateDoneDirFc, summaryFile);
LOG.summary_file_for_job(jobSummaryString);
/* LOG.info("Deleting JobSummary file: ["+summaryFile+"]") */
LOG.deleting_jobsummary_file(summaryFile.toString()).info();
intermediateDoneDirFc.delete(summaryFile, false);
summaryFile = null;
}
Path targetDir = canonicalHistoryLogPath(jobId, completeTime);
addDirectoryToSerialNumberIndex(targetDir);
makeDoneSubdir(targetDir);
if (historyFile != null) {
Path toPath = doneDirFc.makeQualified(new Path(targetDir, historyFile.getName()));
if (!toPath.equals(historyFile)) {
moveToDoneNow(historyFile, toPath);
historyFile = toPath;
}
}
if (confFile != null) {
Path toPath = doneDirFc.makeQualified(new Path(targetDir, confFile.getName()));
if (!toPath.equals(confFile)) {
moveToDoneNow(confFile, toPath);
confFile = toPath;
}
}
state = HistoryInfoState.IN_DONE;
} catch (Throwable t) {
/* LOG.error("Error while trying to move a job to done",t) */
LOG.error_while_trying_move_job_done(t.toString()).error();
this.state = HistoryInfoState.MOVE_FAILED;
}
}
/**
* Parse a job from the JobHistoryFile, if the underlying file is not going to be deleted.
*
* @return the Job or null if the underlying file was deleted.
* @throws IOException if there is an error trying to read the file.
*/
public synchronized Job loadJob() throws IOException {
return new CompletedJob(
conf, jobIndexInfo.getJobId(), historyFile, false, jobIndexInfo.getUser(), this, aclsMgr);
}
/**
* Return the history file. This should only be used for testing.
*
* @return the history file.
*/
synchronized Path getHistoryFile() {
return historyFile;
}
protected synchronized void delete() throws IOException {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("deleting "+historyFile+" and "+confFile) */
LOG.deleting_and(historyFile.toString(), confFile.toString()).debug();
}
state = HistoryInfoState.DELETED;
doneDirFc.delete(doneDirFc.makeQualified(historyFile), false);
doneDirFc.delete(doneDirFc.makeQualified(confFile), false);
}
public JobIndexInfo getJobIndexInfo() {
return jobIndexInfo;
}
public JobId getJobId() {
return jobIndexInfo.getJobId();
}
public synchronized Path getConfFile() {
return confFile;
}
public synchronized Configuration loadConfFile() throws IOException {
FileContext fc = FileContext.getFileContext(confFile.toUri(), conf);
Configuration jobConf = new Configuration(false);
jobConf.addResource(fc.open(confFile), confFile.toString());
return jobConf;
}
}
private SerialNumberIndex serialNumberIndex = null;
protected JobListCache jobListCache = null;
// Maintains a list of known done subdirectories.
private final Set<Path> existingDoneSubdirs = Collections.synchronizedSet(new HashSet<Path>());
/**
* Maintains a mapping between intermediate user directories and the last known modification time.
*/
private ConcurrentMap<String, UserLogDir> userDirModificationTimeMap =
new ConcurrentHashMap<String, UserLogDir>();
private JobACLsManager aclsMgr;
@VisibleForTesting Configuration conf;
private String serialNumberFormat;
private Path doneDirPrefixPath = null; // folder for completed jobs
private FileContext doneDirFc; // done Dir FileContext
private Path intermediateDoneDirPath = null; // Intermediate Done Dir Path
private FileContext intermediateDoneDirFc; // Intermediate Done Dir
// FileContext
@VisibleForTesting protected ThreadPoolExecutor moveToDoneExecutor = null;
private long maxHistoryAge = 0;
public HistoryFileManager() {
super(HistoryFileManager.class.getName());
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
this.conf = conf;
int serialNumberLowDigits = 3;
serialNumberFormat =
("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");
long maxFSWaitTime =
conf.getLong(
JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);
this.aclsMgr = new JobACLsManager(conf);
maxHistoryAge =
conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
jobListCache = createJobListCache();
serialNumberIndex =
new SerialNumberIndex(
conf.getInt(
JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));
int numMoveThreads =
conf.getInt(
JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
ThreadFactory tf =
new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
moveToDoneExecutor =
new ThreadPoolExecutor(
numMoveThreads,
numMoveThreads,
1,
TimeUnit.HOURS,
new LinkedBlockingQueue<Runnable>(),
tf);
super.serviceInit(conf);
}
@VisibleForTesting
void createHistoryDirs(Clock clock, long intervalCheckMillis, long timeOutMillis)
throws IOException {
long start = clock.getTime();
boolean done = false;
int counter = 0;
while (!done && ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
try {
Thread.sleep(intervalCheckMillis);
} catch (InterruptedException ex) {
throw new YarnRuntimeException(ex);
}
}
if (!done) {
throw new YarnRuntimeException(
"Timed out '" + timeOutMillis + "ms' waiting for FileSystem to become available");
}
}
/**
* DistributedFileSystem returns a RemoteException with a message stating SafeModeException in it.
* So this is only way to check it is because of being in safe mode.
*/
private boolean isBecauseSafeMode(Throwable ex) {
return ex.toString().contains("SafeModeException");
}
/**
* Returns TRUE if the history dirs were created, FALSE if they could not be created because the
* FileSystem is not reachable or in safe mode and throws and exception otherwise.
*/
@VisibleForTesting
boolean tryCreatingHistoryDirs(boolean logWait) throws IOException {
boolean succeeded = true;
String doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
try {
doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
mkdir(
doneDirFc,
doneDirPrefixPath,
new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
} catch (ConnectException ex) {
if (logWait) {
/* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be available") */
LOG.waiting_for_filesystem_available(
String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
.tag("methodCall")
.info();
}
succeeded = false;
} catch (IOException e) {
if (isBecauseSafeMode(e)) {
succeeded = false;
if (logWait) {
/* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be out of safe mode") */
LOG.waiting_for_filesystem_out_safe_mode(
String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
.tag("methodCall")
.info();
}
} else {
throw new YarnRuntimeException(
"Error creating done directory: [" + doneDirPrefixPath + "]", e);
}
}
if (succeeded) {
String intermediateDoneDirPrefix =
JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
try {
intermediateDoneDirPath =
FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
mkdir(
intermediateDoneDirFc,
intermediateDoneDirPath,
new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
} catch (ConnectException ex) {
succeeded = false;
if (logWait) {
/* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be available") */
LOG.waiting_for_filesystem_available(
String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
.tag("methodCall")
.info();
}
} catch (IOException e) {
if (isBecauseSafeMode(e)) {
succeeded = false;
if (logWait) {
/* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be out of safe mode") */
LOG.waiting_for_filesystem_out_safe_mode(
String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
.tag("methodCall")
.info();
}
} else {
throw new YarnRuntimeException(
"Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
}
}
}
return succeeded;
}
@Override
public void serviceStop() throws Exception {
ShutdownThreadsHelper.shutdownExecutorService(moveToDoneExecutor);
super.serviceStop();
}
protected JobListCache createJobListCache() {
return new JobListCache(
conf.getInt(
JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,
JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE),
maxHistoryAge);
}
private void mkdir(FileContext fc, Path path, FsPermission fsp) throws IOException {
if (!fc.util().exists(path)) {
try {
fc.mkdir(path, fsp, true);
FileStatus fsStatus = fc.getFileStatus(path);
/* LOG.info("Perms after creating "+fsStatus.getPermission().toShort()+", Expected: "+fsp.toShort()) */
LOG.perms_after_creating_expected(
String.valueOf(fsStatus.getPermission().toShort()), String.valueOf(fsp.toShort()))
.tag("methodCall")
.info();
if (fsStatus.getPermission().toShort() != fsp.toShort()) {
/* LOG.info("Explicitly setting permissions to : "+fsp.toShort()+", "+fsp) */
LOG.explicitly_setting_permissions(String.valueOf(fsp.toShort()), fsp.toString())
.tag("methodCall")
.info();
fc.setPermission(path, fsp);
}
} catch (FileAlreadyExistsException e) {
/* LOG.info("Directory: ["+path+"] already exists.") */
LOG.directory_already_exists(path.toString()).info();
}
}
}
/** Populates index data structures. Should only be called at initialization times. */
@SuppressWarnings("unchecked")
void initExisting() throws IOException {
/* LOG.info("Initializing Existing Jobs...") */
LOG.initializing_existing_jobs().info();
List<FileStatus> timestampedDirList = findTimestampedDirectories();
// Sort first just so insertion is in a consistent order
Collections.sort(timestampedDirList);
for (FileStatus fs : timestampedDirList) {
// TODO Could verify the correct format for these directories.
addDirectoryToSerialNumberIndex(fs.getPath());
addDirectoryToJobListCache(fs.getPath());
}
}
private void removeDirectoryFromSerialNumberIndex(Path serialDirPath) {
String serialPart = serialDirPath.getName();
String timeStampPart = JobHistoryUtils.getTimestampPartFromPath(serialDirPath.toString());
if (timeStampPart == null) {
/* LOG.warn("Could not find timestamp portion from path: "+serialDirPath.toString()+". Continuing with next") */
LOG.could_not_find_timestamp_portion_from_co(String.valueOf(serialDirPath.toString()))
.tag("methodCall")
.warn();
return;
}
if (serialPart == null) {
/* LOG.warn("Could not find serial portion from path: "+serialDirPath.toString()+". Continuing with next") */
LOG.could_not_find_serial_portion_from_conti(String.valueOf(serialDirPath.toString()))
.tag("methodCall")
.warn();
return;
}
serialNumberIndex.remove(serialPart, timeStampPart);
}
private void addDirectoryToSerialNumberIndex(Path serialDirPath) {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Adding "+serialDirPath+" to serial index") */
LOG.adding_serial_index(serialDirPath.toString()).debug();
}
String serialPart = serialDirPath.getName();
String timestampPart = JobHistoryUtils.getTimestampPartFromPath(serialDirPath.toString());
if (timestampPart == null) {
/* LOG.warn("Could not find timestamp portion from path: "+serialDirPath+". Continuing with next") */
LOG.could_not_find_timestamp_portion_from_co(serialDirPath.toString()).warn();
return;
}
if (serialPart == null) {
/* LOG.warn("Could not find serial portion from path: "+serialDirPath.toString()+". Continuing with next") */
LOG.could_not_find_serial_portion_from_conti(String.valueOf(serialDirPath.toString()))
.tag("methodCall")
.warn();
} else {
serialNumberIndex.add(serialPart, timestampPart);
}
}
private void addDirectoryToJobListCache(Path path) throws IOException {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Adding "+path+" to job list cache.") */
LOG.adding_job_list_cache(path.toString()).debug();
}
List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path, doneDirFc);
for (FileStatus fs : historyFileList) {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Adding in history for "+fs.getPath()) */
LOG.adding_history_for(String.valueOf(fs.getPath())).tag("methodCall").debug();
}
JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
String summaryFileName =
JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
HistoryFileInfo fileInfo =
new HistoryFileInfo(
fs.getPath(),
new Path(fs.getPath().getParent(), confFileName),
new Path(fs.getPath().getParent(), summaryFileName),
jobIndexInfo,
true);
jobListCache.addIfAbsent(fileInfo);
}
}
private static List<FileStatus> scanDirectory(Path path, FileContext fc, PathFilter pathFilter)
throws IOException {
path = fc.makeQualified(path);
List<FileStatus> jhStatusList = new ArrayList<FileStatus>();
RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path);
while (fileStatusIter.hasNext()) {
FileStatus fileStatus = fileStatusIter.next();
Path filePath = fileStatus.getPath();
if (fileStatus.isFile() && pathFilter.accept(filePath)) {
jhStatusList.add(fileStatus);
}
}
return jhStatusList;
}
protected List<FileStatus> scanDirectoryForHistoryFiles(Path path, FileContext fc)
throws IOException {
return scanDirectory(path, fc, JobHistoryUtils.getHistoryFileFilter());
}
/**
* Finds all history directories with a timestamp component by scanning the filesystem. Used when
* the JobHistory server is started.
*
* @return list of history directories
*/
protected List<FileStatus> findTimestampedDirectories() throws IOException {
List<FileStatus> fsList =
JobHistoryUtils.localGlobber(doneDirFc, doneDirPrefixPath, DONE_BEFORE_SERIAL_TAIL);
return fsList;
}
/**
* Scans the intermediate directory to find user directories. Scans these for history files if the
* modification time for the directory has changed. Once it finds history files it starts the
* process of moving them to the done directory.
*
* @throws IOException if there was a error while scanning
*/
void scanIntermediateDirectory() throws IOException {
// TODO it would be great to limit how often this happens, except in the
// case where we are looking for a particular job.
List<FileStatus> userDirList =
JobHistoryUtils.localGlobber(intermediateDoneDirFc, intermediateDoneDirPath, "");
/* LOG.debug("Scanning intermediate dirs") */
LOG.scanning_intermediate_dirs().debug();
for (FileStatus userDir : userDirList) {
String name = userDir.getPath().getName();
UserLogDir dir = userDirModificationTimeMap.get(name);
if (dir == null) {
dir = new UserLogDir();
UserLogDir old = userDirModificationTimeMap.putIfAbsent(name, dir);
if (old != null) {
dir = old;
}
}
dir.scanIfNeeded(userDir);
}
}
/**
* Scans the specified path and populates the intermediate cache.
*
* @param absPath
* @throws IOException
*/
private void scanIntermediateDirectory(final Path absPath) throws IOException {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Scanning intermediate dir "+absPath) */
LOG.scanning_intermediate_dir(absPath.toString()).debug();
}
List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(absPath, intermediateDoneDirFc);
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Found "+fileStatusList.size()+" files") */
LOG.found_files(String.valueOf(fileStatusList.size())).tag("methodCall").debug();
}
for (FileStatus fs : fileStatusList) {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("scanning file: "+fs.getPath()) */
LOG.scanning_file(String.valueOf(fs.getPath())).tag("methodCall").debug();
}
JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
String summaryFileName =
JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
HistoryFileInfo fileInfo =
new HistoryFileInfo(
fs.getPath(),
new Path(fs.getPath().getParent(), confFileName),
new Path(fs.getPath().getParent(), summaryFileName),
jobIndexInfo,
false);
final HistoryFileInfo old = jobListCache.addIfAbsent(fileInfo);
if (old == null || old.didMoveFail()) {
final HistoryFileInfo found = (old == null) ? fileInfo : old;
long cutoff = System.currentTimeMillis() - maxHistoryAge;
if (found.getJobIndexInfo().getFinishTime() <= cutoff) {
try {
found.delete();
} catch (IOException e) {
/* LOG.warn("Error cleaning up a HistoryFile that is out of date.",e) */
LOG.error_cleaning_historyfile_that_out_date(e.toString()).warn();
}
} else {
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Scheduling move to done of "+found) */
LOG.scheduling_move_done(found.toString()).debug();
}
moveToDoneExecutor.execute(
new Runnable() {
@Override
public void run() {
try {
found.moveToDone();
} catch (IOException e) {
/* LOG.info("Failed to process fileInfo for job: "+found.getJobId(),e) */
LOG.failed_process_fileinfo_for_job(
String.valueOf(found.getJobId()), e.toString())
.tag("methodCall")
.info();
}
}
});
}
} else if (old != null && !old.isMovePending()) {
// This is a duplicate so just delete it
if (LogGlobal.isDebugEnabled()) {
/* LOG.debug("Duplicate: deleting") */
LOG.duplicate_deleting().debug();
}
fileInfo.delete();
}
}
}
/**
* Searches the job history file FileStatus list for the specified JobId.
*
* @param fileStatusList fileStatus list of Job History Files.
* @param jobId The JobId to find.
* @return A FileInfo object for the jobId, null if not found.
* @throws IOException
*/
private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList, JobId jobId)
throws IOException {
for (FileStatus fs : fileStatusList) {
JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
if (jobIndexInfo.getJobId().equals(jobId)) {
String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
String summaryFileName =
JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
HistoryFileInfo fileInfo =
new HistoryFileInfo(
fs.getPath(),
new Path(fs.getPath().getParent(), confFileName),
new Path(fs.getPath().getParent(), summaryFileName),
jobIndexInfo,
true);
return fileInfo;
}
}
return null;
}
/**
* Scans old directories known by the idToDateString map for the specified jobId. If the number of
* directories is higher than the supported size of the idToDateString cache, the jobId will not
* be found.
*
* @param jobId the jobId.
* @return
* @throws IOException
*/
private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
String boxedSerialNumber =
JobHistoryUtils.serialNumberDirectoryComponent(jobId, serialNumberFormat);
Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
if (dateStringSet == null) {
return null;
}
for (String timestampPart : dateStringSet) {
Path logDir = canonicalHistoryLogPath(jobId, timestampPart);
List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(logDir, doneDirFc);
HistoryFileInfo fileInfo = getJobFileInfo(fileStatusList, jobId);
if (fileInfo != null) {
return fileInfo;
}
}
return null;
}
public Collection<HistoryFileInfo> getAllFileInfo() throws IOException {
scanIntermediateDirectory();
return jobListCache.values();
}
public HistoryFileInfo getFileInfo(JobId jobId) throws IOException {
// FileInfo available in cache.
HistoryFileInfo fileInfo = jobListCache.get(jobId);
if (fileInfo != null) {
return fileInfo;
}
// OK so scan the intermediate to be sure we did not lose it that way
scanIntermediateDirectory();
fileInfo = jobListCache.get(jobId);
if (fileInfo != null) {
return fileInfo;
}
// Intermediate directory does not contain job. Search through older ones.
fileInfo = scanOldDirsForJob(jobId);
if (fileInfo != null) {
return fileInfo;
}
return null;
}
private void moveToDoneNow(final Path src, final Path target) throws IOException {
/* LOG.info("Moving "+src.toString()+" to "+target.toString()) */
LOG.moving(String.valueOf(src.toString()), String.valueOf(target.toString()))
.tag("methodCall")
.info();
intermediateDoneDirFc.rename(src, target, Options.Rename.NONE);
}
private String getJobSummary(FileContext fc, Path path) throws IOException {
Path qPath = fc.makeQualified(path);
FSDataInputStream in = fc.open(qPath);
String jobSummaryString = in.readUTF();
in.close();
return jobSummaryString;
}
private void makeDoneSubdir(Path path) throws IOException {
try {
doneDirFc.getFileStatus(path);
existingDoneSubdirs.add(path);
} catch (FileNotFoundException fnfE) {
try {
FsPermission fsp = new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION);
doneDirFc.mkdir(path, fsp, true);
FileStatus fsStatus = doneDirFc.getFileStatus(path);
/* LOG.info("Perms after creating "+fsStatus.getPermission().toShort()+", Expected: "+fsp.toShort()) */
LOG.perms_after_creating_expected(
String.valueOf(fsStatus.getPermission().toShort()), String.valueOf(fsp.toShort()))
.tag("methodCall")
.info();
if (fsStatus.getPermission().toShort() != fsp.toShort()) {
/* LOG.info("Explicitly setting permissions to : "+fsp.toShort()+", "+fsp) */
LOG.explicitly_setting_permissions(String.valueOf(fsp.toShort()), fsp.toString())
.tag("methodCall")
.info();
doneDirFc.setPermission(path, fsp);
}
existingDoneSubdirs.add(path);
} catch (FileAlreadyExistsException faeE) { // Nothing to do.
}
}
}
private Path canonicalHistoryLogPath(JobId id, String timestampComponent) {
return new Path(
doneDirPrefixPath,
JobHistoryUtils.historyLogSubdirectory(id, timestampComponent, serialNumberFormat));
}
private Path canonicalHistoryLogPath(JobId id, long millisecondTime) {
String timestampComponent = JobHistoryUtils.timestampDirectoryComponent(millisecondTime);
return new Path(
doneDirPrefixPath,
JobHistoryUtils.historyLogSubdirectory(id, timestampComponent, serialNumberFormat));
}
private long getEffectiveTimestamp(long finishTime, FileStatus fileStatus) {
if (finishTime == 0) {
return fileStatus.getModificationTime();
}
return finishTime;
}
private void deleteJobFromDone(HistoryFileInfo fileInfo) throws IOException {
jobListCache.delete(fileInfo);
fileInfo.delete();
}
List<FileStatus> getHistoryDirsForCleaning(long cutoff) throws IOException {
return JobHistoryUtils.getHistoryDirsForCleaning(doneDirFc, doneDirPrefixPath, cutoff);
}
/**
* Clean up older history files.
*
* @throws IOException on any error trying to remove the entries.
*/
@SuppressWarnings("unchecked")
void clean() throws IOException {
long cutoff = System.currentTimeMillis() - maxHistoryAge;
boolean halted = false;
List<FileStatus> serialDirList = getHistoryDirsForCleaning(cutoff);
// Sort in ascending order. Relies on YYYY/MM/DD/Serial
Collections.sort(serialDirList);
for (FileStatus serialDir : serialDirList) {
List<FileStatus> historyFileList =
scanDirectoryForHistoryFiles(serialDir.getPath(), doneDirFc);
for (FileStatus historyFile : historyFileList) {
JobIndexInfo jobIndexInfo =
FileNameIndexUtils.getIndexInfo(historyFile.getPath().getName());
long effectiveTimestamp = getEffectiveTimestamp(jobIndexInfo.getFinishTime(), historyFile);
if (effectiveTimestamp <= cutoff) {
HistoryFileInfo fileInfo = this.jobListCache.get(jobIndexInfo.getJobId());
if (fileInfo == null) {
String confFileName =
JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
fileInfo =
new HistoryFileInfo(
historyFile.getPath(),
new Path(historyFile.getPath().getParent(), confFileName),
null,
jobIndexInfo,
true);
}
deleteJobFromDone(fileInfo);
} else {
halted = true;
break;
}
}
if (!halted) {
deleteDir(serialDir);
removeDirectoryFromSerialNumberIndex(serialDir.getPath());
existingDoneSubdirs.remove(serialDir.getPath());
} else {
break; // Don't scan any more directories.
}
}
}
protected boolean deleteDir(FileStatus serialDir)
throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException,
IOException {
return doneDirFc.delete(doneDirFc.makeQualified(serialDir.getPath()), true);
}
// for test
@VisibleForTesting
void setMaxHistoryAge(long newValue) {
maxHistoryAge = newValue;
}
}
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class DefaultCodec implements Configurable, CompressionCodec, DirectDecompressionCodec {
private static final /* LogLOG=LogFactory.getLog(DefaultCodec.class); */ IoNamespace LOG =
LoggerFactory.getLogger(IoNamespace.class, new SimpleLogger());
Configuration conf;
@Override
public void setConf(Configuration conf) {
this.conf = conf;
}
@Override
public Configuration getConf() {
return conf;
}
@Override
public CompressionOutputStream createOutputStream(OutputStream out) throws IOException {
// This may leak memory if called in a loop. The createCompressor() call
// may cause allocation of an untracked direct-backed buffer if native
// libs are being used (even if you close the stream). A Compressor
// object should be reused between successive calls.
/* LOG.warn("DefaultCodec.createOutputStream() may leak memory. "+"Create a compressor first.") */
LOG.defaultcodec_createoutputstream_may_leak().warn();
return new CompressorStream(
out, createCompressor(), conf.getInt("io.file.buffer.size", 4 * 1024));
}
@Override
public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor)
throws IOException {
return new CompressorStream(out, compressor, conf.getInt("io.file.buffer.size", 4 * 1024));
}
@Override
public Class<? extends Compressor> getCompressorType() {
return ZlibFactory.getZlibCompressorType(conf);
}
@Override
public Compressor createCompressor() {
return ZlibFactory.getZlibCompressor(conf);
}
@Override
public CompressionInputStream createInputStream(InputStream in) throws IOException {
return new DecompressorStream(
in, createDecompressor(), conf.getInt("io.file.buffer.size", 4 * 1024));
}
@Override
public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor)
throws IOException {
return new DecompressorStream(in, decompressor, conf.getInt("io.file.buffer.size", 4 * 1024));
}
@Override
public Class<? extends Decompressor> getDecompressorType() {
return ZlibFactory.getZlibDecompressorType(conf);
}
@Override
public Decompressor createDecompressor() {
return ZlibFactory.getZlibDecompressor(conf);
}
/** {@inheritDoc} */
@Override
public DirectDecompressor createDirectDecompressor() {
return ZlibFactory.getZlibDirectDecompressor(conf);
}
@Override
public String getDefaultExtension() {
return ".deflate";
}
}
/**
* This class parses the configured list of fencing methods, and is responsible for trying each one
* in turn while logging informative output.
*
* <p>The fencing methods are configured as a carriage-return separated list. Each line in the list
* is of the form:
*
* <p><code>com.example.foo.MyMethod(arg string)</code> or <code>com.example.foo.MyMethod</code> The
* class provided must implement the {@link FenceMethod} interface. The fencing methods that ship
* with Hadoop may also be referred to by shortened names:
*
* <p>
*
* <ul>
* <li><code>shell(/path/to/some/script.sh args...)</code>
* <li><code>sshfence(...)</code> (see {@link SshFenceByTcpPort})
* </ul>
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class NodeFencer {
private static final String CLASS_RE = "([a-zA-Z0-9\\.\\$]+)";
private static final Pattern CLASS_WITH_ARGUMENT = Pattern.compile(CLASS_RE + "\\((.+?)\\)");
private static final Pattern CLASS_WITHOUT_ARGUMENT = Pattern.compile(CLASS_RE);
private static final Pattern HASH_COMMENT_RE = Pattern.compile("#.*$");
private static final /* LogLOG=LogFactory.getLog(NodeFencer.class); */ HaNamespace LOG =
LoggerFactory.getLogger(HaNamespace.class, new SimpleLogger());
/** Standard fencing methods included with Hadoop. */
private static final Map<String, Class<? extends FenceMethod>> STANDARD_METHODS =
ImmutableMap.<String, Class<? extends FenceMethod>>of(
"shell", ShellCommandFencer.class,
"sshfence", SshFenceByTcpPort.class);
private final List<FenceMethodWithArg> methods;
NodeFencer(Configuration conf, String spec) throws BadFencingConfigurationException {
this.methods = parseMethods(conf, spec);
}
public static NodeFencer create(Configuration conf, String confKey)
throws BadFencingConfigurationException {
String confStr = conf.get(confKey);
if (confStr == null) {
return null;
}
return new NodeFencer(conf, confStr);
}
public boolean fence(HAServiceTarget fromSvc) {
/* LOG.info("====== Beginning Service Fencing Process... ======") */
LOG.beginning_service_fencing_process().info();
int i = 0;
for (FenceMethodWithArg method : methods) {
/* LOG.info("Trying method "+(++i)+"/"+methods.size()+": "+method) */
LOG.trying_method((++i), String.valueOf(methods.size()), method.toString())
.tag("mathExp")
.tag("methodCall")
.info();
try {
if (method.method.tryFence(fromSvc, method.arg)) {
/* LOG.info("====== Fencing successful by method "+method+" ======") */
LOG.fencing_successful_method(method.toString()).info();
return true;
}
} catch (BadFencingConfigurationException e) {
/* LOG.error("Fencing method "+method+" misconfigured",e) */
LOG.fencing_method_misconfigured(method.toString(), e.toString()).error();
continue;
} catch (Throwable t) {
/* LOG.error("Fencing method "+method+" failed with an unexpected error.",t) */
LOG.fencing_method_failed_with_unexpected_er(method.toString(), t.toString()).error();
continue;
}
/* LOG.warn("Fencing method "+method+" was unsuccessful.") */
LOG.fencing_method_was_unsuccessful(method.toString()).warn();
}
/* LOG.error("Unable to fence service by any configured method.") */
LOG.unable_fence_service_any_configured_meth().error();
return false;
}
private static List<FenceMethodWithArg> parseMethods(Configuration conf, String spec)
throws BadFencingConfigurationException {
String[] lines = spec.split("\\s*\n\\s*");
List<FenceMethodWithArg> methods = Lists.newArrayList();
for (String line : lines) {
line = HASH_COMMENT_RE.matcher(line).replaceAll("");
line = line.trim();
if (!line.isEmpty()) {
methods.add(parseMethod(conf, line));
}
}
return methods;
}
private static FenceMethodWithArg parseMethod(Configuration conf, String line)
throws BadFencingConfigurationException {
Matcher m;
if ((m = CLASS_WITH_ARGUMENT.matcher(line)).matches()) {
String className = m.group(1);
String arg = m.group(2);
return createFenceMethod(conf, className, arg);
} else if ((m = CLASS_WITHOUT_ARGUMENT.matcher(line)).matches()) {
String className = m.group(1);
return createFenceMethod(conf, className, null);
} else {
throw new BadFencingConfigurationException("Unable to parse line: '" + line + "'");
}
}
private static FenceMethodWithArg createFenceMethod(
Configuration conf, String clazzName, String arg) throws BadFencingConfigurationException {
Class<?> clazz;
try {
// See if it's a short name for one of the built-in methods
clazz = STANDARD_METHODS.get(clazzName);
if (clazz == null) {
// Try to instantiate the user's custom method
clazz = Class.forName(clazzName);
}
} catch (Exception e) {
throw new BadFencingConfigurationException(
"Could not find configured fencing method " + clazzName, e);
}
// Check that it implements the right interface
if (!FenceMethod.class.isAssignableFrom(clazz)) {
throw new BadFencingConfigurationException(
"Class " + clazzName + " does not implement FenceMethod");
}
FenceMethod method = (FenceMethod) ReflectionUtils.newInstance(clazz, conf);
method.checkArgs(arg);
return new FenceMethodWithArg(method, arg);
}
private static class FenceMethodWithArg {
private final FenceMethod method;
private final String arg;
private FenceMethodWithArg(FenceMethod method, String arg) {
this.method = method;
this.arg = arg;
}
@Override
public String toString() {
return method.getClass().getCanonicalName() + "(" + arg + ")";
}
}
}
