/**
* Add a file path to the current set of classpath entries. It adds the file to cache as well.
* Intended to be used by user code.
*
* @param file Path of the file to be added
* @param conf Configuration that contains the classpath setting
* @param fs FileSystem with respect to which {@code archivefile} should be interpreted.
*/
public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs)
throws IOException {
String filepath = file.toUri().getPath();
String classpath = conf.get("mapred.job.classpath.files");
conf.set(
"mapred.job.classpath.files",
classpath == null ? filepath : classpath + System.getProperty("path.separator") + filepath);
URI uri = fs.makeQualified(file).toUri();
addCacheFile(uri, conf);
}
/**
* Add an archive path to the current set of classpath entries. It adds the archive to cache as
* well. Intended to be used by user code.
*
* @param archive Path of the archive to be added
* @param conf Configuration that contains the classpath setting
* @param fs FileSystem with respect to which {@code archive} should be interpreted.
*/
public static void addArchiveToClassPath(Path archive, Configuration conf, FileSystem fs)
throws IOException {
String archivepath = archive.toUri().getPath();
String classpath = conf.get("mapred.job.classpath.archives");
conf.set(
"mapred.job.classpath.archives",
classpath == null
? archivepath
: classpath + System.getProperty("path.separator") + archivepath);
URI uri = fs.makeQualified(archive).toUri();
addCacheArchive(uri, conf);
}
private static URI addArchiveToClassPathHelper(Path archive, Configuration conf)
throws IOException {
String classpath = conf.get("mapred.job.classpath.archives");
// the scheme/authority use ':' as separator. put the unqualified path in classpath
String archivePath = archive.toUri().getPath();
conf.set(
"mapred.job.classpath.archives",
classpath == null
? archivePath
: classpath + System.getProperty("path.separator") + archivePath);
return archive.makeQualified(archive.getFileSystem(conf)).toUri();
}
/**
* Set the configuration with the given set of files
*
* @param files The list of files that need to be localized
* @param conf Configuration which will be changed
*/
public static void setCacheFiles(URI[] files, Configuration conf) {
String sfiles = StringUtils.uriToString(files);
conf.set("mapred.cache.files", sfiles);
}
/**
* Set the configuration with the given set of archives
*
* @param archives The list of archives that need to be localized
* @param conf Configuration which will be changed
*/
public static void setCacheArchives(URI[] archives, Configuration conf) {
String sarchives = StringUtils.uriToString(archives);
conf.set("mapred.cache.archives", sarchives);
}
/**
* This method allows you to create symlinks in the current working directory of the task to all
* the cache files/archives
*
* @param conf the jobconf
*/
public static void createSymlink(Configuration conf) {
conf.set("mapred.create.symlink", "yes");
}
/**
* This is to check the timestamp of the files to be localized. Used by internal MapReduce code.
*
* @param conf Configuration which stores the timestamp's
* @param timestamps comma separated list of timestamps of files. The order should be the same as
* the order in which the files are added.
*/
public static void setFileTimestamps(Configuration conf, String timestamps) {
conf.set(CACHE_FILES_TIMESTAMPS, timestamps);
}
/**
* Set the configuration with the given set of files. Intended to be used by user code.
*
* @param files The list of files that need to be localized
* @param conf Configuration which will be changed
*/
public static void setCacheFiles(URI[] files, Configuration conf) {
String sfiles = StringUtils.uriToString(files);
conf.set(CACHE_FILES, sfiles);
}
/**
* Add a file to be localized to the conf. Intended to be used by user code.
*
* @param uri The uri of the cache to be localized
* @param conf Configuration to add the cache to
*/
public static void addCacheFile(URI uri, Configuration conf) {
String files = conf.get(CACHE_FILES);
conf.set(CACHE_FILES, files == null ? uri.toString() : files + "," + uri.toString());
}
public static void setSharedFileLength(Configuration conf, String length) {
conf.set("mapred.cache.shared.files.length", length);
}
/**
* Add a file that has been localized to the conf.. Used by internal DistributedCache code.
*
* @param conf The conf to modify to contain the localized caches
* @param str a comma separated list of local files
*/
public static void addLocalFiles(Configuration conf, String str) {
String files = conf.get(CACHE_LOCALFILES);
conf.set(CACHE_LOCALFILES, files == null ? str : files + "," + str);
}
/**
* Add a archives to be localized to the conf. Intended to be used by user code.
*
* @param uri The uri of the cache to be localized
* @param conf Configuration to add the cache to
*/
public static void addCacheArchive(URI uri, Configuration conf) {
String archives = conf.get(CACHE_ARCHIVES);
conf.set(CACHE_ARCHIVES, archives == null ? uri.toString() : archives + "," + uri.toString());
}
/**
* Add a archive that has been localized to the conf. Used by internal DistributedCache code.
*
* @param conf The conf to modify to contain the localized caches
* @param str a comma separated list of local archives
*/
public static void addLocalArchives(Configuration conf, String str) {
String archives = conf.get(CACHE_LOCALARCHIVES);
conf.set(CACHE_LOCALARCHIVES, archives == null ? str : archives + "," + str);
}
/**
* Set the conf to contain the location for localized files. Used by internal DistributedCache
* code.
*
* @param conf The conf to modify to contain the localized caches
* @param str a comma separated list of local files
*/
public static void setLocalFiles(Configuration conf, String str) {
conf.set(CACHE_LOCALFILES, str);
}
/**
* Set the conf to contain the location for localized archives. Used by internal DistributedCache
* code.
*
* @param conf The conf to modify to contain the localized caches
* @param str a comma separated list of local archives
*/
public static void setLocalArchives(Configuration conf, String str) {
conf.set(CACHE_LOCALARCHIVES, str);
}
/**
* This is to check the timestamp of the archives to be localized
*
* @param conf Configuration which stores the timestamp's
* @param timestamps comma separated list of timestamps of archives. The order should be the same
* as the order in which the archives are added.
*/
public static void setArchiveTimestamps(Configuration conf, String timestamps) {
conf.set("mapred.cache.archives.timestamps", timestamps);
}
/**
* Set the conf to contain the location for localized files
*
* @param conf The conf to modify to contain the localized caches
* @param str a comma separated list of local files
*/
public static void setLocalSharedFiles(Configuration conf, String str) {
conf.set("mapred.cache.shared.localFiles", str);
}
/**
* This is to check the timestamp of the files to be localized
*
* @param conf Configuration which stores the timestamp's
* @param timestamps comma separated list of timestamps of files. The order should be the same as
* the order in which the files are added.
*/
public static void setFileTimestamps(Configuration conf, String timestamps) {
conf.set("mapred.cache.files.timestamps", timestamps);
}
/**
* Add a file to be localized to the conf
*
* @param uri The uri of the cache to be localized
* @param conf Configuration to add the cache to
*/
public static void addCacheFile(URI uri, Configuration conf) {
String files = conf.get("mapred.cache.files");
conf.set("mapred.cache.files", files == null ? uri.toString() : files + "," + uri.toString());
}
/**
* Set the conf to contain the location for localized archives
*
* @param conf The conf to modify to contain the localized caches
* @param str a comma separated list of local archives
*/
public static void setLocalArchives(Configuration conf, String str) {
conf.set("mapred.cache.localArchives", str);
}
/**
* This is to check the timestamp of the archives to be localized. Used by internal MapReduce
* code.
*
* @param conf Configuration which stores the timestamp's
* @param timestamps comma separated list of timestamps of archives. The order should be the same
* as the order in which the archives are added.
*/
public static void setArchiveTimestamps(Configuration conf, String timestamps) {
conf.set(CACHE_ARCHIVES_TIMESTAMPS, timestamps);
}
/**
* Add a archives to be localized to the conf
*
* @param uri The uri of the cache to be localized
* @param conf Configuration to add the cache to
*/
public static void addSharedCacheArchive(URI uri, Configuration conf) {
String archives = conf.get("mapred.cache.shared.archives");
conf.set(
"mapred.cache.shared.archives",
archives == null ? uri.toString() : archives + "," + uri.toString());
}
/**
* This method allows you to create symlinks in the current working directory of the task to all
* the cache files/archives. Intended to be used by user code.
*
* @param conf the jobconf
*/
public static void createSymlink(Configuration conf) {
conf.set(CACHE_SYMLINK, "yes");
}
FileDataGenNew(String HDFSMaster) {
fsConf.set("fs.default.name", HDFSMaster);
}
/**
* Set the configuration with the given set of archives. Intended to be used by user code.
*
* @param archives The list of archives that need to be localized
* @param conf Configuration which will be changed
*/
public static void setCacheArchives(URI[] archives, Configuration conf) {
String sarchives = StringUtils.uriToString(archives);
conf.set(CACHE_ARCHIVES, sarchives);
}
