/**
  * Add a file path to the current set of classpath entries. It adds the file to cache as well.
  * Intended to be used by user code.
  *
  * @param file Path of the file to be added
  * @param conf Configuration that contains the classpath setting
  * @param fs FileSystem with respect to which {@code archivefile} should be interpreted.
  */
 public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs)
     throws IOException {
   String filepath = file.toUri().getPath();
   String classpath = conf.get("mapred.job.classpath.files");
   conf.set(
       "mapred.job.classpath.files",
       classpath == null ? filepath : classpath + System.getProperty("path.separator") + filepath);
   URI uri = fs.makeQualified(file).toUri();
   addCacheFile(uri, conf);
 }
  /**
   * Add an archive path to the current set of classpath entries. It adds the archive to cache as
   * well. Intended to be used by user code.
   *
   * @param archive Path of the archive to be added
   * @param conf Configuration that contains the classpath setting
   * @param fs FileSystem with respect to which {@code archive} should be interpreted.
   */
  public static void addArchiveToClassPath(Path archive, Configuration conf, FileSystem fs)
      throws IOException {
    String archivepath = archive.toUri().getPath();
    String classpath = conf.get("mapred.job.classpath.archives");
    conf.set(
        "mapred.job.classpath.archives",
        classpath == null
            ? archivepath
            : classpath + System.getProperty("path.separator") + archivepath);
    URI uri = fs.makeQualified(archive).toUri();

    addCacheArchive(uri, conf);
  }
Esempio n. 3
0
  private static URI addArchiveToClassPathHelper(Path archive, Configuration conf)
      throws IOException {

    String classpath = conf.get("mapred.job.classpath.archives");

    // the scheme/authority use ':' as separator. put the unqualified path in classpath
    String archivePath = archive.toUri().getPath();

    conf.set(
        "mapred.job.classpath.archives",
        classpath == null
            ? archivePath
            : classpath + System.getProperty("path.separator") + archivePath);
    return archive.makeQualified(archive.getFileSystem(conf)).toUri();
  }
Esempio n. 4
0
 /**
  * Set the configuration with the given set of files
  *
  * @param files The list of files that need to be localized
  * @param conf Configuration which will be changed
  */
 public static void setCacheFiles(URI[] files, Configuration conf) {
   String sfiles = StringUtils.uriToString(files);
   conf.set("mapred.cache.files", sfiles);
 }
Esempio n. 5
0
 /**
  * Set the configuration with the given set of archives
  *
  * @param archives The list of archives that need to be localized
  * @param conf Configuration which will be changed
  */
 public static void setCacheArchives(URI[] archives, Configuration conf) {
   String sarchives = StringUtils.uriToString(archives);
   conf.set("mapred.cache.archives", sarchives);
 }
Esempio n. 6
0
 /**
  * This method allows you to create symlinks in the current working directory of the task to all
  * the cache files/archives
  *
  * @param conf the jobconf
  */
 public static void createSymlink(Configuration conf) {
   conf.set("mapred.create.symlink", "yes");
 }
 /**
  * This is to check the timestamp of the files to be localized. Used by internal MapReduce code.
  *
  * @param conf Configuration which stores the timestamp's
  * @param timestamps comma separated list of timestamps of files. The order should be the same as
  *     the order in which the files are added.
  */
 public static void setFileTimestamps(Configuration conf, String timestamps) {
   conf.set(CACHE_FILES_TIMESTAMPS, timestamps);
 }
 /**
  * Set the configuration with the given set of files. Intended to be used by user code.
  *
  * @param files The list of files that need to be localized
  * @param conf Configuration which will be changed
  */
 public static void setCacheFiles(URI[] files, Configuration conf) {
   String sfiles = StringUtils.uriToString(files);
   conf.set(CACHE_FILES, sfiles);
 }
 /**
  * Add a file to be localized to the conf. Intended to be used by user code.
  *
  * @param uri The uri of the cache to be localized
  * @param conf Configuration to add the cache to
  */
 public static void addCacheFile(URI uri, Configuration conf) {
   String files = conf.get(CACHE_FILES);
   conf.set(CACHE_FILES, files == null ? uri.toString() : files + "," + uri.toString());
 }
Esempio n. 10
0
 public static void setSharedFileLength(Configuration conf, String length) {
   conf.set("mapred.cache.shared.files.length", length);
 }
Esempio n. 11
0
 /**
  * Add a file that has been localized to the conf.. Used by internal DistributedCache code.
  *
  * @param conf The conf to modify to contain the localized caches
  * @param str a comma separated list of local files
  */
 public static void addLocalFiles(Configuration conf, String str) {
   String files = conf.get(CACHE_LOCALFILES);
   conf.set(CACHE_LOCALFILES, files == null ? str : files + "," + str);
 }
Esempio n. 12
0
 /**
  * Add a archives to be localized to the conf. Intended to be used by user code.
  *
  * @param uri The uri of the cache to be localized
  * @param conf Configuration to add the cache to
  */
 public static void addCacheArchive(URI uri, Configuration conf) {
   String archives = conf.get(CACHE_ARCHIVES);
   conf.set(CACHE_ARCHIVES, archives == null ? uri.toString() : archives + "," + uri.toString());
 }
Esempio n. 13
0
 /**
  * Add a archive that has been localized to the conf. Used by internal DistributedCache code.
  *
  * @param conf The conf to modify to contain the localized caches
  * @param str a comma separated list of local archives
  */
 public static void addLocalArchives(Configuration conf, String str) {
   String archives = conf.get(CACHE_LOCALARCHIVES);
   conf.set(CACHE_LOCALARCHIVES, archives == null ? str : archives + "," + str);
 }
Esempio n. 14
0
 /**
  * Set the conf to contain the location for localized files. Used by internal DistributedCache
  * code.
  *
  * @param conf The conf to modify to contain the localized caches
  * @param str a comma separated list of local files
  */
 public static void setLocalFiles(Configuration conf, String str) {
   conf.set(CACHE_LOCALFILES, str);
 }
Esempio n. 15
0
 /**
  * Set the conf to contain the location for localized archives. Used by internal DistributedCache
  * code.
  *
  * @param conf The conf to modify to contain the localized caches
  * @param str a comma separated list of local archives
  */
 public static void setLocalArchives(Configuration conf, String str) {
   conf.set(CACHE_LOCALARCHIVES, str);
 }
Esempio n. 16
0
 /**
  * This is to check the timestamp of the archives to be localized
  *
  * @param conf Configuration which stores the timestamp's
  * @param timestamps comma separated list of timestamps of archives. The order should be the same
  *     as the order in which the archives are added.
  */
 public static void setArchiveTimestamps(Configuration conf, String timestamps) {
   conf.set("mapred.cache.archives.timestamps", timestamps);
 }
Esempio n. 17
0
 /**
  * Set the conf to contain the location for localized files
  *
  * @param conf The conf to modify to contain the localized caches
  * @param str a comma separated list of local files
  */
 public static void setLocalSharedFiles(Configuration conf, String str) {
   conf.set("mapred.cache.shared.localFiles", str);
 }
Esempio n. 18
0
 /**
  * This is to check the timestamp of the files to be localized
  *
  * @param conf Configuration which stores the timestamp's
  * @param timestamps comma separated list of timestamps of files. The order should be the same as
  *     the order in which the files are added.
  */
 public static void setFileTimestamps(Configuration conf, String timestamps) {
   conf.set("mapred.cache.files.timestamps", timestamps);
 }
Esempio n. 19
0
 /**
  * Add a file to be localized to the conf
  *
  * @param uri The uri of the cache to be localized
  * @param conf Configuration to add the cache to
  */
 public static void addCacheFile(URI uri, Configuration conf) {
   String files = conf.get("mapred.cache.files");
   conf.set("mapred.cache.files", files == null ? uri.toString() : files + "," + uri.toString());
 }
Esempio n. 20
0
 /**
  * Set the conf to contain the location for localized archives
  *
  * @param conf The conf to modify to contain the localized caches
  * @param str a comma separated list of local archives
  */
 public static void setLocalArchives(Configuration conf, String str) {
   conf.set("mapred.cache.localArchives", str);
 }
Esempio n. 21
0
 /**
  * This is to check the timestamp of the archives to be localized. Used by internal MapReduce
  * code.
  *
  * @param conf Configuration which stores the timestamp's
  * @param timestamps comma separated list of timestamps of archives. The order should be the same
  *     as the order in which the archives are added.
  */
 public static void setArchiveTimestamps(Configuration conf, String timestamps) {
   conf.set(CACHE_ARCHIVES_TIMESTAMPS, timestamps);
 }
Esempio n. 22
0
 /**
  * Add a archives to be localized to the conf
  *
  * @param uri The uri of the cache to be localized
  * @param conf Configuration to add the cache to
  */
 public static void addSharedCacheArchive(URI uri, Configuration conf) {
   String archives = conf.get("mapred.cache.shared.archives");
   conf.set(
       "mapred.cache.shared.archives",
       archives == null ? uri.toString() : archives + "," + uri.toString());
 }
Esempio n. 23
0
 /**
  * This method allows you to create symlinks in the current working directory of the task to all
  * the cache files/archives. Intended to be used by user code.
  *
  * @param conf the jobconf
  */
 public static void createSymlink(Configuration conf) {
   conf.set(CACHE_SYMLINK, "yes");
 }
Esempio n. 24
0
 FileDataGenNew(String HDFSMaster) {
   fsConf.set("fs.default.name", HDFSMaster);
 }
Esempio n. 25
0
 /**
  * Set the configuration with the given set of archives. Intended to be used by user code.
  *
  * @param archives The list of archives that need to be localized
  * @param conf Configuration which will be changed
  */
 public static void setCacheArchives(URI[] archives, Configuration conf) {
   String sarchives = StringUtils.uriToString(archives);
   conf.set(CACHE_ARCHIVES, sarchives);
 }