Exemplo n.º 1
0
  // TODO - Move this to MR!
  // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
  // long[], boolean[], Path[], FileType)
  private static void parseDistributedCacheArtifacts(
      Configuration conf,
      Map<String, LocalResource> localResources,
      LocalResourceType type,
      URI[] uris,
      long[] timestamps,
      long[] sizes,
      boolean visibilities[],
      Path[] pathsToPutOnClasspath)
      throws IOException {

    if (uris != null) {
      // Sanity check
      if ((uris.length != timestamps.length)
          || (uris.length != sizes.length)
          || (uris.length != visibilities.length)) {
        throw new IllegalArgumentException(
            "Invalid specification for "
                + "distributed-cache artifacts of type "
                + type
                + " :"
                + " #uris="
                + uris.length
                + " #timestamps="
                + timestamps.length
                + " #visibilities="
                + visibilities.length);
      }

      Map<String, Path> classPaths = new HashMap<String, Path>();
      if (pathsToPutOnClasspath != null) {
        for (Path p : pathsToPutOnClasspath) {
          FileSystem remoteFS = p.getFileSystem(conf);
          p =
              remoteFS.resolvePath(
                  p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()));
          classPaths.put(p.toUri().getPath().toString(), p);
        }
      }
      for (int i = 0; i < uris.length; ++i) {
        URI u = uris[i];
        Path p = new Path(u);
        FileSystem remoteFS = p.getFileSystem(conf);
        p =
            remoteFS.resolvePath(
                p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()));
        // Add URI fragment or just the filename
        Path name = new Path((null == u.getFragment()) ? p.getName() : u.getFragment());
        if (name.isAbsolute()) {
          throw new IllegalArgumentException("Resource name must be relative");
        }
        String linkName = name.toUri().getPath();
        localResources.put(
            linkName,
            BuilderUtils.newLocalResource(
                p.toUri(),
                type,
                visibilities[i] ? LocalResourceVisibility.PUBLIC : LocalResourceVisibility.PRIVATE,
                sizes[i],
                timestamps[i]));
      }
    }
  }
Exemplo n.º 2
0
 @Override
 public boolean isFileInHDFS(FileSystem fs, Path path) throws IOException {
   // In case of viewfs we need to lookup where the actual file is to know the filesystem in use.
   // resolvePath is a sure shot way of knowing which file system the file is.
   return "hdfs".equals(fs.resolvePath(path).toUri().getScheme());
 }