// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
private static void parseDistributedCacheArtifacts(
Configuration conf,
Map<String, LocalResource> localResources,
LocalResourceType type,
URI[] uris,
long[] timestamps,
long[] sizes,
boolean visibilities[],
Path[] pathsToPutOnClasspath)
throws IOException {
if (uris != null) {
// Sanity check
if ((uris.length != timestamps.length)
|| (uris.length != sizes.length)
|| (uris.length != visibilities.length)) {
throw new IllegalArgumentException(
"Invalid specification for "
+ "distributed-cache artifacts of type "
+ type
+ " :"
+ " #uris="
+ uris.length
+ " #timestamps="
+ timestamps.length
+ " #visibilities="
+ visibilities.length);
}
Map<String, Path> classPaths = new HashMap<String, Path>();
if (pathsToPutOnClasspath != null) {
for (Path p : pathsToPutOnClasspath) {
FileSystem remoteFS = p.getFileSystem(conf);
p =
remoteFS.resolvePath(
p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()));
classPaths.put(p.toUri().getPath().toString(), p);
}
}
for (int i = 0; i < uris.length; ++i) {
URI u = uris[i];
Path p = new Path(u);
FileSystem remoteFS = p.getFileSystem(conf);
p =
remoteFS.resolvePath(
p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()));
// Add URI fragment or just the filename
Path name = new Path((null == u.getFragment()) ? p.getName() : u.getFragment());
if (name.isAbsolute()) {
throw new IllegalArgumentException("Resource name must be relative");
}
String linkName = name.toUri().getPath();
localResources.put(
linkName,
BuilderUtils.newLocalResource(
p.toUri(),
type,
visibilities[i] ? LocalResourceVisibility.PUBLIC : LocalResourceVisibility.PRIVATE,
sizes[i],
timestamps[i]));
}
}
}
@Override
public boolean isFileInHDFS(FileSystem fs, Path path) throws IOException {
// In case of viewfs we need to lookup where the actual file is to know the filesystem in use.
// resolvePath is a sure shot way of knowing which file system the file is.
return "hdfs".equals(fs.resolvePath(path).toUri().getScheme());
}