public static FileDesc loadFile(Path root, Path file, int blocSize)
throws NoSuchAlgorithmException, FileNotFoundException, IOException {
MessageDigest md = MessageDigest.getInstance("SHA-512");
MessageDigest fileMd = MessageDigest.getInstance("SHA-512");
FileDesc desc = new FileDesc(file.toString(), null, null);
List<Bloc> list = new ArrayList<Bloc>();
try (FileInputStream fis = new FileInputStream(root.resolve(file).toString())) {
byte[] buf = new byte[blocSize];
byte[] h;
int s;
while ((s = fis.read(buf)) != -1) {
int c;
while (s < buf.length && (c = fis.read()) != -1) buf[s++] = (byte) c;
fileMd.update(buf, 0, s);
// padding
byte p = 0;
while (s < buf.length) buf[s++] = ++p;
h = md.digest(buf);
Bloc bloc = new Bloc(RollingChecksum.compute(buf), new Hash(h));
list.add(bloc);
}
h = fileMd.digest();
desc.fileHash = new Hash(h);
desc.blocs = list.toArray(new Bloc[0]);
}
return desc;
}
public static String absolutePath(Path p) {
if (p == null) return "";
StringBuilder sb = new StringBuilder();
Path parentPath = p.getParent();
if (parentPath == null) return "/";
sb.append(absolutePath(parentPath));
if (sb.length() > 1) sb.append("/");
sb.append(p.getName());
return sb.toString();
}
public void setClassPath(final Path classpath) {
this.pathComponents.removeAllElements();
if (classpath != null) {
final Path actualClasspath = classpath.concatSystemClasspath("ignore");
final String[] pathElements = actualClasspath.list();
for (int i = 0; i < pathElements.length; ++i) {
try {
this.addPathElement(pathElements[i]);
} catch (BuildException ex) {
}
}
}
}
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
if (m != null) m.reset(root.relativize(dir).toString());
return m == null || m.matches() || m.hitEnd()
? FileVisitResult.CONTINUE
: FileVisitResult.SKIP_SUBTREE;
}
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (attrs.isRegularFile()) {
if (m != null) m.reset(root.relativize(file).toString());
if (m == null || m.matches())
futures.add(exec.submit(new FileLoader(root, file, blocSize)));
}
return FileVisitResult.CONTINUE;
}
public static void load(Collection<FileDesc> files, Path root, int blocSize, Pattern pattern)
throws IOException {
root = root.toAbsolutePath().normalize();
Visitor visitor = new Visitor(root, blocSize, pattern);
Files.walkFileTree(root, visitor);
for (Future<FileDesc> future : visitor.futures()) {
try {
files.add(future.get());
} catch (Exception e) {
log.error("", e);
}
}
}
/**
* open a file for writing
*
* @param hdfsPath !null path -
* @return !null stream
*/
@Override
public OutputStream openFileForWrite(final Path src) {
if (isRunningAsUser()) {
return super.openFileForWrite(src);
}
if (true) throw new UnsupportedOperationException("Fix This"); // ToDo
final FileSystem fs = getDFS();
try {
Path parent = src.getParent();
guaranteeDirectory(parent);
return FileSystem.create(fs, src, FULL_FILE_ACCESS);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* open a file for reading
*
* @param hdfsPath !null path - probably of an existing file
* @return !null stream
*/
@Override
public InputStream openFileForRead(Path src) {
if (isRunningAsUser()) {
return super.openFileForRead(src);
}
String hdfsPath = src.toString();
if (isFileNameLocal(hdfsPath)) {
try {
return new FileInputStream(hdfsPath); // better be local
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
}
if (true) throw new UnsupportedOperationException("Fix This"); // ToDo
final FileSystem fs = getDFS();
try {
return fs.open(src);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public FileDesc call() throws Exception {
return loadFile(root, root.relativize(file), blocSize);
}