Example #1
0
  public static FileDesc loadFile(Path root, Path file, int blocSize)
      throws NoSuchAlgorithmException, FileNotFoundException, IOException {
    MessageDigest md = MessageDigest.getInstance("SHA-512");
    MessageDigest fileMd = MessageDigest.getInstance("SHA-512");

    FileDesc desc = new FileDesc(file.toString(), null, null);
    List<Bloc> list = new ArrayList<Bloc>();
    try (FileInputStream fis = new FileInputStream(root.resolve(file).toString())) {
      byte[] buf = new byte[blocSize];
      byte[] h;
      int s;
      while ((s = fis.read(buf)) != -1) {
        int c;
        while (s < buf.length && (c = fis.read()) != -1) buf[s++] = (byte) c;
        fileMd.update(buf, 0, s);

        // padding
        byte p = 0;
        while (s < buf.length) buf[s++] = ++p;
        h = md.digest(buf);
        Bloc bloc = new Bloc(RollingChecksum.compute(buf), new Hash(h));
        list.add(bloc);
      }
      h = fileMd.digest();
      desc.fileHash = new Hash(h);
      desc.blocs = list.toArray(new Bloc[0]);
    }
    return desc;
  }
 public static String absolutePath(Path p) {
   if (p == null) return "";
   StringBuilder sb = new StringBuilder();
   Path parentPath = p.getParent();
   if (parentPath == null) return "/";
   sb.append(absolutePath(parentPath));
   if (sb.length() > 1) sb.append("/");
   sb.append(p.getName());
   return sb.toString();
 }
 public void setClassPath(final Path classpath) {
   this.pathComponents.removeAllElements();
   if (classpath != null) {
     final Path actualClasspath = classpath.concatSystemClasspath("ignore");
     final String[] pathElements = actualClasspath.list();
     for (int i = 0; i < pathElements.length; ++i) {
       try {
         this.addPathElement(pathElements[i]);
       } catch (BuildException ex) {
       }
     }
   }
 }
Example #4
0
 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
     throws IOException {
   if (m != null) m.reset(root.relativize(dir).toString());
   return m == null || m.matches() || m.hitEnd()
       ? FileVisitResult.CONTINUE
       : FileVisitResult.SKIP_SUBTREE;
 }
Example #5
0
 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
   if (attrs.isRegularFile()) {
     if (m != null) m.reset(root.relativize(file).toString());
     if (m == null || m.matches())
       futures.add(exec.submit(new FileLoader(root, file, blocSize)));
   }
   return FileVisitResult.CONTINUE;
 }
Example #6
0
 public static void load(Collection<FileDesc> files, Path root, int blocSize, Pattern pattern)
     throws IOException {
   root = root.toAbsolutePath().normalize();
   Visitor visitor = new Visitor(root, blocSize, pattern);
   Files.walkFileTree(root, visitor);
   for (Future<FileDesc> future : visitor.futures()) {
     try {
       files.add(future.get());
     } catch (Exception e) {
       log.error("", e);
     }
   }
 }
 /**
  * open a file for writing
  *
  * @param hdfsPath !null path -
  * @return !null stream
  */
 @Override
 public OutputStream openFileForWrite(final Path src) {
   if (isRunningAsUser()) {
     return super.openFileForWrite(src);
   }
   if (true) throw new UnsupportedOperationException("Fix This"); // ToDo
   final FileSystem fs = getDFS();
   try {
     Path parent = src.getParent();
     guaranteeDirectory(parent);
     return FileSystem.create(fs, src, FULL_FILE_ACCESS);
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
 /**
  * open a file for reading
  *
  * @param hdfsPath !null path - probably of an existing file
  * @return !null stream
  */
 @Override
 public InputStream openFileForRead(Path src) {
   if (isRunningAsUser()) {
     return super.openFileForRead(src);
   }
   String hdfsPath = src.toString();
   if (isFileNameLocal(hdfsPath)) {
     try {
       return new FileInputStream(hdfsPath); // better be local
     } catch (FileNotFoundException e) {
       throw new RuntimeException(e);
     }
   }
   if (true) throw new UnsupportedOperationException("Fix This"); // ToDo
   final FileSystem fs = getDFS();
   try {
     return fs.open(src);
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
Example #9
0
 public FileDesc call() throws Exception {
   return loadFile(root, root.relativize(file), blocSize);
 }