/**
  * The src file is under FS, and the dst is on the local disk. Copy it from FS control to the
  * local dst name. If src and dst are directories, the copyCrc parameter determines whether to
  * copy CRC files.
  */
 public void copyToLocalFile(Path src, Path dst, boolean copyCrc) throws IOException {
   if (!fs.isDirectory(src)) { // source is a file
     fs.copyToLocalFile(src, dst);
     FileSystem localFs = getLocal(getConf()).getRawFileSystem();
     if (localFs.isDirectory(dst)) {
       dst = new Path(dst, src.getName());
     }
     dst = getChecksumFile(dst);
     if (localFs.exists(dst)) { // remove old local checksum file
       localFs.delete(dst, true);
     }
     Path checksumFile = getChecksumFile(src);
     if (copyCrc && fs.exists(checksumFile)) { // copy checksum file
       fs.copyToLocalFile(checksumFile, dst);
     }
   } else {
     FileStatus[] srcs = listStatus(src);
     for (FileStatus srcFile : srcs) {
       copyToLocalFile(srcFile.getPath(), new Path(dst, srcFile.getPath().getName()), copyCrc);
     }
   }
 }
Ejemplo n.º 2
0
  public BufferedReader loadDataFromFile(String filepath, long offset) {
    try {
      Path pt = new Path(filepath);
      FileSystem fs = FileSystem.get(fsConf);
      InputStreamReader isr;
      if (fs.isDirectory(pt)) { // multiple parts
        isr = new InputStreamReader(OpenMultiplePartsWithOffset(fs, pt, offset));
      } else { // single file
        FSDataInputStream fileHandler = fs.open(pt);
        if (offset > 0) fileHandler.seek(offset);
        isr = new InputStreamReader(fileHandler);
      }

      BufferedReader reader = new BufferedReader(isr);
      if (offset > 0) reader.readLine(); // skip first line in case of seek
      return reader;
    } catch (FileNotFoundException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    }
    assert false : "Should not reach here!";
    return null;
  }