/** * If <code>f</code> is a file, this method will make a single call to S3. If <code>f</code> is a * directory, this method will make a maximum of (<i>n</i> / 1000) + 2 calls to S3, where <i>n</i> * is the total number of files and directories contained directly in <code>f</code>. */ @Override public FileStatus[] listStatus(Path f) throws IOException { Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); if (key.length() > 0) { FileMetadata meta = store.retrieveMetadata(key); if (meta != null) { return new FileStatus[] {newFile(meta, absolutePath)}; } } URI pathUri = absolutePath.toUri(); Set<FileStatus> status = new TreeSet<FileStatus>(); String priorLastKey = null; do { PartialListing listing = store.list(key, S3_MAX_LISTING_LENGTH, priorLastKey, false); for (FileMetadata fileMetadata : listing.getFiles()) { Path subpath = keyToPath(fileMetadata.getKey()); String relativePath = pathUri.relativize(subpath.toUri()).getPath(); if (fileMetadata.getKey().equals(key + "/")) { // this is just the directory we have been asked to list } else if (relativePath.endsWith(FOLDER_SUFFIX)) { status.add( newDirectory( new Path( absolutePath, relativePath.substring(0, relativePath.indexOf(FOLDER_SUFFIX))))); } else { status.add(newFile(fileMetadata, subpath)); } } for (String commonPrefix : listing.getCommonPrefixes()) { Path subpath = keyToPath(commonPrefix); String relativePath = pathUri.relativize(subpath.toUri()).getPath(); status.add(newDirectory(new Path(absolutePath, relativePath))); } priorLastKey = listing.getPriorLastKey(); } while (priorLastKey != null); if (status.isEmpty() && key.length() > 0 && store.retrieveMetadata(key + FOLDER_SUFFIX) == null) { throw new FileNotFoundException("File " + f + " does not exist."); } return status.toArray(new FileStatus[status.size()]); }
@Override public FileStatus getFileStatus(Path f) throws IOException { Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); if (key.length() == 0) { // root always exists return newDirectory(absolutePath); } if (LOG.isDebugEnabled()) { LOG.debug("getFileStatus retrieving metadata for key '" + key + "'"); } FileMetadata meta = store.retrieveMetadata(key); if (meta != null) { if (LOG.isDebugEnabled()) { LOG.debug("getFileStatus returning 'file' for key '" + key + "'"); } return newFile(meta, absolutePath); } if (store.retrieveMetadata(key + FOLDER_SUFFIX) != null) { if (LOG.isDebugEnabled()) { LOG.debug( "getFileStatus returning 'directory' for key '" + key + "' as '" + key + FOLDER_SUFFIX + "' exists"); } return newDirectory(absolutePath); } if (LOG.isDebugEnabled()) { LOG.debug("getFileStatus listing key '" + key + "'"); } PartialListing listing = store.list(key, 1); if (listing.getFiles().length > 0 || listing.getCommonPrefixes().length > 0) { if (LOG.isDebugEnabled()) { LOG.debug("getFileStatus returning 'directory' for key '" + key + "' as it has contents"); } return newDirectory(absolutePath); } if (LOG.isDebugEnabled()) { LOG.debug("getFileStatus could not find key '" + key + "'"); } throw new FileNotFoundException("No such file or directory '" + absolutePath + "'"); }
@Override public synchronized void seek(long pos) throws IOException { in.close(); LOG.info("Opening key '" + key + "' for reading at position '" + pos + "'"); in = store.retrieve(key, pos); this.pos = pos; }
@Override public boolean delete(Path f, boolean recurse) throws IOException { FileStatus status; try { status = getFileStatus(f); } catch (FileNotFoundException e) { if (LOG.isDebugEnabled()) { LOG.debug("Delete called for '" + f + "' but file does not exist, so returning false"); } return false; } Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); if (status.isDirectory()) { if (!recurse && listStatus(f).length > 0) { throw new IOException( "Can not delete " + f + " at is a not empty directory and recurse option is false"); } createParent(f); if (LOG.isDebugEnabled()) { LOG.debug("Deleting directory '" + f + "'"); } String priorLastKey = null; do { PartialListing listing = store.list(key, S3_MAX_LISTING_LENGTH, priorLastKey, true); for (FileMetadata file : listing.getFiles()) { store.delete(file.getKey()); } priorLastKey = listing.getPriorLastKey(); } while (priorLastKey != null); try { store.delete(key + FOLDER_SUFFIX); } catch (FileNotFoundException e) { // this is fine, we don't require a marker } } else { if (LOG.isDebugEnabled()) { LOG.debug("Deleting file '" + f + "'"); } createParent(f); store.delete(key); } return true; }
// rename() and delete() use this method to ensure that the parent directory // of the source does not vanish. private void createParent(Path path) throws IOException { Path parent = path.getParent(); if (parent != null) { String key = pathToKey(makeAbsolute(parent)); if (key.length() > 0) { store.storeEmptyFile(key + FOLDER_SUFFIX); } } }
@Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); if (store == null) { store = createDefaultStore(conf); } store.initialize(uri, conf); setConf(conf); this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); this.workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this); }
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { FileStatus fs = getFileStatus(f); // will throw if the file doesn't exist if (fs.isDirectory()) { throw new IOException("'" + f + "' is a directory"); } LOG.info("Opening '" + f + "' for reading"); Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); return new FSDataInputStream( new BufferedFSInputStream( new NativeS3FsInputStream(store, statistics, store.retrieve(key), key), bufferSize)); }
private boolean mkdir(Path f) throws IOException { try { FileStatus fileStatus = getFileStatus(f); if (!fileStatus.isDir()) { throw new IOException( String.format("Can't make directory for path '%s' since it is a file.", f)); } } catch (FileNotFoundException e) { LOG.debug("Making dir '" + f + "' in S3"); String key = pathToKey(f) + FOLDER_SUFFIX; store.storeEmptyFile(key); } return true; }
@Override public boolean rename(Path src, Path dst) throws IOException { String srcKey = pathToKey(makeAbsolute(src)); if (srcKey.length() == 0) { // Cannot rename root of file system return false; } final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - "; // Figure out the final destination String dstKey; try { boolean dstIsFile = getFileStatus(dst).isFile(); if (dstIsFile) { if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "returning false as dst is an already existing file"); } return false; } else { if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "using dst as output directory"); } dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName()))); } } catch (FileNotFoundException e) { if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "using dst as output destination"); } dstKey = pathToKey(makeAbsolute(dst)); try { if (getFileStatus(dst.getParent()).isFile()) { if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "returning false as dst parent exists and is a file"); } return false; } } catch (FileNotFoundException ex) { if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "returning false as dst parent does not exist"); } return false; } } boolean srcIsFile; try { srcIsFile = getFileStatus(src).isFile(); } catch (FileNotFoundException e) { if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "returning false as src does not exist"); } return false; } if (srcIsFile) { if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "src is file, so doing copy then delete in S3"); } store.copy(srcKey, dstKey); store.delete(srcKey); } else { if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "src is directory, so copying contents"); } store.storeEmptyFile(dstKey + FOLDER_SUFFIX); List<String> keysToDelete = new ArrayList<String>(); String priorLastKey = null; do { PartialListing listing = store.list(srcKey, S3_MAX_LISTING_LENGTH, priorLastKey, true); for (FileMetadata file : listing.getFiles()) { keysToDelete.add(file.getKey()); store.copy(file.getKey(), dstKey + file.getKey().substring(srcKey.length())); } priorLastKey = listing.getPriorLastKey(); } while (priorLastKey != null); if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "all files in src copied, now removing src files"); } for (String key : keysToDelete) { store.delete(key); } try { store.delete(srcKey + FOLDER_SUFFIX); } catch (FileNotFoundException e) { // this is fine, we don't require a marker } if (LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "done"); } } return true; }