/** * internal implementation of directory creation * * @param path path to file * @return boolean file is created * @throws IOException if specified path is file instead of directory */ private boolean mkdir(Path path) throws IOException { Path absolutePath = makeAbsolute(path); if (!store.objectExists(absolutePath)) { store.createDirectory(absolutePath); } // TODO: define a consistent semantic for a directory/subdirectory // in the hadoop:swift bridge. Hadoop FS assumes that there // are files and directories, whereas SwiftFS assumes // that there are just "objects" /* FileStatus fileStatus; try { fileStatus = getFileStatus(absolutePath); if (!fileStatus.isDir()) { throw new SwiftException(String.format( "Can't make directory for path '%s' since it exists and is not a directory: %s", path, fileStatus)); } } catch (FileNotFoundException e) { if (LOG.isDebugEnabled()) { LOG.debug("Making dir '" + path + "' in Swift"); } //file is not found: it must be created store.createDirectory(absolutePath); } */ return true; }
/** * Return an array containing hostnames, offset and size of portions of the given file. For a * nonexistent file or regions, null will be returned. * * <p>This call is most helpful with DFS, where it returns hostnames of machines that contain the * given file. * * <p>The FileSystem will simply return an elt containing 'localhost'. */ @Override public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { // Check if requested file in Swift is more than 5Gb. In this case // each block has its own location -which may be determinable // from the Swift client API, depending on the remote server final FileStatus[] listOfFileBlocks = store.listSubPaths(file.getPath()); List<URI> locations = new ArrayList<URI>(); if (listOfFileBlocks.length > 1) { for (FileStatus fileStatus : listOfFileBlocks) { if (SwiftObjectPath.fromPath(uri, fileStatus.getPath()) .equals(SwiftObjectPath.fromPath(uri, file.getPath()))) { continue; } locations.addAll(store.getObjectLocation(fileStatus.getPath())); } } else { locations = store.getObjectLocation(file.getPath()); } final String[] names = new String[locations.size()]; final String[] hosts = new String[locations.size()]; int i = 0; for (URI location : locations) { hosts[i] = location.getHost(); names[i] = location.getAuthority(); i++; } return new BlockLocation[] {new BlockLocation(names, hosts, 0, file.getLen())}; }
/** * List the statuses of the files/directories in the given path if the path is a directory. * * @param f given path * @return the statuses of the files/directories in the given path * @throws IOException */ @Override public FileStatus[] listStatus(Path f) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("SwiftFileSystem.listStatus for: " + f); } return store.listSubPaths(f); }
/** * internal implementation of directory creation * * @param path path to file * @return boolean file is created * @throws IOException if specified path is file instead of directory */ private boolean mkdir(Path path) throws IOException { Path absolutePath = makeAbsolute(path); // if (!store.objectExists(absolutePath)) { // store.createDirectory(absolutePath); // } // TODO: define a consistent semantic for a directory/subdirectory // in the hadoop:swift bridge. Hadoop FS assumes that there // are files and directories, whereas SwiftFS assumes // that there are just "objects" FileStatus fileStatus; try { fileStatus = getFileStatus(absolutePath); if (!SwiftUtils.isDirectory(fileStatus)) { throw new SwiftNotDirectoryException( path, String.format(": can't mkdir since it is not a directory: %s", fileStatus)); } else { if (LOG.isDebugEnabled()) { LOG.debug("skipping mkdir(" + path + ") as it exists already"); } } } catch (FileNotFoundException e) { if (LOG.isDebugEnabled()) { LOG.debug("Making dir '" + path + "' in Swift"); } // file is not found: it must be created store.createDirectory(absolutePath); } return true; }
/** * Delete the entire tree. This is an internal one with slightly different behavior: if an entry * is missing, a {@link FileNotFoundException} is raised. This lets the caller distinguish a file * not found with other reasons for failure, so handles race conditions in recursive directory * deletes better. * * <p>The problem being addressed is: caller A requests a recursive directory of directory /dir ; * caller B requests a delete of a file /dir/file, between caller A enumerating the files * contents, and requesting a delete of /dir/file. We want to recognise the special case "directed * file is no longer there" and not convert that into a failure * * @param path the path to delete. * @param recursive if path is a directory and set to true, the directory is deleted else throws * an exception if the directory is not empty case of a file the recursive can be set to * either true or false. * @return true if the object was deleted * @throws IOException IO problems * @throws FileNotFoundException if a file/dir being deleted is not there - this includes entries * below the specified path, (if the path is a dir and recursive is true) */ private boolean innerDelete(Path path, boolean recursive) throws IOException { Path target = makeAbsolute(path); final FileStatus fileStatus; fileStatus = getFileStatus(path); if (LOG.isDebugEnabled()) { LOG.debug("Deleting path '" + path + "'"); } if (!SwiftUtils.isDirectory(fileStatus)) { // simple file: delete it if (LOG.isDebugEnabled()) { LOG.debug("Deleting simple file '" + path + "'"); } store.deleteObject(target); } else { // it's a directory if (LOG.isDebugEnabled()) { LOG.debug("Deleting directory '" + path + "'"); } // get all entries List<FileStatus> children = store.listDirectory(target, true, true); // look to see if there are now any children if (!children.isEmpty() && !recursive) { // if there are children, unless this is a recursive operation, fail immediately throw new SwiftOperationFailedException("Directory " + path + " is not empty."); } // delete the children for (FileStatus child : children) { Path childPath = child.getPath(); try { store.deleteObject(childPath); } catch (FileNotFoundException e) { // the path went away -race conditions. // do not fail, as the outcome is still OK. LOG.info("Path " + childPath + " is no longer present"); } } // here any children that existed have been deleted // so rm the directory (which is a no-op for /) store.rmdir(target); } return true; }
/** * Renames Path src to Path dst. On swift this uses copy-and-delete and <i>is not atomic</i>. * * @param src path * @param dst path * @return true if directory renamed, false otherwise * @throws IOException on problems */ @Override public boolean rename(Path src, Path dst) throws IOException { try { store.rename(makeAbsolute(src), makeAbsolute(dst)); // success return true; } catch (SwiftOperationFailedException e) { // downgrade to a failure return false; } catch (FileNotFoundException e) { // downgrade to a failure return false; } }
/** * Delete a file or directory * * @param path the path to delete. * @param recursive if path is a directory and set to true, the directory is deleted else throws * an exception if the directory is not empty case of a file the recursive can be set to * either true or false. * @return true if a file was found and deleted * @throws IOException */ @Override public boolean delete(Path path, boolean recursive) throws IOException { LOG.debug("SwiftFileSystem.delete"); Path absolutePath = makeAbsolute(path); final FileStatus fileStatus; try { fileStatus = getFileStatus(path); } catch (FileNotFoundException e) { if (LOG.isDebugEnabled()) { LOG.debug("Delete called for '" + path + "' but file does not exist, so returning false"); } return false; } if (!fileStatus.isDir()) { // simple file: delete it if (LOG.isDebugEnabled()) { LOG.debug("Deleting file '" + path + "'"); } store.deleteObject(absolutePath); } else { // it's a directory if (LOG.isDebugEnabled()) { LOG.debug("Deleting directory '" + path + "'"); } FileStatus[] contents = listStatus(absolutePath); if (contents == null) { // the directory went away during the non-atomic stages of the operation. // Return false as it was not this thread doing the deletion. return false; } if ((contents.length != 0) && (!recursive)) { throw new IOException("Directory " + path.toString() + " is not empty."); } for (FileStatus p : contents) { if (!delete(p.getPath(), recursive)) { return false; } } } return true; }
/** * default class initialization * * @param fsuri path to Swift * @param conf Hadoop configuration * @throws IOException */ @Override public void initialize(URI fsuri, Configuration conf) throws IOException { super.initialize(fsuri, conf); setConf(conf); if (store == null) { store = new SwiftNativeFileSystemStore(); } this.uri = fsuri; this.workingDir = new Path("/user", System.getProperty("user.name")) .makeQualified(uri, new Path(System.getProperty("user.name"))); if (LOG.isDebugEnabled()) { LOG.debug( "Initializing SwiftNativeFileSystem against URI " + uri + " and working dir " + workingDir); } store.initialize(uri, conf); LOG.debug("SwiftFileSystem initialized"); }
/** * default class initialization * * @param fsuri path to Swift * @param conf Hadoop configuration * @throws IOException */ @Override public void initialize(URI fsuri, Configuration conf) throws IOException { super.initialize(fsuri, conf); setConf(conf); if (store == null) { store = new SwiftNativeFileSystemStore(); } this.uri = fsuri; // the URI given maps to the x-ref in the config, that is retained // for visibility/comparison, but behind the scenes it is converted // into the references relative URI.create(String.format("%s://%s:%d/", fsuri.getScheme(), fsuri.getHost(), fsuri.getPort())); this.workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this); if (LOG.isDebugEnabled()) { LOG.debug( "Initializing SwiftNativeFileSystem against URI " + uri + " and working dir " + workingDir); } store.initialize(uri, conf); LOG.debug("SwiftFileSystem initialized"); }
/** * internal implementation of directory creation * * @param path path to file * @return boolean file is created * @throws IOException if specified path is file instead of directory */ private boolean mkdir(Path path) throws IOException { Path absolutePath = makeAbsolute(path); FileStatus fileStatus; try { fileStatus = getFileStatus(absolutePath); if (!SwiftUtils.isDirectory(fileStatus)) { throw new SwiftNotDirectoryException( path, String.format(": can't mkdir since it is not a directory: %s", fileStatus)); } else { if (LOG.isDebugEnabled()) { LOG.debug("skipping mkdir(" + path + ") as it exists already"); } } } catch (FileNotFoundException e) { if (LOG.isDebugEnabled()) { LOG.debug("Making dir '" + path + "' in Swift"); } // file is not found: it must be created store.createDirectory(absolutePath); } return true; }
/** * Renames Path src to Path dst. On swift this uses copy-and-delete and <i>is not atomic</i>. * * @param src path * @param dst path * @return true if directory renamed, false otherwise * @throws IOException on problems */ @Override public boolean rename(Path src, Path dst) throws IOException { return store.renameDirectory(src, dst); }
/** * Return a file status object that represents the path. * * @param f The path we want information from * @return a FileStatus object */ @Override public FileStatus getFileStatus(Path f) throws IOException { final FileStatus objectMetadata = store.getObjectMetadata(f); return objectMetadata; }
/** * Delete the entire tree. This is an internal one with slightly different behavior: if an entry * is missing, a {@link FileNotFoundException} is raised. This lets the caller distinguish a file * not found with other reasons for failure, so handles race conditions in recursive directory * deletes better. * * <p>The problem being addressed is: caller A requests a recursive directory of directory /dir ; * caller B requests a delete of a file /dir/file, between caller A enumerating the files * contents, and requesting a delete of /dir/file. We want to recognise the special case "directed * file is no longer there" and not convert that into a failure * * @param path the path to delete. * @param recursive if path is a directory and set to true, the directory is deleted else throws * an exception if the directory is not empty case of a file the recursive can be set to * either true or false. * @return true if the object was deleted * @throws IOException IO problems * @throws FileNotFoundException if a file/dir being deleted is not there - this includes entries * below the specified path, (if the path is a dir and recursive is true) */ private boolean innerDelete(Path path, boolean recursive) throws IOException { Path absolutePath = makeAbsolute(path); final FileStatus fileStatus; fileStatus = getFileStatus(path); if (LOG.isDebugEnabled()) { LOG.debug("Deleting path '" + path + "'"); } if (!SwiftUtils.isDirectory(fileStatus)) { // simple file: delete it if (LOG.isDebugEnabled()) { LOG.debug("Deleting simple file '" + path + "'"); } store.deleteObject(absolutePath); } else { // it's a directory if (LOG.isDebugEnabled()) { LOG.debug("Deleting directory '" + path + "'"); } FileStatus[] contents = listStatus(absolutePath); if (contents == null) { // the directory went away during the non-atomic stages of the operation. // Return false as it was not this thread doing the deletion. if (LOG.isDebugEnabled()) { LOG.debug("Path '" + path + "' has no status -it has 'gone away'"); } return false; } // now build a list without ourselves in it Path dirPath = fileStatus.getPath(); if (LOG.isDebugEnabled()) { LOG.debug("Found " + contents.length + " child entries under " + dirPath); } ArrayList<FileStatus> children = new ArrayList<FileStatus>(contents.length); for (FileStatus child : contents) { if (!(child.getPath().equals(dirPath))) { if (LOG.isDebugEnabled()) { LOG.debug(child.toString()); } children.add(child); } else { if (LOG.isDebugEnabled()) { LOG.debug("skipping own entry"); } } } // look to see if there are now any children if (!children.isEmpty() && !recursive) { // if there are unless this is a recursive operation, fail immediately throw new SwiftException("Directory " + path + " is not empty."); } // delete the children for (FileStatus child : children) { Path childPath = child.getPath(); try { if (!innerDelete(childPath, true)) { if (LOG.isDebugEnabled()) { LOG.debug("Failed to recursively delete '" + childPath + "'"); } return false; } } catch (FileNotFoundException e) { // the path went away -race conditions. // do not fail, as the outcome is still OK. LOG.info("Path " + childPath + " is no longer present"); } } // here any children that existed have been deleted // so rm the directory store.rmdir(absolutePath); } return true; }
/** * Return a file status object that represents the path. * * @param f The path we want information from * @return a FileStatus object */ @Override public FileStatus getFileStatus(Path f) throws IOException { return store.getObjectMetadata(f); }