/**
* Returns an {@link AlluxioURI} representation for the {@link UnderFileSystem} given a base UFS
* URI, and the Alluxio path from the base.
*
* <p>The default implementation simply concatenates the path to the base URI. This should be
* overridden if a subclass needs alternate functionality.
*
* @param ufsBaseUri the base {@link AlluxioURI} in the ufs
* @param alluxioPath the path in Alluxio from the given base
* @return the UFS {@link AlluxioURI} representing the Alluxio path
*/
public AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath) {
return new AlluxioURI(
ufsBaseUri.getScheme(),
ufsBaseUri.getAuthority(),
PathUtils.concatPath(ufsBaseUri.getPath(), alluxioPath),
ufsBaseUri.getQueryMap());
}
/**
* Transforms the list of {@link AlluxioURI} in a new list of Strings, where each string is {@link
* AlluxioURI#getPath()}.
*
* @param uris the list of {@link AlluxioURI}s to be stripped
* @return a new list of strings mapping the input URIs to theri path component
*/
private List<String> stripURIList(List<AlluxioURI> uris) {
final List<String> pathStrings = new ArrayList<>(uris.size());
for (final AlluxioURI uri : uris) {
pathStrings.add(uri.getPath());
}
return pathStrings;
}
/**
* Returns an array of strings naming the files and directories in the directory denoted by this
* abstract pathname, and all of its subdirectories.
*
* <p>If this abstract pathname does not denote a directory, then this method returns {@code
* null}. Otherwise an array of strings is returned, one for each file or directory in the
* directory and its subdirectories. Names denoting the directory itself and the directory's
* parent directory are not included in the result. Each string is a path relative to the given
* directory.
*
* <p>There is no guarantee that the name strings in the resulting array will appear in any
* specific order; they are not, in particular, guaranteed to appear in alphabetical order.
*
* @param path the abstract pathname to list
* @return An array of strings naming the files and directories in the directory denoted by this
* abstract pathname and its subdirectories. The array will be empty if the directory is
* empty. Returns {@code null} if this abstract pathname does not denote a directory.
* @throws IOException if a non-Alluxio error occurs
*/
public String[] listRecursive(String path) throws IOException {
// Clean the path by creating a URI and turning it back to a string
AlluxioURI uri = new AlluxioURI(path);
path = uri.toString();
List<String> returnPaths = new ArrayList<>();
Queue<String> pathsToProcess = new ArrayDeque<>();
// We call list initially, so we can return null if the path doesn't denote a directory
String[] subpaths = list(path);
if (subpaths == null) {
return null;
} else {
for (String subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(path, subp));
}
}
while (!pathsToProcess.isEmpty()) {
String p = pathsToProcess.remove();
returnPaths.add(p.substring(path.length() + 1));
// Add all of its subpaths
subpaths = list(p);
if (subpaths != null) {
for (String subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(p, subp));
}
}
}
return returnPaths.toArray(new String[returnPaths.size()]);
}
@Test
public void listFilesTest() throws Exception {
CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(64);
HashSet<Long> ids = new HashSet<Long>();
HashSet<Long> dirIds = new HashSet<Long>();
for (int i = 0; i < 10; i++) {
AlluxioURI dir = new AlluxioURI("/i" + i);
mFsMaster.createDirectory(dir, CreateDirectoryOptions.defaults());
dirIds.add(mFsMaster.getFileId(dir));
for (int j = 0; j < 10; j++) {
ids.add(mFsMaster.createFile(dir.join("j" + j), options));
}
}
HashSet<Long> listedIds = new HashSet<>();
HashSet<Long> listedDirIds = new HashSet<>();
List<FileInfo> infoList = mFsMaster.getFileInfoList(new AlluxioURI("/"), true);
for (FileInfo info : infoList) {
long id = info.getFileId();
listedDirIds.add(id);
for (FileInfo fileInfo : mFsMaster.getFileInfoList(new AlluxioURI(info.getPath()), true)) {
listedIds.add(fileInfo.getFileId());
}
}
Assert.assertEquals(ids, listedIds);
Assert.assertEquals(dirIds, listedDirIds);
}
public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
if (depth < 1) {
return;
} else if (depth == 1 || (path.hashCode() % 10 == 0)) {
// Sometimes we want to try deleting a path when we're not all the way down, which is what
// the second condition is for.
doDelete(path);
} else {
if (concurrencyDepth > 0) {
ExecutorService executor = Executors.newCachedThreadPool();
try {
ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
for (int i = 0; i < FILES_PER_NODE; i++) {
Callable<Void> call =
(new ConcurrentDeleter(
depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i))));
futures.add(executor.submit(call));
}
for (Future<Void> f : futures) {
f.get();
}
} finally {
executor.shutdown();
}
} else {
for (int i = 0; i < FILES_PER_NODE; i++) {
exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
}
}
doDelete(path);
}
}
@Override
public boolean rename(Path src, Path dst) throws IOException {
LOG.info("rename({}, {})", src, dst);
if (mStatistics != null) {
mStatistics.incrementWriteOps(1);
}
AlluxioURI srcPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(src));
AlluxioURI dstPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(dst));
ensureExists(srcPath);
URIStatus dstStatus;
try {
dstStatus = mFileSystem.getStatus(dstPath);
} catch (IOException | AlluxioException e) {
dstStatus = null;
}
// If the destination is an existing folder, try to move the src into the folder
if (dstStatus != null && dstStatus.isFolder()) {
dstPath = dstPath.join(srcPath.getName());
}
try {
mFileSystem.rename(srcPath, dstPath);
return true;
} catch (IOException | AlluxioException e) {
LOG.error("Failed to rename {} to {}", src, dst, e);
return false;
}
}
// TODO(calvin): See if this method is still necessary
public static Pair<String, String> parse(AlluxioURI path) {
Preconditions.checkArgument(path != null, "path may not be null");
if (path.hasScheme()) {
String header = path.getScheme() + "://";
String authority = (path.hasAuthority()) ? path.getAuthority() : "";
if (header.equals(Constants.HEADER)
|| header.equals(Constants.HEADER_FT)
|| isHadoopUnderFS(header)
|| header.equals(Constants.HEADER_S3)
|| header.equals(Constants.HEADER_S3A)
|| header.equals(Constants.HEADER_S3N)
|| header.equals(Constants.HEADER_OSS)
|| header.equals(Constants.HEADER_GCS)) {
if (path.getPath().isEmpty()) {
return new Pair<>(header + authority, AlluxioURI.SEPARATOR);
} else {
return new Pair<>(header + authority, path.getPath());
}
} else if (header.equals("file://")) {
return new Pair<>(AlluxioURI.SEPARATOR, path.getPath());
}
} else if (path.isPathAbsolute()) {
return new Pair<>(AlluxioURI.SEPARATOR, path.getPath());
}
return null;
}
/**
* Constructs a new instance of {@link S3UnderFileSystem}.
*
* @param uri the {@link AlluxioURI} for this UFS
* @param conf the configuration for Alluxio
* @param awsCredentials AWS Credentials configuration for S3 Access
* @throws ServiceException when a connection to S3 could not be created
*/
public S3UnderFileSystem(AlluxioURI uri, Configuration conf, AWSCredentials awsCredentials)
throws ServiceException {
super(uri, conf);
String bucketName = uri.getHost();
mBucketName = bucketName;
Jets3tProperties props = new Jets3tProperties();
if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
props.setProperty("httpclient.proxy-autodetect", "false");
props.setProperty("httpclient.proxy-host", conf.get(Constants.UNDERFS_S3_PROXY_HOST));
props.setProperty("httpclient.proxy-port", conf.get(Constants.UNDERFS_S3_PROXY_PORT));
}
if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.https-only",
Boolean.toString(conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
}
if (conf.containsKey(Constants.UNDERFS_S3_ENDPOINT)) {
props.setProperty("s3service.s3-endpoint", conf.get(Constants.UNDERFS_S3_ENDPOINT));
if (conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.s3-endpoint-https-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTPS_PORT));
} else {
props.setProperty(
"s3service.s3-endpoint-http-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTP_PORT));
}
}
if (conf.containsKey(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
props.setProperty(
"s3service.disable-dns-buckets", conf.get(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS));
}
LOG.debug("Initializing S3 underFs with properties: {}", props.getProperties());
mClient = new RestS3Service(awsCredentials, null, null, props);
mBucketPrefix = PathUtils.normalizePath(Constants.HEADER_S3N + mBucketName, PATH_SEPARATOR);
}
/** Tests load metadata on list. */
@Test
public void loadMetadata() throws Exception {
String dirName = "loadMetaDataRoot";
String rootDir = PathUtils.concatPath(mUnderfsAddress, dirName);
mUfs.mkdirs(rootDir, true);
String rootFile1 = PathUtils.concatPath(rootDir, "file1");
createEmptyFile(rootFile1);
String rootFile2 = PathUtils.concatPath(rootDir, "file2");
createEmptyFile(rootFile2);
AlluxioURI rootAlluxioURI = new AlluxioURI("/" + dirName);
FileSystem client = mLocalAlluxioClusterResource.get().getClient();
client.listStatus(
rootAlluxioURI, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));
try {
client.createDirectory(rootAlluxioURI, CreateDirectoryOptions.defaults());
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(rootAlluxioURI), e.getMessage());
}
AlluxioURI file1URI = rootAlluxioURI.join("file1");
try {
client.createFile(file1URI, CreateFileOptions.defaults()).close();
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file1URI), e.getMessage());
}
AlluxioURI file2URI = rootAlluxioURI.join("file2");
try {
client.createFile(file2URI, CreateFileOptions.defaults()).close();
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file2URI), e.getMessage());
}
}
public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
if (depth < 1) {
return;
} else if (depth == 1) {
long fileId = mFsMaster.createFile(path, CreateFileOptions.defaults());
Assert.assertEquals(fileId, mFsMaster.getFileId(path));
// verify the user permission for file
FileInfo fileInfo = mFsMaster.getFileInfo(fileId);
Assert.assertEquals("", fileInfo.getUserName());
Assert.assertEquals(0644, (short) fileInfo.getPermission());
} else {
mFsMaster.createDirectory(path, CreateDirectoryOptions.defaults());
Assert.assertNotNull(mFsMaster.getFileId(path));
long dirId = mFsMaster.getFileId(path);
Assert.assertNotEquals(-1, dirId);
FileInfo dirInfo = mFsMaster.getFileInfo(dirId);
Assert.assertEquals("", dirInfo.getUserName());
Assert.assertEquals(0755, (short) dirInfo.getPermission());
}
if (concurrencyDepth > 0) {
ExecutorService executor = Executors.newCachedThreadPool();
try {
ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
for (int i = 0; i < FILES_PER_NODE; i++) {
Callable<Void> call =
(new ConcurrentCreator(
depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i))));
futures.add(executor.submit(call));
}
for (Future<Void> f : futures) {
f.get();
}
} finally {
executor.shutdown();
}
} else {
for (int i = 0; i < FILES_PER_NODE; i++) {
exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
}
}
}
/**
* Constructs a new Swift {@link UnderFileSystem}.
*
* @param uri the {@link AlluxioURI} for this UFS
*/
public SwiftUnderFileSystem(AlluxioURI uri) {
super(uri);
String containerName = uri.getHost();
LOG.debug("Constructor init: {}", containerName);
AccountConfig config = new AccountConfig();
if (Configuration.containsKey(Constants.SWIFT_API_KEY)) {
config.setPassword(Configuration.get(Constants.SWIFT_API_KEY));
} else if (Configuration.containsKey(Constants.SWIFT_PASSWORD_KEY)) {
config.setPassword(Configuration.get(Constants.SWIFT_PASSWORD_KEY));
}
config.setAuthUrl(Configuration.get(Constants.SWIFT_AUTH_URL_KEY));
String authMethod = Configuration.get(Constants.SWIFT_AUTH_METHOD_KEY);
if (authMethod != null) {
config.setUsername(Configuration.get(Constants.SWIFT_USER_KEY));
config.setTenantName(Configuration.get(Constants.SWIFT_TENANT_KEY));
switch (authMethod) {
case Constants.SWIFT_AUTH_KEYSTONE:
config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
break;
case Constants.SWIFT_AUTH_SWIFTAUTH:
// swiftauth authenticates directly against swift
// note: this method is supported in swift object storage api v1
config.setAuthenticationMethod(AuthenticationMethod.BASIC);
break;
default:
config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
// tempauth requires authentication header to be of the form tenant:user.
// JOSS however generates header of the form user:tenant.
// To resolve this, we switch user with tenant
config.setTenantName(Configuration.get(Constants.SWIFT_USER_KEY));
config.setUsername(Configuration.get(Constants.SWIFT_TENANT_KEY));
}
}
ObjectMapper mapper = new ObjectMapper();
mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
mContainerName = containerName;
mAccount = new AccountFactory(config).createAccount();
// Do not allow container cache to avoid stale directory listings
mAccount.setAllowContainerCaching(false);
mAccess = mAccount.authenticate();
Container container = mAccount.getContainer(containerName);
if (!container.exists()) {
container.create();
}
mContainerPrefix = Constants.HEADER_SWIFT + mContainerName + PATH_SEPARATOR;
}
public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
if (depth < 1) {
return;
} else if (depth == 1 || (depth < mDepth && path.hashCode() % 10 < 3)) {
// Sometimes we want to try renaming a path when we're not all the way down, which is what
// the second condition is for. We have to create the path in the destination up till what
// we're renaming. This might already exist, so createFile could throw a
// FileAlreadyExistsException, which we silently handle.
AlluxioURI srcPath = mRootPath.join(path);
AlluxioURI dstPath = mRootPath2.join(path);
long fileId = mFsMaster.getFileId(srcPath);
try {
CreateDirectoryOptions options = CreateDirectoryOptions.defaults().setRecursive(true);
mFsMaster.createDirectory(dstPath.getParent(), options);
} catch (FileAlreadyExistsException e) {
// This is an acceptable exception to get, since we don't know if the parent has been
// created yet by another thread.
} catch (InvalidPathException e) {
// This could happen if we are renaming something that's a child of the root.
}
mFsMaster.rename(srcPath, dstPath);
Assert.assertEquals(fileId, mFsMaster.getFileId(dstPath));
} else if (concurrencyDepth > 0) {
ExecutorService executor = Executors.newCachedThreadPool();
try {
ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
for (int i = 0; i < FILES_PER_NODE; i++) {
Callable<Void> call =
(new ConcurrentRenamer(
depth - 1,
concurrencyDepth - 1,
mRootPath,
mRootPath2,
path.join(Integer.toString(i))));
futures.add(executor.submit(call));
}
for (Future<Void> f : futures) {
f.get();
}
} finally {
executor.shutdown();
}
} else {
for (int i = 0; i < FILES_PER_NODE; i++) {
exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
}
}
}
Key(AlluxioURI uri) {
mScheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase();
mAuthority = uri.getAuthority() == null ? "" : uri.getAuthority().toLowerCase();
}