@Override
public String[] list(String path) throws IOException {
String prefix = addFolderSuffixIfNotPresent(stripContainerPrefixIfPresent(path));
prefix = prefix.equals(PATH_SEPARATOR) ? "" : prefix;
Collection<DirectoryOrObject> objects = listInternal(prefix);
Set<String> children = new HashSet<>();
final String self = stripFolderSuffixIfPresent(prefix);
boolean foundSelf = false;
for (DirectoryOrObject object : objects) {
String child = stripFolderSuffixIfPresent(object.getName());
String noPrefix = CommonUtils.stripPrefixIfPresent(child, prefix);
if (!noPrefix.equals(self)) {
children.add(noPrefix);
} else {
foundSelf = true;
}
}
if (!foundSelf) {
// Path does not exist
return null;
}
return children.toArray(new String[children.size()]);
}
/** Tests if list correctly returns file or folder names for a large directory. */
@Test
public void listLargeDirectory() throws IOException {
LargeDirectoryConfig config = prepareLargeDirectoryTest();
String[] children = config.getChildren();
// Retry for some time to allow list operation eventual consistency for S3 and GCS.
// See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
// https://cloud.google.com/storage/docs/consistency for more details.
// Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
String[] results = new String[] {};
for (int i = 0; i < 20; i++) {
results = mUfs.list(config.getTopLevelDirectory());
if (children.length == results.length) {
break;
}
CommonUtils.sleepMs(500);
}
Assert.assertEquals(children.length, results.length);
Arrays.sort(results);
for (int i = 0; i < children.length; ++i) {
Assert.assertTrue(
results[i].equals(
CommonUtils.stripPrefixIfPresent(
children[i], PathUtils.normalizePath(config.getTopLevelDirectory(), "/"))));
}
}
/**
* Creates a directory flagged file with the folder suffix.
*
* @param path the path to create a folder
* @return true if the operation was successful, false otherwise
*/
private boolean mkdirsInternal(String path) {
try {
String keyAsFolder =
addFolderSuffixIfNotPresent(
CommonUtils.stripPrefixIfPresent(path, Constants.HEADER_SWIFT));
// We do not check if a file with same name exists, i.e. a file with name
// 'swift://swift-container/path' and a folder with name 'swift://swift-container/path/'
// may both exist simultaneously
SwiftDirectClient.put(mAccess, keyAsFolder).close();
return true;
} catch (IOException e) {
LOG.error("Failed to create directory: {}", path, e);
return false;
}
}
@Override
public OutputStream create(String path, CreateOptions options) throws IOException {
LOG.debug("Create method: {}", path);
// create will attempt to create the parent directory if it does not already exist
if (!mkdirs(getParentPath(path), true)) {
// fail if the parent directory does not exist and creation was unsuccessful
LOG.error("Parent directory creation unsuccessful for {}", path);
return null;
}
final String strippedPath = CommonUtils.stripPrefixIfPresent(path, Constants.HEADER_SWIFT);
// TODO(adit): remove special handling of */_SUCCESS objects
if (strippedPath.endsWith("_SUCCESS")) {
// when path/_SUCCESS is created, there is need to create path as
// an empty object. This is required by Spark in case Spark
// accesses path directly, bypassing Alluxio
String plainName = CommonUtils.stripSuffixIfPresent(strippedPath, "_SUCCESS");
SwiftDirectClient.put(mAccess, plainName).close();
}
// We do not check if a folder with the same name exists
return SwiftDirectClient.put(mAccess, strippedPath);
}
/**
* Strips the Swift container prefix from the path if it is present. For example, for input path
* swift://my-container-name/my-path/file, the output would be my-path/file. This method will
* leave paths without a prefix unaltered, ie. my-path/file returns my-path/file.
*
* @param path the path to strip
* @return the path without the Swift container prefix
*/
private String stripContainerPrefixIfPresent(final String path) {
return CommonUtils.stripPrefixIfPresent(path, mContainerPrefix);
}