/** Tests if list correctly returns file names. */
@Test
public void list() throws IOException {
String testDirNonEmpty = PathUtils.concatPath(mUnderfsAddress, "testDirNonEmpty1");
String testDirNonEmptyChildDir = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmpty2");
String testDirNonEmptyChildFile = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmptyF");
String testDirNonEmptyChildDirFile =
PathUtils.concatPath(testDirNonEmptyChildDir, "testDirNonEmptyChildDirF");
mUfs.mkdirs(testDirNonEmpty, false);
mUfs.mkdirs(testDirNonEmptyChildDir, false);
createEmptyFile(testDirNonEmptyChildFile);
createEmptyFile(testDirNonEmptyChildDirFile);
String[] expectedResTopDir = new String[] {"testDirNonEmpty2", "testDirNonEmptyF"};
// Some file systems may prefix with a slash
String[] expectedResTopDir2 = new String[] {"/testDirNonEmpty2", "/testDirNonEmptyF"};
Arrays.sort(expectedResTopDir);
Arrays.sort(expectedResTopDir2);
String[] resTopDir = mUfs.list(testDirNonEmpty);
Arrays.sort(resTopDir);
Assert.assertTrue(
Arrays.equals(expectedResTopDir, resTopDir)
|| Arrays.equals(expectedResTopDir2, resTopDir));
Assert.assertTrue(
mUfs.list(testDirNonEmptyChildDir)[0].equals("testDirNonEmptyChildDirF")
|| mUfs.list(testDirNonEmptyChildDir)[0].equals("/testDirNonEmptyChildDirF"));
}
@Override
public boolean rename(String src, String dst) throws IOException {
if (!exists(src)) {
LOG.error("Unable to rename {} to {} because source does not exist.", src, dst);
return false;
}
if (exists(dst)) {
LOG.error("Unable to rename {} to {} because destination already exists.", src, dst);
return false;
}
// Source exists and destination does not exist
if (isFolder(src)) {
// Rename the source folder first
if (!copy(convertToFolderName(src), convertToFolderName(dst))) {
return false;
}
// Rename each child in the src folder to destination/child
String[] children = list(src);
for (String child : children) {
if (!rename(PathUtils.concatPath(src, child), PathUtils.concatPath(dst, child))) {
return false;
}
}
// Delete src and everything under src
return delete(src, true);
}
// Source is a file and Destination does not exist
return copy(src, dst) && deleteInternal(src);
}
/**
* Returns an array of strings naming the files and directories in the directory denoted by this
* abstract pathname, and all of its subdirectories.
*
* <p>If this abstract pathname does not denote a directory, then this method returns {@code
* null}. Otherwise an array of strings is returned, one for each file or directory in the
* directory and its subdirectories. Names denoting the directory itself and the directory's
* parent directory are not included in the result. Each string is a path relative to the given
* directory.
*
* <p>There is no guarantee that the name strings in the resulting array will appear in any
* specific order; they are not, in particular, guaranteed to appear in alphabetical order.
*
* @param path the abstract pathname to list
* @return An array of strings naming the files and directories in the directory denoted by this
* abstract pathname and its subdirectories. The array will be empty if the directory is
* empty. Returns {@code null} if this abstract pathname does not denote a directory.
* @throws IOException if a non-Alluxio error occurs
*/
public String[] listRecursive(String path) throws IOException {
// Clean the path by creating a URI and turning it back to a string
AlluxioURI uri = new AlluxioURI(path);
path = uri.toString();
List<String> returnPaths = new ArrayList<>();
Queue<String> pathsToProcess = new ArrayDeque<>();
// We call list initially, so we can return null if the path doesn't denote a directory
String[] subpaths = list(path);
if (subpaths == null) {
return null;
} else {
for (String subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(path, subp));
}
}
while (!pathsToProcess.isEmpty()) {
String p = pathsToProcess.remove();
returnPaths.add(p.substring(path.length() + 1));
// Add all of its subpaths
subpaths = list(p);
if (subpaths != null) {
for (String subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(p, subp));
}
}
}
return returnPaths.toArray(new String[returnPaths.size()]);
}
/**
* Tests an empty directory can be deleted. Tests a non empty directory will not be deleted if
* recursive is not specified. Tests a non empty directory will be deleted if recursive is
* specified.
*/
@Test
public void deleteDir() throws IOException {
String testDirEmpty = PathUtils.concatPath(mUnderfsAddress, "testDirEmpty");
String testDirNonEmpty = PathUtils.concatPath(mUnderfsAddress, "testDirNonEmpty1");
String testDirNonEmptyChildDir = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmpty2");
String testDirNonEmptyChildFile = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmptyF");
String testDirNonEmptyChildDirFile =
PathUtils.concatPath(testDirNonEmptyChildDir, "testDirNonEmptyChildDirF");
mUfs.mkdirs(testDirEmpty, false);
mUfs.mkdirs(testDirNonEmpty, false);
mUfs.mkdirs(testDirNonEmptyChildDir, false);
createEmptyFile(testDirNonEmptyChildFile);
createEmptyFile(testDirNonEmptyChildDirFile);
mUfs.delete(testDirEmpty, false);
Assert.assertFalse(mUfs.exists(testDirEmpty));
try {
mUfs.delete(testDirNonEmpty, false);
} catch (IOException e) {
// Some File systems may throw IOException
}
Assert.assertTrue(mUfs.exists(testDirNonEmpty));
mUfs.delete(testDirNonEmpty, true);
Assert.assertFalse(mUfs.exists(testDirNonEmpty));
Assert.assertFalse(mUfs.exists(testDirNonEmptyChildDir));
Assert.assertFalse(mUfs.exists(testDirNonEmptyChildFile));
Assert.assertFalse(mUfs.exists(testDirNonEmptyChildDirFile));
}
// Prepare directory tree for pagination tests
private LargeDirectoryConfig prepareLargeDirectoryTest() throws IOException {
final String filePrefix = "a_";
final String folderPrefix = "b_";
String topLevelDirectory = PathUtils.concatPath(mUnderfsAddress, "topLevelDir");
final int numFiles = 100;
String[] children = new String[numFiles + numFiles];
// Make top level directory
mUfs.mkdirs(topLevelDirectory, false);
// Make the children files
for (int i = 0; i < numFiles; ++i) {
children[i] = PathUtils.concatPath(topLevelDirectory, filePrefix + String.format("%04d", i));
createEmptyFile(children[i]);
}
// Make the children folders
for (int i = 0; i < numFiles; ++i) {
children[numFiles + i] =
PathUtils.concatPath(topLevelDirectory, folderPrefix + String.format("%04d", i));
mUfs.mkdirs(children[numFiles + i], false);
}
return new LargeDirectoryConfig(topLevelDirectory, children);
}
/** Tests {@link UnderFileSystem#rename(String, String)} works file to new location. */
@Test
public void renameFile() throws IOException {
String testFileSrc = PathUtils.concatPath(mUnderfsAddress, "testFileSrc");
String testFileDst = PathUtils.concatPath(mUnderfsAddress, "testFileDst");
createEmptyFile(testFileSrc);
mUfs.rename(testFileSrc, testFileDst);
Assert.assertFalse(mUfs.exists(testFileSrc));
Assert.assertTrue(mUfs.exists(testFileDst));
}
/** Tests {@link UnderFileSystem#getFileSize(String)} correctly returns the file size. */
@Test
public void getFileSize() throws IOException {
String testFileEmpty = PathUtils.concatPath(mUnderfsAddress, "testFileEmpty");
String testFileNonEmpty = PathUtils.concatPath(mUnderfsAddress, "testFileNonEmpty");
createEmptyFile(testFileEmpty);
createTestBytesFile(testFileNonEmpty);
Assert.assertEquals(mUfs.getFileSize(testFileEmpty), 0);
Assert.assertEquals(mUfs.getFileSize(testFileNonEmpty), TEST_BYTES.length);
}
/**
* Tests if {@link UnderFileSystem#isFile(String)} correctly returns true for files and false
* otherwise.
*/
@Test
public void isFile() throws IOException {
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
String testDir = PathUtils.concatPath(mUnderfsAddress, "testDir");
Assert.assertFalse(mUfs.isFile(testFile));
createEmptyFile(testFile);
mUfs.mkdirs(testDir, false);
Assert.assertTrue(mUfs.isFile(testFile));
Assert.assertFalse(mUfs.isFile(testDir));
}
/** Tests {@link UnderFileSystem#rename(String, String)} works file to a folder if supported. */
@Test
public void renameFileToFolder() throws IOException {
String testFileSrc = PathUtils.concatPath(mUnderfsAddress, "testFileSrc");
String testFileDst = PathUtils.concatPath(mUnderfsAddress, "testDirDst");
String testFileFinalDst = PathUtils.concatPath(testFileDst, "testFileSrc");
createEmptyFile(testFileSrc);
mUfs.mkdirs(testFileDst, false);
if (mUfs.rename(testFileSrc, testFileDst)) {
Assert.assertFalse(mUfs.exists(testFileSrc));
Assert.assertTrue(mUfs.exists(testFileFinalDst));
}
}
/**
* Constructs a new instance of {@link S3UnderFileSystem}.
*
* @param uri the {@link AlluxioURI} for this UFS
* @param conf the configuration for Alluxio
* @param awsCredentials AWS Credentials configuration for S3 Access
* @throws ServiceException when a connection to S3 could not be created
*/
public S3UnderFileSystem(AlluxioURI uri, Configuration conf, AWSCredentials awsCredentials)
throws ServiceException {
super(uri, conf);
String bucketName = uri.getHost();
mBucketName = bucketName;
Jets3tProperties props = new Jets3tProperties();
if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
props.setProperty("httpclient.proxy-autodetect", "false");
props.setProperty("httpclient.proxy-host", conf.get(Constants.UNDERFS_S3_PROXY_HOST));
props.setProperty("httpclient.proxy-port", conf.get(Constants.UNDERFS_S3_PROXY_PORT));
}
if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.https-only",
Boolean.toString(conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
}
if (conf.containsKey(Constants.UNDERFS_S3_ENDPOINT)) {
props.setProperty("s3service.s3-endpoint", conf.get(Constants.UNDERFS_S3_ENDPOINT));
if (conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.s3-endpoint-https-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTPS_PORT));
} else {
props.setProperty(
"s3service.s3-endpoint-http-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTP_PORT));
}
}
if (conf.containsKey(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
props.setProperty(
"s3service.disable-dns-buckets", conf.get(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS));
}
LOG.debug("Initializing S3 underFs with properties: {}", props.getProperties());
mClient = new RestS3Service(awsCredentials, null, null, props);
mBucketPrefix = PathUtils.normalizePath(Constants.HEADER_S3N + mBucketName, PATH_SEPARATOR);
}
// Creates a block file and write an increasing byte array into it
private void createBlockFile(String filename, int len) throws IOException, InvalidPathException {
UnderFileSystem ufs = UnderFileSystem.get(filename, mMasterConfiguration);
ufs.mkdirs(PathUtils.getParent(filename), true);
OutputStream out = ufs.create(filename);
out.write(BufferUtils.getIncreasingByteArray(len), 0, len);
out.close();
}
/**
* Returns an {@link AlluxioURI} representation for the {@link UnderFileSystem} given a base UFS
* URI, and the Alluxio path from the base.
*
* <p>The default implementation simply concatenates the path to the base URI. This should be
* overridden if a subclass needs alternate functionality.
*
* @param ufsBaseUri the base {@link AlluxioURI} in the ufs
* @param alluxioPath the path in Alluxio from the given base
* @return the UFS {@link AlluxioURI} representing the Alluxio path
*/
public AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath) {
return new AlluxioURI(
ufsBaseUri.getScheme(),
ufsBaseUri.getAuthority(),
PathUtils.concatPath(ufsBaseUri.getPath(), alluxioPath),
ufsBaseUri.getQueryMap());
}
/**
* Creates a new file output stream.
*
* @param path the file path
* @param options the client options
* @throws IOException if an I/O error occurs
*/
public FileOutStream(AlluxioURI path, OutStreamOptions options) throws IOException {
mUri = Preconditions.checkNotNull(path);
mNonce = IdUtils.getRandomNonNegativeLong();
mBlockSize = options.getBlockSizeBytes();
mAlluxioStorageType = options.getAlluxioStorageType();
mUnderStorageType = options.getUnderStorageType();
mContext = FileSystemContext.INSTANCE;
mPreviousBlockOutStreams = new LinkedList<BufferedBlockOutStream>();
if (mUnderStorageType.isSyncPersist()) {
updateUfsPath();
String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
// TODO(jiri): Implement collection of temporary files left behind by dead clients.
mUnderStorageOutputStream = ufs.create(tmpPath, (int) mBlockSize);
} else {
mUfsPath = null;
mUnderStorageOutputStream = null;
}
mClosed = false;
mCanceled = false;
mShouldCacheCurrentBlock = mAlluxioStorageType.isStore();
mBytesWritten = 0;
mLocationPolicy =
Preconditions.checkNotNull(
options.getLocationPolicy(),
PreconditionMessage.FILE_WRITE_LOCATION_POLICY_UNSPECIFIED);
}
/** Tests if list correctly returns file or folder names for a large directory. */
@Test
public void listLargeDirectory() throws IOException {
LargeDirectoryConfig config = prepareLargeDirectoryTest();
String[] children = config.getChildren();
// Retry for some time to allow list operation eventual consistency for S3 and GCS.
// See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
// https://cloud.google.com/storage/docs/consistency for more details.
// Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
String[] results = new String[] {};
for (int i = 0; i < 20; i++) {
results = mUfs.list(config.getTopLevelDirectory());
if (children.length == results.length) {
break;
}
CommonUtils.sleepMs(500);
}
Assert.assertEquals(children.length, results.length);
Arrays.sort(results);
for (int i = 0; i < children.length; ++i) {
Assert.assertTrue(
results[i].equals(
CommonUtils.stripPrefixIfPresent(
children[i], PathUtils.normalizePath(config.getTopLevelDirectory(), "/"))));
}
}
/** Tests a file can be deleted. */
@Test
public void deleteFile() throws IOException {
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
createEmptyFile(testFile);
mUfs.delete(testFile, false);
Assert.assertFalse(mUfs.exists(testFile));
}
/** Tests if file creation is atomic. */
@Test
public void createAtomic() throws IOException {
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
OutputStream stream = mUfs.create(testFile);
stream.write(TEST_BYTES);
Assert.assertFalse(mUfs.exists(testFile));
stream.close();
}
/** Tests {@link FileOutStream#write(byte[], int, int)}. */
@Test
public void writeTest3() throws Exception {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
writeTest3Util(new AlluxioURI(uniqPath + "/file_" + k + "_" + op.hashCode()), k, op);
}
}
}
/** Tests that a file can be created and validates the data written to it. */
@Test
public void createOpen() throws IOException {
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
createTestBytesFile(testFile);
byte[] buf = new byte[TEST_BYTES.length];
int bytesRead = mUfs.open(testFile).read(buf);
Assert.assertTrue(bytesRead == TEST_BYTES.length);
Assert.assertTrue(Arrays.equals(buf, TEST_BYTES));
}
/** Tests load metadata on list. */
@Test
public void loadMetadata() throws Exception {
String dirName = "loadMetaDataRoot";
String rootDir = PathUtils.concatPath(mUnderfsAddress, dirName);
mUfs.mkdirs(rootDir, true);
String rootFile1 = PathUtils.concatPath(rootDir, "file1");
createEmptyFile(rootFile1);
String rootFile2 = PathUtils.concatPath(rootDir, "file2");
createEmptyFile(rootFile2);
AlluxioURI rootAlluxioURI = new AlluxioURI("/" + dirName);
FileSystem client = mLocalAlluxioClusterResource.get().getClient();
client.listStatus(
rootAlluxioURI, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));
try {
client.createDirectory(rootAlluxioURI, CreateDirectoryOptions.defaults());
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(rootAlluxioURI), e.getMessage());
}
AlluxioURI file1URI = rootAlluxioURI.join("file1");
try {
client.createFile(file1URI, CreateFileOptions.defaults()).close();
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file1URI), e.getMessage());
}
AlluxioURI file2URI = rootAlluxioURI.join("file2");
try {
client.createFile(file2URI, CreateFileOptions.defaults()).close();
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file2URI), e.getMessage());
}
}
@Override
public String[] list(String path) throws IOException {
// if the path not exists, or it is a file, then should return null
if (!exists(path) || isFile(path)) {
return null;
}
// Non recursive list
path = PathUtils.normalizePath(path, PATH_SEPARATOR);
return listInternal(path, false);
}
/**
* @inheritDoc Rename will overwrite destination if it already exists
* @param source the source file or folder name
* @param destination the destination file or folder name
* @return true if succeed, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
@Override
public boolean rename(String source, String destination) throws IOException {
String strippedSourcePath = stripContainerPrefixIfPresent(source);
String strippedDestinationPath = stripContainerPrefixIfPresent(destination);
if (isDirectory(destination)) {
// If destination is a directory target is a file or folder within that directory
strippedDestinationPath =
PathUtils.concatPath(
strippedDestinationPath,
FilenameUtils.getName(stripFolderSuffixIfPresent(strippedSourcePath)));
}
if (isDirectory(source)) {
// Source is a directory
strippedSourcePath = addFolderSuffixIfNotPresent(strippedSourcePath);
strippedDestinationPath = addFolderSuffixIfNotPresent(strippedDestinationPath);
// Rename the source folder first
if (!copy(strippedSourcePath, strippedDestinationPath)) {
return false;
}
// TODO(adit): Use pagination to list large directories and merge duplicate call in delete
// Rename each child in the source folder to destination/child
String[] children = list(source);
for (String child : children) {
// TODO(adit): See how we can do this with better performance
// Recursive call
if (!rename(
PathUtils.concatPath(source, child),
PathUtils.concatPath(
mContainerPrefix, PathUtils.concatPath(strippedDestinationPath, child)))) {
return false;
}
}
// Delete source and everything under source
return delete(source, true);
}
// Source is a file and destination is also a file
return copy(strippedSourcePath, strippedDestinationPath) && delete(source, false);
}
/**
* Tests {@link UnderFileSystem#getModificationTimeMs(String)} returns a reasonably accurate time.
*/
@Test
public void getModTime() throws IOException {
long slack = 1000; // Some file systems may report nearest second.
long start = System.currentTimeMillis();
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
createTestBytesFile(testFile);
long end = System.currentTimeMillis();
long modTime = mUfs.getModificationTimeMs(testFile);
Assert.assertTrue(modTime >= start - slack);
Assert.assertTrue(modTime <= end + slack);
}
/**
* Tests {@link UnderFileSystem#mkdirs(String, boolean)} correctly creates a directory. Tests
* {@link UnderFileSystem#mkdirs(String, boolean)} correctly makes parent directories if
* createParent is specified.
*/
@Test
public void mkdirs() throws IOException {
// make sure the underfs address dir exists already
mUfs.mkdirs(mUnderfsAddress, true);
// empty lsr should be empty
Assert.assertEquals(0, mUfs.listRecursive(mUnderfsAddress).length);
String testDirTop = PathUtils.concatPath(mUnderfsAddress, "testDirTop");
String testDir1 = PathUtils.concatPath(mUnderfsAddress, "1");
String testDir2 = PathUtils.concatPath(testDir1, "2");
String testDir3 = PathUtils.concatPath(testDir2, "3");
String testDirDeep = PathUtils.concatPath(testDir3, "testDirDeep");
mUfs.mkdirs(testDirTop, false);
Assert.assertTrue(mUfs.exists(testDirTop));
mUfs.mkdirs(testDirDeep, true);
Assert.assertTrue(mUfs.exists(testDir1));
Assert.assertTrue(mUfs.exists(testDir2));
Assert.assertTrue(mUfs.exists(testDir3));
Assert.assertTrue(mUfs.exists(testDirDeep));
}
/**
* Tests writing to a file for longer than HEARTBEAT_INTERVAL_MS to make sure the sessionId
* doesn't change. Tracks [ALLUXIO-171].
*/
@Test
public void longWrite() throws Exception {
AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
final int length = 2;
FileOutStream os =
mFileSystem.createFile(
filePath, CreateFileOptions.defaults().setWriteType(WriteType.THROUGH));
os.write((byte) 0);
Thread.sleep(Configuration.getInt(PropertyKey.USER_HEARTBEAT_INTERVAL_MS) * 2);
os.write((byte) 1);
os.close();
checkWrite(filePath, UnderStorageType.SYNC_PERSIST, length, length);
}
private static String createStartAlluxioCommand(String command) {
List<String> commands = new ArrayList<>();
commands.add(String.format("echo 'Starting Alluxio with %s'", command));
if (installJavaFromUrl()) {
commands.add("export JAVA_HOME=" + Configuration.get(PropertyKey.INTEGRATION_MESOS_JDK_PATH));
commands.add("export PATH=$PATH:$JAVA_HOME/bin");
}
commands.add("mkdir conf");
commands.add("touch conf/alluxio-env.sh");
// If a jar is supplied, start Alluxio from the jar. Otherwise assume that Alluxio is already
// installed at PropertyKey.HOME.
if (installAlluxioFromUrl()) {
commands.add("rm *.tar.gz");
commands.add("mv alluxio-* alluxio");
}
String home = installAlluxioFromUrl() ? "alluxio" : Configuration.get(PropertyKey.HOME);
commands.add(
String.format("cp %s conf", PathUtils.concatPath(home, "conf", "log4j.properties")));
commands.add(PathUtils.concatPath(home, "integration", "bin", command));
return Joiner.on(" && ").join(commands);
}
/** Tests writing to a file and specify the location to be localhost. */
@Test
public void writeSpecifyLocal() throws Exception {
AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
final int length = 2;
FileOutStream os =
mFileSystem.createFile(
filePath,
CreateFileOptions.defaults()
.setWriteType(WriteType.CACHE_THROUGH)
.setLocationPolicy(new LocalFirstPolicy()));
os.write((byte) 0);
os.write((byte) 1);
os.close();
checkWrite(filePath, UnderStorageType.SYNC_PERSIST, length, length);
}
/**
* Tests if out-of-order writes are possible. Writes could be out-of-order when the following are
* both true: - a "large" write (over half the internal buffer size) follows a smaller write. -
* the "large" write does not cause the internal buffer to overflow.
*/
@Test
public void outOfOrderWrite() throws Exception {
AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
FileOutStream os =
mFileSystem.createFile(
filePath, CreateFileOptions.defaults().setWriteType(WriteType.MUST_CACHE));
// Write something small, so it is written into the buffer, and not directly to the file.
os.write((byte) 0);
// A length greater than 0.5 * BUFFER_BYTES and less than BUFFER_BYTES.
int length = (BUFFER_BYTES * 3) / 4;
// Write a large amount of data (larger than BUFFER_BYTES/2, but will not overflow the buffer.
os.write(BufferUtils.getIncreasingByteArray(1, length));
os.close();
checkWrite(filePath, UnderStorageType.NO_PERSIST, length + 1, length + 1);
}
/**
* Lists the files in the given path, the paths will be their logical names and not contain the
* folder suffix.
*
* @param path the key to list
* @param recursive if true will list children directories as well
* @return an array of the file and folder names in this directory
* @throws IOException if an I/O error occurs
*/
private String[] listInternal(String path, boolean recursive) throws IOException {
try {
path = stripPrefixIfPresent(path);
path = PathUtils.normalizePath(path, PATH_SEPARATOR);
path = path.equals(PATH_SEPARATOR) ? "" : path;
// Gets all the objects under the path, because we have no idea if there are non Alluxio
// managed "directories"
S3Object[] objs = mClient.listObjects(mBucketName, path, "");
if (recursive) {
List<String> ret = new ArrayList<>();
for (S3Object obj : objs) {
// Remove parent portion of the key
String child = getChildName(obj.getKey(), path);
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
// Only add if the path is not empty (removes results equal to the path)
if (!child.isEmpty()) {
ret.add(child);
}
}
return ret.toArray(new String[ret.size()]);
}
// Non recursive list
Set<String> children = new HashSet<String>();
for (S3Object obj : objs) {
// Remove parent portion of the key
String child = getChildName(obj.getKey(), path);
// Remove any portion after the path delimiter
int childNameIndex = child.indexOf(PATH_SEPARATOR);
child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
// Add to the set of children, the set will deduplicate.
if (!child.isEmpty()) {
children.add(child);
}
}
return children.toArray(new String[children.size()]);
} catch (ServiceException e) {
LOG.error("Failed to list path {}", path, e);
return null;
}
}
/** Tests if list recursive correctly returns all file names in all subdirectories. */
@Test
public void listRecursive() throws IOException {
String root = mUnderfsAddress;
// TODO(andrew): Should this directory be created in LocalAlluxioCluster creation code?
mUfs.mkdirs(root, true);
// Empty lsr should be empty
Assert.assertEquals(0, mUfs.listRecursive(root).length);
// Create a tree of subdirectories and files
String sub1 = PathUtils.concatPath(root, "sub1");
String sub2 = PathUtils.concatPath(root, "sub2");
String sub11 = PathUtils.concatPath(sub1, "sub11");
String file11 = PathUtils.concatPath(sub11, "file11");
String file2 = PathUtils.concatPath(sub2, "file2");
String file = PathUtils.concatPath(root, "file");
// lsr of nonexistent path should be null
Assert.assertNull(mUfs.listRecursive(sub1));
mUfs.mkdirs(sub1, false);
mUfs.mkdirs(sub2, false);
mUfs.mkdirs(sub11, false);
createEmptyFile(file11);
createEmptyFile(file2);
createEmptyFile(file);
// lsr from root should return paths relative to the root
String[] expectedResRoot = {
"sub1", "sub2", "sub1/sub11", "sub1/sub11/file11", "sub2/file2", "file"
};
String[] actualResRoot = mUfs.listRecursive(root);
Arrays.sort(expectedResRoot);
Arrays.sort(actualResRoot);
Assert.assertArrayEquals(expectedResRoot, actualResRoot);
// lsr from sub1 should return paths relative to sub1
String[] expectedResSub1 = {"sub11", "sub11/file11"};
String[] actualResSub1 = mUfs.listRecursive(sub1);
Arrays.sort(expectedResSub1);
Arrays.sort(actualResSub1);
Assert.assertArrayEquals(expectedResSub1, actualResSub1);
// lsr of file should be null
Assert.assertNull(mUfs.listRecursive(file));
}
@Override
public boolean delete(String path, boolean recursive) throws IOException {
if (!recursive) {
if (isFolder(path) && listInternal(path, false).length != 0) {
LOG.error(
"Unable to delete "
+ path
+ " because it is a non empty directory. Specify "
+ "recursive as true in order to delete non empty directories.");
return false;
}
return deleteInternal(path);
}
// Get all relevant files
String[] pathsToDelete = listInternal(path, true);
for (String pathToDelete : pathsToDelete) {
// If we fail to deleteInternal one file, stop
if (!deleteInternal(PathUtils.concatPath(path, pathToDelete))) {
LOG.error("Failed to delete path {}, aborting delete.", pathToDelete);
return false;
}
}
return deleteInternal(path);
}