public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
if (depth < 1) {
return;
} else if (depth == 1 || (path.hashCode() % 10 == 0)) {
// Sometimes we want to try deleting a path when we're not all the way down, which is what
// the second condition is for.
doDelete(path);
} else {
if (concurrencyDepth > 0) {
ExecutorService executor = Executors.newCachedThreadPool();
try {
ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
for (int i = 0; i < FILES_PER_NODE; i++) {
Callable<Void> call =
(new ConcurrentDeleter(
depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i))));
futures.add(executor.submit(call));
}
for (Future<Void> f : futures) {
f.get();
}
} finally {
executor.shutdown();
}
} else {
for (int i = 0; i < FILES_PER_NODE; i++) {
exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
}
}
doDelete(path);
}
}
public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
if (depth < 1) {
return;
} else if (depth == 1 || (depth < mDepth && path.hashCode() % 10 < 3)) {
// Sometimes we want to try renaming a path when we're not all the way down, which is what
// the second condition is for. We have to create the path in the destination up till what
// we're renaming. This might already exist, so createFile could throw a
// FileAlreadyExistsException, which we silently handle.
AlluxioURI srcPath = mRootPath.join(path);
AlluxioURI dstPath = mRootPath2.join(path);
long fileId = mFsMaster.getFileId(srcPath);
try {
CreateDirectoryOptions options = CreateDirectoryOptions.defaults().setRecursive(true);
mFsMaster.createDirectory(dstPath.getParent(), options);
} catch (FileAlreadyExistsException e) {
// This is an acceptable exception to get, since we don't know if the parent has been
// created yet by another thread.
} catch (InvalidPathException e) {
// This could happen if we are renaming something that's a child of the root.
}
mFsMaster.rename(srcPath, dstPath);
Assert.assertEquals(fileId, mFsMaster.getFileId(dstPath));
} else if (concurrencyDepth > 0) {
ExecutorService executor = Executors.newCachedThreadPool();
try {
ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
for (int i = 0; i < FILES_PER_NODE; i++) {
Callable<Void> call =
(new ConcurrentRenamer(
depth - 1,
concurrencyDepth - 1,
mRootPath,
mRootPath2,
path.join(Integer.toString(i))));
futures.add(executor.submit(call));
}
for (Future<Void> f : futures) {
f.get();
}
} finally {
executor.shutdown();
}
} else {
for (int i = 0; i < FILES_PER_NODE; i++) {
exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
}
}
}
@Test
public void listFilesTest() throws Exception {
CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(64);
HashSet<Long> ids = new HashSet<Long>();
HashSet<Long> dirIds = new HashSet<Long>();
for (int i = 0; i < 10; i++) {
AlluxioURI dir = new AlluxioURI("/i" + i);
mFsMaster.createDirectory(dir, CreateDirectoryOptions.defaults());
dirIds.add(mFsMaster.getFileId(dir));
for (int j = 0; j < 10; j++) {
ids.add(mFsMaster.createFile(dir.join("j" + j), options));
}
}
HashSet<Long> listedIds = new HashSet<>();
HashSet<Long> listedDirIds = new HashSet<>();
List<FileInfo> infoList = mFsMaster.getFileInfoList(new AlluxioURI("/"), true);
for (FileInfo info : infoList) {
long id = info.getFileId();
listedDirIds.add(id);
for (FileInfo fileInfo : mFsMaster.getFileInfoList(new AlluxioURI(info.getPath()), true)) {
listedIds.add(fileInfo.getFileId());
}
}
Assert.assertEquals(ids, listedIds);
Assert.assertEquals(dirIds, listedDirIds);
}
@Override
public boolean rename(Path src, Path dst) throws IOException {
LOG.info("rename({}, {})", src, dst);
if (mStatistics != null) {
mStatistics.incrementWriteOps(1);
}
AlluxioURI srcPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(src));
AlluxioURI dstPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(dst));
ensureExists(srcPath);
URIStatus dstStatus;
try {
dstStatus = mFileSystem.getStatus(dstPath);
} catch (IOException | AlluxioException e) {
dstStatus = null;
}
// If the destination is an existing folder, try to move the src into the folder
if (dstStatus != null && dstStatus.isFolder()) {
dstPath = dstPath.join(srcPath.getName());
}
try {
mFileSystem.rename(srcPath, dstPath);
return true;
} catch (IOException | AlluxioException e) {
LOG.error("Failed to rename {} to {}", src, dst, e);
return false;
}
}
/** Tests load metadata on list. */
@Test
public void loadMetadata() throws Exception {
String dirName = "loadMetaDataRoot";
String rootDir = PathUtils.concatPath(mUnderfsAddress, dirName);
mUfs.mkdirs(rootDir, true);
String rootFile1 = PathUtils.concatPath(rootDir, "file1");
createEmptyFile(rootFile1);
String rootFile2 = PathUtils.concatPath(rootDir, "file2");
createEmptyFile(rootFile2);
AlluxioURI rootAlluxioURI = new AlluxioURI("/" + dirName);
FileSystem client = mLocalAlluxioClusterResource.get().getClient();
client.listStatus(
rootAlluxioURI, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));
try {
client.createDirectory(rootAlluxioURI, CreateDirectoryOptions.defaults());
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(rootAlluxioURI), e.getMessage());
}
AlluxioURI file1URI = rootAlluxioURI.join("file1");
try {
client.createFile(file1URI, CreateFileOptions.defaults()).close();
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file1URI), e.getMessage());
}
AlluxioURI file2URI = rootAlluxioURI.join("file2");
try {
client.createFile(file2URI, CreateFileOptions.defaults()).close();
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file2URI), e.getMessage());
}
}
public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
if (depth < 1) {
return;
} else if (depth == 1) {
long fileId = mFsMaster.createFile(path, CreateFileOptions.defaults());
Assert.assertEquals(fileId, mFsMaster.getFileId(path));
// verify the user permission for file
FileInfo fileInfo = mFsMaster.getFileInfo(fileId);
Assert.assertEquals("", fileInfo.getUserName());
Assert.assertEquals(0644, (short) fileInfo.getPermission());
} else {
mFsMaster.createDirectory(path, CreateDirectoryOptions.defaults());
Assert.assertNotNull(mFsMaster.getFileId(path));
long dirId = mFsMaster.getFileId(path);
Assert.assertNotEquals(-1, dirId);
FileInfo dirInfo = mFsMaster.getFileInfo(dirId);
Assert.assertEquals("", dirInfo.getUserName());
Assert.assertEquals(0755, (short) dirInfo.getPermission());
}
if (concurrencyDepth > 0) {
ExecutorService executor = Executors.newCachedThreadPool();
try {
ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
for (int i = 0; i < FILES_PER_NODE; i++) {
Callable<Void> call =
(new ConcurrentCreator(
depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i))));
futures.add(executor.submit(call));
}
for (Future<Void> f : futures) {
f.get();
}
} finally {
executor.shutdown();
}
} else {
for (int i = 0; i < FILES_PER_NODE; i++) {
exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
}
}
}
