public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception { if (depth < 1) { return; } else if (depth == 1 || (path.hashCode() % 10 == 0)) { // Sometimes we want to try deleting a path when we're not all the way down, which is what // the second condition is for. doDelete(path); } else { if (concurrencyDepth > 0) { ExecutorService executor = Executors.newCachedThreadPool(); try { ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE); for (int i = 0; i < FILES_PER_NODE; i++) { Callable<Void> call = (new ConcurrentDeleter( depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i)))); futures.add(executor.submit(call)); } for (Future<Void> f : futures) { f.get(); } } finally { executor.shutdown(); } } else { for (int i = 0; i < FILES_PER_NODE; i++) { exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i))); } } doDelete(path); } }
public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception { if (depth < 1) { return; } else if (depth == 1 || (depth < mDepth && path.hashCode() % 10 < 3)) { // Sometimes we want to try renaming a path when we're not all the way down, which is what // the second condition is for. We have to create the path in the destination up till what // we're renaming. This might already exist, so createFile could throw a // FileAlreadyExistsException, which we silently handle. AlluxioURI srcPath = mRootPath.join(path); AlluxioURI dstPath = mRootPath2.join(path); long fileId = mFsMaster.getFileId(srcPath); try { CreateDirectoryOptions options = CreateDirectoryOptions.defaults().setRecursive(true); mFsMaster.createDirectory(dstPath.getParent(), options); } catch (FileAlreadyExistsException e) { // This is an acceptable exception to get, since we don't know if the parent has been // created yet by another thread. } catch (InvalidPathException e) { // This could happen if we are renaming something that's a child of the root. } mFsMaster.rename(srcPath, dstPath); Assert.assertEquals(fileId, mFsMaster.getFileId(dstPath)); } else if (concurrencyDepth > 0) { ExecutorService executor = Executors.newCachedThreadPool(); try { ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE); for (int i = 0; i < FILES_PER_NODE; i++) { Callable<Void> call = (new ConcurrentRenamer( depth - 1, concurrencyDepth - 1, mRootPath, mRootPath2, path.join(Integer.toString(i)))); futures.add(executor.submit(call)); } for (Future<Void> f : futures) { f.get(); } } finally { executor.shutdown(); } } else { for (int i = 0; i < FILES_PER_NODE; i++) { exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i))); } } }
@Test public void listFilesTest() throws Exception { CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(64); HashSet<Long> ids = new HashSet<Long>(); HashSet<Long> dirIds = new HashSet<Long>(); for (int i = 0; i < 10; i++) { AlluxioURI dir = new AlluxioURI("/i" + i); mFsMaster.createDirectory(dir, CreateDirectoryOptions.defaults()); dirIds.add(mFsMaster.getFileId(dir)); for (int j = 0; j < 10; j++) { ids.add(mFsMaster.createFile(dir.join("j" + j), options)); } } HashSet<Long> listedIds = new HashSet<>(); HashSet<Long> listedDirIds = new HashSet<>(); List<FileInfo> infoList = mFsMaster.getFileInfoList(new AlluxioURI("/"), true); for (FileInfo info : infoList) { long id = info.getFileId(); listedDirIds.add(id); for (FileInfo fileInfo : mFsMaster.getFileInfoList(new AlluxioURI(info.getPath()), true)) { listedIds.add(fileInfo.getFileId()); } } Assert.assertEquals(ids, listedIds); Assert.assertEquals(dirIds, listedDirIds); }
@Override public boolean rename(Path src, Path dst) throws IOException { LOG.info("rename({}, {})", src, dst); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } AlluxioURI srcPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(src)); AlluxioURI dstPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(dst)); ensureExists(srcPath); URIStatus dstStatus; try { dstStatus = mFileSystem.getStatus(dstPath); } catch (IOException | AlluxioException e) { dstStatus = null; } // If the destination is an existing folder, try to move the src into the folder if (dstStatus != null && dstStatus.isFolder()) { dstPath = dstPath.join(srcPath.getName()); } try { mFileSystem.rename(srcPath, dstPath); return true; } catch (IOException | AlluxioException e) { LOG.error("Failed to rename {} to {}", src, dst, e); return false; } }
/** Tests load metadata on list. */ @Test public void loadMetadata() throws Exception { String dirName = "loadMetaDataRoot"; String rootDir = PathUtils.concatPath(mUnderfsAddress, dirName); mUfs.mkdirs(rootDir, true); String rootFile1 = PathUtils.concatPath(rootDir, "file1"); createEmptyFile(rootFile1); String rootFile2 = PathUtils.concatPath(rootDir, "file2"); createEmptyFile(rootFile2); AlluxioURI rootAlluxioURI = new AlluxioURI("/" + dirName); FileSystem client = mLocalAlluxioClusterResource.get().getClient(); client.listStatus( rootAlluxioURI, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always)); try { client.createDirectory(rootAlluxioURI, CreateDirectoryOptions.defaults()); Assert.fail("create is expected to fail with FileAlreadyExistsException"); } catch (FileAlreadyExistsException e) { Assert.assertEquals( ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(rootAlluxioURI), e.getMessage()); } AlluxioURI file1URI = rootAlluxioURI.join("file1"); try { client.createFile(file1URI, CreateFileOptions.defaults()).close(); Assert.fail("create is expected to fail with FileAlreadyExistsException"); } catch (FileAlreadyExistsException e) { Assert.assertEquals( ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file1URI), e.getMessage()); } AlluxioURI file2URI = rootAlluxioURI.join("file2"); try { client.createFile(file2URI, CreateFileOptions.defaults()).close(); Assert.fail("create is expected to fail with FileAlreadyExistsException"); } catch (FileAlreadyExistsException e) { Assert.assertEquals( ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file2URI), e.getMessage()); } }
public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception { if (depth < 1) { return; } else if (depth == 1) { long fileId = mFsMaster.createFile(path, CreateFileOptions.defaults()); Assert.assertEquals(fileId, mFsMaster.getFileId(path)); // verify the user permission for file FileInfo fileInfo = mFsMaster.getFileInfo(fileId); Assert.assertEquals("", fileInfo.getUserName()); Assert.assertEquals(0644, (short) fileInfo.getPermission()); } else { mFsMaster.createDirectory(path, CreateDirectoryOptions.defaults()); Assert.assertNotNull(mFsMaster.getFileId(path)); long dirId = mFsMaster.getFileId(path); Assert.assertNotEquals(-1, dirId); FileInfo dirInfo = mFsMaster.getFileInfo(dirId); Assert.assertEquals("", dirInfo.getUserName()); Assert.assertEquals(0755, (short) dirInfo.getPermission()); } if (concurrencyDepth > 0) { ExecutorService executor = Executors.newCachedThreadPool(); try { ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE); for (int i = 0; i < FILES_PER_NODE; i++) { Callable<Void> call = (new ConcurrentCreator( depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i)))); futures.add(executor.submit(call)); } for (Future<Void> f : futures) { f.get(); } } finally { executor.shutdown(); } } else { for (int i = 0; i < FILES_PER_NODE; i++) { exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i))); } } }