@Test public void deleteDirectoryWithDirectoriesTest2() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); mFsMaster.createDirectory( new AlluxioURI("/testFolder/testFolder2"), CreateDirectoryOptions.defaults()); long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile"), CreateFileOptions.defaults()); long fileId2 = mFsMaster.createFile( new AlluxioURI("/testFolder/testFolder2/testFile2"), CreateFileOptions.defaults()); Assert.assertEquals(1, mFsMaster.getFileId(new AlluxioURI("/testFolder"))); Assert.assertEquals(2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2"))); Assert.assertEquals(fileId, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile"))); Assert.assertEquals( fileId2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2/testFile2"))); try { mFsMaster.delete(new AlluxioURI("/testFolder/testFolder2"), false); Assert.fail("Deleting a nonempty directory nonrecursively should fail"); } catch (DirectoryNotEmptyException e) { Assert.assertEquals( ExceptionMessage.DELETE_NONEMPTY_DIRECTORY_NONRECURSIVE.getMessage("testFolder2"), e.getMessage()); } Assert.assertEquals(1, mFsMaster.getFileId(new AlluxioURI("/testFolder"))); Assert.assertEquals(2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2"))); Assert.assertEquals(fileId, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile"))); Assert.assertEquals( fileId2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2/testFile2"))); }
@Test public void notFileCompletionTest() throws Exception { mThrown.expect(FileDoesNotExistException.class); mFsMaster.createDirectory(new AlluxioURI("/testFile"), CreateDirectoryOptions.defaults()); CompleteFileOptions options = CompleteFileOptions.defaults(); mFsMaster.completeFile(new AlluxioURI("/testFile"), options); }
@Test public void listFilesTest() throws Exception { CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(64); HashSet<Long> ids = new HashSet<Long>(); HashSet<Long> dirIds = new HashSet<Long>(); for (int i = 0; i < 10; i++) { AlluxioURI dir = new AlluxioURI("/i" + i); mFsMaster.createDirectory(dir, CreateDirectoryOptions.defaults()); dirIds.add(mFsMaster.getFileId(dir)); for (int j = 0; j < 10; j++) { ids.add(mFsMaster.createFile(dir.join("j" + j), options)); } } HashSet<Long> listedIds = new HashSet<>(); HashSet<Long> listedDirIds = new HashSet<>(); List<FileInfo> infoList = mFsMaster.getFileInfoList(new AlluxioURI("/"), true); for (FileInfo info : infoList) { long id = info.getFileId(); listedDirIds.add(id); for (FileInfo fileInfo : mFsMaster.getFileInfoList(new AlluxioURI(info.getPath()), true)) { listedIds.add(fileInfo.getFileId()); } } Assert.assertEquals(ids, listedIds); Assert.assertEquals(dirIds, listedDirIds); }
@Test public void deleteEmptyDirectoryTest() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); Assert.assertEquals(1, mFsMaster.getFileId(new AlluxioURI("/testFolder"))); mFsMaster.delete(new AlluxioURI("/testFolder"), true); Assert.assertEquals( IdUtils.INVALID_FILE_ID, mFsMaster.getFileId(new AlluxioURI("/testFolder"))); }
@Test public void createDirectoryTest() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); FileInfo fileInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder"))); Assert.assertTrue(fileInfo.isFolder()); Assert.assertEquals("", fileInfo.getUserName()); Assert.assertEquals(0755, (short) fileInfo.getPermission()); }
@Test public void deleteDirectoryWithDirectoriesTest() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); mFsMaster.createDirectory( new AlluxioURI("/testFolder/testFolder2"), CreateDirectoryOptions.defaults()); long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile"), CreateFileOptions.defaults()); long fileId2 = mFsMaster.createFile( new AlluxioURI("/testFolder/testFolder2/testFile2"), CreateFileOptions.defaults()); Assert.assertEquals(1, mFsMaster.getFileId(new AlluxioURI("/testFolder"))); Assert.assertEquals(2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2"))); Assert.assertEquals(fileId, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile"))); Assert.assertEquals( fileId2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2/testFile2"))); mFsMaster.delete(new AlluxioURI("/testFolder"), true); Assert.assertEquals( IdUtils.INVALID_FILE_ID, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2/testFile2"))); }
/** Tests that deleting a file from a folder updates the folder's last modification time. */ @Test public void lastModificationTimeDeleteTest() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); mFsMaster.createFile(new AlluxioURI("/testFolder/testFile"), CreateFileOptions.defaults()); long folderId = mFsMaster.getFileId(new AlluxioURI("/testFolder")); long modificationTimeBeforeDelete = mFsMaster.getFileInfo(folderId).getLastModificationTimeMs(); CommonUtils.sleepMs(2); mFsMaster.delete(new AlluxioURI("/testFolder/testFile"), true); long modificationTimeAfterDelete = mFsMaster.getFileInfo(folderId).getLastModificationTimeMs(); Assert.assertTrue(modificationTimeBeforeDelete < modificationTimeAfterDelete); }
@Test public void renameToDeeper() throws Exception { CreateFileOptions createFileOptions = CreateFileOptions.defaults().setRecursive(true); CreateDirectoryOptions createDirectoryOptions = CreateDirectoryOptions.defaults().setRecursive(true); mThrown.expect(InvalidPathException.class); mFsMaster.createDirectory(new AlluxioURI("/testDir1/testDir2"), createDirectoryOptions); mFsMaster.createFile( new AlluxioURI("/testDir1/testDir2/testDir3/testFile3"), createFileOptions); mFsMaster.rename( new AlluxioURI("/testDir1/testDir2"), new AlluxioURI("/testDir1/testDir2/testDir3/testDir4")); }
@Test public void lastModificationTimeCreateFileTest() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); long opTimeMs = TEST_CURRENT_TIME; CreateFileOptions options = CreateFileOptions.defaults().setOperationTimeMs(opTimeMs); try (LockedInodePath inodePath = mInodeTree.lockInodePath( new AlluxioURI("/testFolder/testFile"), InodeTree.LockMode.WRITE)) { mFsMaster.createFileInternal(inodePath, options); } FileInfo folderInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder"))); Assert.assertEquals(opTimeMs, folderInfo.getLastModificationTimeMs()); }
@Test public void createFilePerfTest() throws Exception { for (int k = 0; k < 200; k++) { CreateDirectoryOptions options = CreateDirectoryOptions.defaults().setRecursive(true); mFsMaster.createDirectory( new AlluxioURI("/testFile").join(Constants.MASTER_COLUMN_FILE_PREFIX + k).join("0"), options); } for (int k = 0; k < 200; k++) { mFsMaster.getFileInfo( mFsMaster.getFileId( new AlluxioURI("/testFile").join(Constants.MASTER_COLUMN_FILE_PREFIX + k).join("0"))); } }
@Test public void ttlCreateFileTest() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); long ttl = 100; CreateFileOptions options = CreateFileOptions.defaults().setTtl(ttl); try (LockedInodePath inodePath = mInodeTree.lockInodePath( new AlluxioURI("/testFolder/testFile"), InodeTree.LockMode.WRITE)) { mFsMaster.createFileInternal(inodePath, options); } FileInfo folderInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile"))); Assert.assertEquals(ttl, folderInfo.getTtl()); }
public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception { if (depth < 1) { return; } else if (depth == 1 || (depth < mDepth && path.hashCode() % 10 < 3)) { // Sometimes we want to try renaming a path when we're not all the way down, which is what // the second condition is for. We have to create the path in the destination up till what // we're renaming. This might already exist, so createFile could throw a // FileAlreadyExistsException, which we silently handle. AlluxioURI srcPath = mRootPath.join(path); AlluxioURI dstPath = mRootPath2.join(path); long fileId = mFsMaster.getFileId(srcPath); try { CreateDirectoryOptions options = CreateDirectoryOptions.defaults().setRecursive(true); mFsMaster.createDirectory(dstPath.getParent(), options); } catch (FileAlreadyExistsException e) { // This is an acceptable exception to get, since we don't know if the parent has been // created yet by another thread. } catch (InvalidPathException e) { // This could happen if we are renaming something that's a child of the root. } mFsMaster.rename(srcPath, dstPath); Assert.assertEquals(fileId, mFsMaster.getFileId(dstPath)); } else if (concurrencyDepth > 0) { ExecutorService executor = Executors.newCachedThreadPool(); try { ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE); for (int i = 0; i < FILES_PER_NODE; i++) { Callable<Void> call = (new ConcurrentRenamer( depth - 1, concurrencyDepth - 1, mRootPath, mRootPath2, path.join(Integer.toString(i)))); futures.add(executor.submit(call)); } for (Future<Void> f : futures) { f.get(); } } finally { executor.shutdown(); } } else { for (int i = 0; i < FILES_PER_NODE; i++) { exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i))); } } }
@Test public void clientFileInfoDirectoryTest() throws Exception { AlluxioURI path = new AlluxioURI("/testFolder"); mFsMaster.createDirectory(path, CreateDirectoryOptions.defaults()); long fileId = mFsMaster.getFileId(path); FileInfo fileInfo = mFsMaster.getFileInfo(fileId); Assert.assertEquals("testFolder", fileInfo.getName()); Assert.assertEquals(1, fileInfo.getFileId()); Assert.assertEquals(0, fileInfo.getLength()); Assert.assertFalse(fileInfo.isCacheable()); Assert.assertTrue(fileInfo.isCompleted()); Assert.assertTrue(fileInfo.isFolder()); Assert.assertFalse(fileInfo.isPersisted()); Assert.assertFalse(fileInfo.isPinned()); Assert.assertEquals("", fileInfo.getUserName()); Assert.assertEquals(0755, (short) fileInfo.getPermission()); }
@Test public void lsTest() throws Exception { CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(64); for (int i = 0; i < 10; i++) { mFsMaster.createDirectory(new AlluxioURI("/i" + i), CreateDirectoryOptions.defaults()); for (int j = 0; j < 10; j++) { mFsMaster.createFile(new AlluxioURI("/i" + i + "/j" + j), options); } } Assert.assertEquals(1, mFsMaster.getFileInfoList(new AlluxioURI("/i0/j0"), true).size()); for (int i = 0; i < 10; i++) { Assert.assertEquals(10, mFsMaster.getFileInfoList(new AlluxioURI("/i" + i), true).size()); } Assert.assertEquals(10, mFsMaster.getFileInfoList(new AlluxioURI("/"), true).size()); }
public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception { if (depth < 1) { return; } else if (depth == 1) { long fileId = mFsMaster.createFile(path, CreateFileOptions.defaults()); Assert.assertEquals(fileId, mFsMaster.getFileId(path)); // verify the user permission for file FileInfo fileInfo = mFsMaster.getFileInfo(fileId); Assert.assertEquals("", fileInfo.getUserName()); Assert.assertEquals(0644, (short) fileInfo.getPermission()); } else { mFsMaster.createDirectory(path, CreateDirectoryOptions.defaults()); Assert.assertNotNull(mFsMaster.getFileId(path)); long dirId = mFsMaster.getFileId(path); Assert.assertNotEquals(-1, dirId); FileInfo dirInfo = mFsMaster.getFileInfo(dirId); Assert.assertEquals("", dirInfo.getUserName()); Assert.assertEquals(0755, (short) dirInfo.getPermission()); } if (concurrencyDepth > 0) { ExecutorService executor = Executors.newCachedThreadPool(); try { ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE); for (int i = 0; i < FILES_PER_NODE; i++) { Callable<Void> call = (new ConcurrentCreator( depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i)))); futures.add(executor.submit(call)); } for (Future<Void> f : futures) { f.get(); } } finally { executor.shutdown(); } } else { for (int i = 0; i < FILES_PER_NODE; i++) { exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i))); } } }
@Test public void ttlExpiredCreateFileTest() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); long ttl = 1; CreateFileOptions options = CreateFileOptions.defaults().setTtl(ttl); long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile1"), options); FileInfo folderInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile1"))); Assert.assertEquals(fileId, folderInfo.getFileId()); Assert.assertEquals(ttl, folderInfo.getTtl()); // Sleep for the ttl expiration. CommonUtils.sleepMs(2 * TTL_CHECKER_INTERVAL_MS); Assert.assertTrue( HeartbeatScheduler.await(HeartbeatContext.MASTER_TTL_CHECK, 10, TimeUnit.SECONDS)); HeartbeatScheduler.schedule(HeartbeatContext.MASTER_TTL_CHECK); Assert.assertTrue( HeartbeatScheduler.await(HeartbeatContext.MASTER_TTL_CHECK, 10, TimeUnit.SECONDS)); mThrown.expect(FileDoesNotExistException.class); mFsMaster.getFileInfo(fileId); }
@Test public void lastModificationTimeRenameTest() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile1"), CreateFileOptions.defaults()); long opTimeMs = TEST_CURRENT_TIME; try (InodePathPair inodePathPair = mInodeTree.lockInodePathPair( new AlluxioURI("/testFolder/testFile1"), InodeTree.LockMode.WRITE_PARENT, new AlluxioURI("/testFolder/testFile2"), InodeTree.LockMode.WRITE)) { LockedInodePath srcPath = inodePathPair.getFirst(); LockedInodePath dstPath = inodePathPair.getSecond(); mFsMaster.renameInternal(srcPath, dstPath, true, opTimeMs); } FileInfo folderInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder"))); Assert.assertEquals(opTimeMs, folderInfo.getLastModificationTimeMs()); }
@Test public void ttlRenameTest() throws Exception { mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults()); long ttl = 1; CreateFileOptions options = CreateFileOptions.defaults().setTtl(ttl); long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile1"), options); try (InodePathPair inodePathPair = mInodeTree.lockInodePathPair( new AlluxioURI("/testFolder/testFile1"), InodeTree.LockMode.WRITE_PARENT, new AlluxioURI("/testFolder/testFile2"), InodeTree.LockMode.WRITE)) { LockedInodePath srcPath = inodePathPair.getFirst(); LockedInodePath dstPath = inodePathPair.getSecond(); mFsMaster.renameInternal(srcPath, dstPath, true, TEST_CURRENT_TIME); } FileInfo folderInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile2"))); Assert.assertEquals(ttl, folderInfo.getTtl()); }
@Test public void createAlreadyExistFileTest() throws Exception { mThrown.expect(FileAlreadyExistsException.class); mFsMaster.createFile(new AlluxioURI("/testFile"), CreateFileOptions.defaults()); mFsMaster.createDirectory(new AlluxioURI("/testFile"), CreateDirectoryOptions.defaults()); }