@Test
public void listFilesTest() throws Exception {
CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(64);
HashSet<Long> ids = new HashSet<Long>();
HashSet<Long> dirIds = new HashSet<Long>();
for (int i = 0; i < 10; i++) {
AlluxioURI dir = new AlluxioURI("/i" + i);
mFsMaster.createDirectory(dir, CreateDirectoryOptions.defaults());
dirIds.add(mFsMaster.getFileId(dir));
for (int j = 0; j < 10; j++) {
ids.add(mFsMaster.createFile(dir.join("j" + j), options));
}
}
HashSet<Long> listedIds = new HashSet<>();
HashSet<Long> listedDirIds = new HashSet<>();
List<FileInfo> infoList = mFsMaster.getFileInfoList(new AlluxioURI("/"), true);
for (FileInfo info : infoList) {
long id = info.getFileId();
listedDirIds.add(id);
for (FileInfo fileInfo : mFsMaster.getFileInfoList(new AlluxioURI(info.getPath()), true)) {
listedIds.add(fileInfo.getFileId());
}
}
Assert.assertEquals(ids, listedIds);
Assert.assertEquals(dirIds, listedDirIds);
}
@Test
public void deleteFileTest() throws Exception {
long fileId = mFsMaster.createFile(new AlluxioURI("/testFile"), CreateFileOptions.defaults());
Assert.assertEquals(fileId, mFsMaster.getFileId(new AlluxioURI("/testFile")));
mFsMaster.delete(new AlluxioURI("/testFile"), true);
Assert.assertEquals(IdUtils.INVALID_FILE_ID, mFsMaster.getFileId(new AlluxioURI("/testFile")));
}
@Test
public void notFileCompletionTest() throws Exception {
mThrown.expect(FileDoesNotExistException.class);
mFsMaster.createDirectory(new AlluxioURI("/testFile"), CreateDirectoryOptions.defaults());
CompleteFileOptions options = CompleteFileOptions.defaults();
mFsMaster.completeFile(new AlluxioURI("/testFile"), options);
}
@Test
public void deleteEmptyDirectoryTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
Assert.assertEquals(1, mFsMaster.getFileId(new AlluxioURI("/testFolder")));
mFsMaster.delete(new AlluxioURI("/testFolder"), true);
Assert.assertEquals(
IdUtils.INVALID_FILE_ID, mFsMaster.getFileId(new AlluxioURI("/testFolder")));
}
@Test
public void createFileTest() throws Exception {
mFsMaster.createFile(new AlluxioURI("/testFile"), CreateFileOptions.defaults());
FileInfo fileInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFile")));
Assert.assertFalse(fileInfo.isFolder());
Assert.assertEquals("", fileInfo.getUserName());
Assert.assertEquals(0644, (short) fileInfo.getPermission());
}
@Test
public void createDirectoryTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
FileInfo fileInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder")));
Assert.assertTrue(fileInfo.isFolder());
Assert.assertEquals("", fileInfo.getUserName());
Assert.assertEquals(0755, (short) fileInfo.getPermission());
}
@Test
public void renameExistingDstTest() throws Exception {
mFsMaster.createFile(new AlluxioURI("/testFile1"), CreateFileOptions.defaults());
mFsMaster.createFile(new AlluxioURI("/testFile2"), CreateFileOptions.defaults());
try {
mFsMaster.rename(new AlluxioURI("/testFile1"), new AlluxioURI("/testFile2"));
Assert.fail("Should not be able to rename to an existing file");
} catch (Exception e) {
// expected
}
}
@Test
public void lastModificationTimeCompleteFileTest() throws Exception {
long fileId = mFsMaster.createFile(new AlluxioURI("/testFile"), CreateFileOptions.defaults());
long opTimeMs = TEST_CURRENT_TIME;
try (LockedInodePath inodePath =
mInodeTree.lockFullInodePath(new AlluxioURI("/testFile"), InodeTree.LockMode.WRITE)) {
mFsMaster.completeFileInternal(new ArrayList<Long>(), inodePath, 0, opTimeMs);
}
FileInfo fileInfo = mFsMaster.getFileInfo(fileId);
Assert.assertEquals(opTimeMs, fileInfo.getLastModificationTimeMs());
}
@Test
public void lastModificationTimeCreateFileTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
long opTimeMs = TEST_CURRENT_TIME;
CreateFileOptions options = CreateFileOptions.defaults().setOperationTimeMs(opTimeMs);
try (LockedInodePath inodePath =
mInodeTree.lockInodePath(
new AlluxioURI("/testFolder/testFile"), InodeTree.LockMode.WRITE)) {
mFsMaster.createFileInternal(inodePath, options);
}
FileInfo folderInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder")));
Assert.assertEquals(opTimeMs, folderInfo.getLastModificationTimeMs());
}
@Test
public void renameToDeeper() throws Exception {
CreateFileOptions createFileOptions = CreateFileOptions.defaults().setRecursive(true);
CreateDirectoryOptions createDirectoryOptions =
CreateDirectoryOptions.defaults().setRecursive(true);
mThrown.expect(InvalidPathException.class);
mFsMaster.createDirectory(new AlluxioURI("/testDir1/testDir2"), createDirectoryOptions);
mFsMaster.createFile(
new AlluxioURI("/testDir1/testDir2/testDir3/testFile3"), createFileOptions);
mFsMaster.rename(
new AlluxioURI("/testDir1/testDir2"),
new AlluxioURI("/testDir1/testDir2/testDir3/testDir4"));
}
@Test
public void createFilePerfTest() throws Exception {
for (int k = 0; k < 200; k++) {
CreateDirectoryOptions options = CreateDirectoryOptions.defaults().setRecursive(true);
mFsMaster.createDirectory(
new AlluxioURI("/testFile").join(Constants.MASTER_COLUMN_FILE_PREFIX + k).join("0"),
options);
}
for (int k = 0; k < 200; k++) {
mFsMaster.getFileInfo(
mFsMaster.getFileId(
new AlluxioURI("/testFile").join(Constants.MASTER_COLUMN_FILE_PREFIX + k).join("0")));
}
}
@Test
public void ttlCreateFileTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
long ttl = 100;
CreateFileOptions options = CreateFileOptions.defaults().setTtl(ttl);
try (LockedInodePath inodePath =
mInodeTree.lockInodePath(
new AlluxioURI("/testFolder/testFile"), InodeTree.LockMode.WRITE)) {
mFsMaster.createFileInternal(inodePath, options);
}
FileInfo folderInfo =
mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile")));
Assert.assertEquals(ttl, folderInfo.getTtl());
}
@Test
public void concurrentRenameTest() throws Exception {
ConcurrentCreator concurrentCreator =
new ConcurrentCreator(DEPTH, CONCURRENCY_DEPTH, ROOT_PATH);
concurrentCreator.call();
int numFiles = mFsMaster.getFileInfoList(ROOT_PATH, true).size();
ConcurrentRenamer concurrentRenamer =
new ConcurrentRenamer(
DEPTH, CONCURRENCY_DEPTH, ROOT_PATH, ROOT_PATH2, AlluxioURI.EMPTY_URI);
concurrentRenamer.call();
Assert.assertEquals(numFiles, mFsMaster.getFileInfoList(ROOT_PATH2, true).size());
}
@Test
public void clientFileInfoEmptyFileTest() throws Exception {
long fileId = mFsMaster.createFile(new AlluxioURI("/testFile"), CreateFileOptions.defaults());
FileInfo fileInfo = mFsMaster.getFileInfo(fileId);
Assert.assertEquals("testFile", fileInfo.getName());
Assert.assertEquals(fileId, fileInfo.getFileId());
Assert.assertEquals(0, fileInfo.getLength());
Assert.assertTrue(fileInfo.isCacheable());
Assert.assertFalse(fileInfo.isCompleted());
Assert.assertFalse(fileInfo.isFolder());
Assert.assertFalse(fileInfo.isPersisted());
Assert.assertFalse(fileInfo.isPinned());
Assert.assertEquals(Constants.NO_TTL, fileInfo.getTtl());
Assert.assertEquals("", fileInfo.getUserName());
Assert.assertEquals(0644, (short) fileInfo.getPermission());
}
// TODO(calvin): This test currently relies on the fact the HDFS client is a cached instance to
// avoid invalid lease exception. This should be fixed.
@Ignore
@Test
public void concurrentCreateJournalTest() throws Exception {
// Makes sure the file id's are the same between a master info and the journal it creates
for (int i = 0; i < 5; i++) {
ConcurrentCreator concurrentCreator =
new ConcurrentCreator(DEPTH, CONCURRENCY_DEPTH, ROOT_PATH);
concurrentCreator.call();
FileSystemMaster fsMaster = createFileSystemMasterFromJournal();
for (FileInfo info : mFsMaster.getFileInfoList(new AlluxioURI("/"), true)) {
AlluxioURI path = new AlluxioURI(info.getPath());
Assert.assertEquals(mFsMaster.getFileId(path), fsMaster.getFileId(path));
}
before();
}
}
@Test
public void clientFileInfoDirectoryTest() throws Exception {
AlluxioURI path = new AlluxioURI("/testFolder");
mFsMaster.createDirectory(path, CreateDirectoryOptions.defaults());
long fileId = mFsMaster.getFileId(path);
FileInfo fileInfo = mFsMaster.getFileInfo(fileId);
Assert.assertEquals("testFolder", fileInfo.getName());
Assert.assertEquals(1, fileInfo.getFileId());
Assert.assertEquals(0, fileInfo.getLength());
Assert.assertFalse(fileInfo.isCacheable());
Assert.assertTrue(fileInfo.isCompleted());
Assert.assertTrue(fileInfo.isFolder());
Assert.assertFalse(fileInfo.isPersisted());
Assert.assertFalse(fileInfo.isPinned());
Assert.assertEquals("", fileInfo.getUserName());
Assert.assertEquals(0755, (short) fileInfo.getPermission());
}
@Test
public void lsTest() throws Exception {
CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(64);
for (int i = 0; i < 10; i++) {
mFsMaster.createDirectory(new AlluxioURI("/i" + i), CreateDirectoryOptions.defaults());
for (int j = 0; j < 10; j++) {
mFsMaster.createFile(new AlluxioURI("/i" + i + "/j" + j), options);
}
}
Assert.assertEquals(1, mFsMaster.getFileInfoList(new AlluxioURI("/i0/j0"), true).size());
for (int i = 0; i < 10; i++) {
Assert.assertEquals(10, mFsMaster.getFileInfoList(new AlluxioURI("/i" + i), true).size());
}
Assert.assertEquals(10, mFsMaster.getFileInfoList(new AlluxioURI("/"), true).size());
}
@Test
public void ttlExpiredCreateFileTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
long ttl = 1;
CreateFileOptions options = CreateFileOptions.defaults().setTtl(ttl);
long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile1"), options);
FileInfo folderInfo =
mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile1")));
Assert.assertEquals(fileId, folderInfo.getFileId());
Assert.assertEquals(ttl, folderInfo.getTtl());
// Sleep for the ttl expiration.
CommonUtils.sleepMs(2 * TTL_CHECKER_INTERVAL_MS);
Assert.assertTrue(
HeartbeatScheduler.await(HeartbeatContext.MASTER_TTL_CHECK, 10, TimeUnit.SECONDS));
HeartbeatScheduler.schedule(HeartbeatContext.MASTER_TTL_CHECK);
Assert.assertTrue(
HeartbeatScheduler.await(HeartbeatContext.MASTER_TTL_CHECK, 10, TimeUnit.SECONDS));
mThrown.expect(FileDoesNotExistException.class);
mFsMaster.getFileInfo(fileId);
}
@Test
public void deleteDirectoryWithDirectoriesTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
mFsMaster.createDirectory(
new AlluxioURI("/testFolder/testFolder2"), CreateDirectoryOptions.defaults());
long fileId =
mFsMaster.createFile(new AlluxioURI("/testFolder/testFile"), CreateFileOptions.defaults());
long fileId2 =
mFsMaster.createFile(
new AlluxioURI("/testFolder/testFolder2/testFile2"), CreateFileOptions.defaults());
Assert.assertEquals(1, mFsMaster.getFileId(new AlluxioURI("/testFolder")));
Assert.assertEquals(2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2")));
Assert.assertEquals(fileId, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile")));
Assert.assertEquals(
fileId2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2/testFile2")));
mFsMaster.delete(new AlluxioURI("/testFolder"), true);
Assert.assertEquals(
IdUtils.INVALID_FILE_ID,
mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2/testFile2")));
}
@Test
public void lastModificationTimeRenameTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
long fileId =
mFsMaster.createFile(new AlluxioURI("/testFolder/testFile1"), CreateFileOptions.defaults());
long opTimeMs = TEST_CURRENT_TIME;
try (InodePathPair inodePathPair =
mInodeTree.lockInodePathPair(
new AlluxioURI("/testFolder/testFile1"),
InodeTree.LockMode.WRITE_PARENT,
new AlluxioURI("/testFolder/testFile2"),
InodeTree.LockMode.WRITE)) {
LockedInodePath srcPath = inodePathPair.getFirst();
LockedInodePath dstPath = inodePathPair.getSecond();
mFsMaster.renameInternal(srcPath, dstPath, true, opTimeMs);
}
FileInfo folderInfo = mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder")));
Assert.assertEquals(opTimeMs, folderInfo.getLastModificationTimeMs());
}
@Test
public void ttlRenameTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
long ttl = 1;
CreateFileOptions options = CreateFileOptions.defaults().setTtl(ttl);
long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile1"), options);
try (InodePathPair inodePathPair =
mInodeTree.lockInodePathPair(
new AlluxioURI("/testFolder/testFile1"),
InodeTree.LockMode.WRITE_PARENT,
new AlluxioURI("/testFolder/testFile2"),
InodeTree.LockMode.WRITE)) {
LockedInodePath srcPath = inodePathPair.getFirst();
LockedInodePath dstPath = inodePathPair.getSecond();
mFsMaster.renameInternal(srcPath, dstPath, true, TEST_CURRENT_TIME);
}
FileInfo folderInfo =
mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile2")));
Assert.assertEquals(ttl, folderInfo.getTtl());
}
@Test
public void concurrentDeleteTest() throws Exception {
ConcurrentCreator concurrentCreator =
new ConcurrentCreator(DEPTH, CONCURRENCY_DEPTH, ROOT_PATH);
concurrentCreator.call();
ConcurrentDeleter concurrentDeleter =
new ConcurrentDeleter(DEPTH, CONCURRENCY_DEPTH, ROOT_PATH);
concurrentDeleter.call();
Assert.assertEquals(0, mFsMaster.getFileInfoList(new AlluxioURI("/"), true).size());
}
/** Tests that deleting a file from a folder updates the folder's last modification time. */
@Test
public void lastModificationTimeDeleteTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
mFsMaster.createFile(new AlluxioURI("/testFolder/testFile"), CreateFileOptions.defaults());
long folderId = mFsMaster.getFileId(new AlluxioURI("/testFolder"));
long modificationTimeBeforeDelete = mFsMaster.getFileInfo(folderId).getLastModificationTimeMs();
CommonUtils.sleepMs(2);
mFsMaster.delete(new AlluxioURI("/testFolder/testFile"), true);
long modificationTimeAfterDelete = mFsMaster.getFileInfo(folderId).getLastModificationTimeMs();
Assert.assertTrue(modificationTimeBeforeDelete < modificationTimeAfterDelete);
}
@Test
public void deleteDirectoryWithFilesTest2() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
long fileId =
mFsMaster.createFile(new AlluxioURI("/testFolder/testFile"), CreateFileOptions.defaults());
Assert.assertEquals(1, mFsMaster.getFileId(new AlluxioURI("/testFolder")));
Assert.assertEquals(fileId, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile")));
try {
mFsMaster.delete(new AlluxioURI("/testFolder"), false);
Assert.fail("Deleting a nonempty directory nonrecursively should fail");
} catch (DirectoryNotEmptyException e) {
Assert.assertEquals(
ExceptionMessage.DELETE_NONEMPTY_DIRECTORY_NONRECURSIVE.getMessage("testFolder"),
e.getMessage());
}
Assert.assertEquals(1, mFsMaster.getFileId(new AlluxioURI("/testFolder")));
Assert.assertEquals(fileId, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile")));
}
@Test
public void createAlreadyExistFileTest() throws Exception {
mThrown.expect(FileAlreadyExistsException.class);
mFsMaster.createFile(new AlluxioURI("/testFile"), CreateFileOptions.defaults());
mFsMaster.createDirectory(new AlluxioURI("/testFile"), CreateDirectoryOptions.defaults());
}
@Test
public void renameNonexistentTest() throws Exception {
mFsMaster.createFile(new AlluxioURI("/testFile1"), CreateFileOptions.defaults());
Assert.assertEquals(IdUtils.INVALID_FILE_ID, mFsMaster.getFileId(new AlluxioURI("/testFile2")));
}
@Test
public void createFileInvalidPathTest2() throws Exception {
mThrown.expect(FileAlreadyExistsException.class);
mFsMaster.createFile(new AlluxioURI("/"), CreateFileOptions.defaults());
}
@Test
public void createFileInvalidPathTest3() throws Exception {
mThrown.expect(InvalidPathException.class);
mFsMaster.createFile(new AlluxioURI("/testFile1"), CreateFileOptions.defaults());
mFsMaster.createFile(new AlluxioURI("/testFile1/testFile2"), CreateFileOptions.defaults());
}
@Test
public void deleteRootTest() throws Exception {
mThrown.expect(InvalidPathException.class);
mThrown.expectMessage(ExceptionMessage.DELETE_ROOT_DIRECTORY.getMessage());
mFsMaster.delete(new AlluxioURI("/"), true);
}