/** Tests a file can be deleted. */
@Test
public void deleteFile() throws IOException {
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
createEmptyFile(testFile);
mUfs.delete(testFile, false);
Assert.assertFalse(mUfs.exists(testFile));
}
// Creates a block file and write an increasing byte array into it
private void createBlockFile(String filename, int len) throws IOException, InvalidPathException {
UnderFileSystem ufs = UnderFileSystem.get(filename, mMasterConfiguration);
ufs.mkdirs(PathUtils.getParent(filename), true);
OutputStream out = ufs.create(filename);
out.write(BufferUtils.getIncreasingByteArray(len), 0, len);
out.close();
}
// Tests that lock block returns the correct path
@Test
public void lockBlockTest() throws Exception {
final int blockSize = (int) WORKER_CAPACITY_BYTES / 2;
CreateFileOptions options =
CreateFileOptions.defaults()
.setBlockSizeBytes(blockSize)
.setWriteType(WriteType.MUST_CACHE);
FileOutStream out = mFileSystem.createFile(new AlluxioURI("/testFile"), options);
URIStatus file = mFileSystem.getStatus(new AlluxioURI("/testFile"));
final long blockId = BlockId.createBlockId(BlockId.getContainerId(file.getFileId()), 0);
out.write(BufferUtils.getIncreasingByteArray(blockSize));
out.close();
String localPath = mBlockWorkerServiceHandler.lockBlock(blockId, SESSION_ID).getBlockPath();
// The local path should exist
Assert.assertNotNull(localPath);
UnderFileSystem ufs = UnderFileSystem.get(localPath, mMasterConfiguration);
byte[] data = new byte[blockSize];
int bytesRead = ufs.open(localPath).read(data);
// The data in the local file should equal the data we wrote earlier
Assert.assertEquals(blockSize, bytesRead);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytesRead, data));
mBlockWorkerServiceHandler.unlockBlock(blockId, SESSION_ID);
}
/** Tests if list correctly returns file names. */
@Test
public void list() throws IOException {
String testDirNonEmpty = PathUtils.concatPath(mUnderfsAddress, "testDirNonEmpty1");
String testDirNonEmptyChildDir = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmpty2");
String testDirNonEmptyChildFile = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmptyF");
String testDirNonEmptyChildDirFile =
PathUtils.concatPath(testDirNonEmptyChildDir, "testDirNonEmptyChildDirF");
mUfs.mkdirs(testDirNonEmpty, false);
mUfs.mkdirs(testDirNonEmptyChildDir, false);
createEmptyFile(testDirNonEmptyChildFile);
createEmptyFile(testDirNonEmptyChildDirFile);
String[] expectedResTopDir = new String[] {"testDirNonEmpty2", "testDirNonEmptyF"};
// Some file systems may prefix with a slash
String[] expectedResTopDir2 = new String[] {"/testDirNonEmpty2", "/testDirNonEmptyF"};
Arrays.sort(expectedResTopDir);
Arrays.sort(expectedResTopDir2);
String[] resTopDir = mUfs.list(testDirNonEmpty);
Arrays.sort(resTopDir);
Assert.assertTrue(
Arrays.equals(expectedResTopDir, resTopDir)
|| Arrays.equals(expectedResTopDir2, resTopDir));
Assert.assertTrue(
mUfs.list(testDirNonEmptyChildDir)[0].equals("testDirNonEmptyChildDirF")
|| mUfs.list(testDirNonEmptyChildDir)[0].equals("/testDirNonEmptyChildDirF"));
}
// Prepare directory tree for pagination tests
private LargeDirectoryConfig prepareLargeDirectoryTest() throws IOException {
final String filePrefix = "a_";
final String folderPrefix = "b_";
String topLevelDirectory = PathUtils.concatPath(mUnderfsAddress, "topLevelDir");
final int numFiles = 100;
String[] children = new String[numFiles + numFiles];
// Make top level directory
mUfs.mkdirs(topLevelDirectory, false);
// Make the children files
for (int i = 0; i < numFiles; ++i) {
children[i] = PathUtils.concatPath(topLevelDirectory, filePrefix + String.format("%04d", i));
createEmptyFile(children[i]);
}
// Make the children folders
for (int i = 0; i < numFiles; ++i) {
children[numFiles + i] =
PathUtils.concatPath(topLevelDirectory, folderPrefix + String.format("%04d", i));
mUfs.mkdirs(children[numFiles + i], false);
}
return new LargeDirectoryConfig(topLevelDirectory, children);
}
/**
* Creates a new file output stream.
*
* @param path the file path
* @param options the client options
* @throws IOException if an I/O error occurs
*/
public FileOutStream(AlluxioURI path, OutStreamOptions options) throws IOException {
mUri = Preconditions.checkNotNull(path);
mNonce = IdUtils.getRandomNonNegativeLong();
mBlockSize = options.getBlockSizeBytes();
mAlluxioStorageType = options.getAlluxioStorageType();
mUnderStorageType = options.getUnderStorageType();
mContext = FileSystemContext.INSTANCE;
mPreviousBlockOutStreams = new LinkedList<BufferedBlockOutStream>();
if (mUnderStorageType.isSyncPersist()) {
updateUfsPath();
String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
// TODO(jiri): Implement collection of temporary files left behind by dead clients.
mUnderStorageOutputStream = ufs.create(tmpPath, (int) mBlockSize);
} else {
mUfsPath = null;
mUnderStorageOutputStream = null;
}
mClosed = false;
mCanceled = false;
mShouldCacheCurrentBlock = mAlluxioStorageType.isStore();
mBytesWritten = 0;
mLocationPolicy =
Preconditions.checkNotNull(
options.getLocationPolicy(),
PreconditionMessage.FILE_WRITE_LOCATION_POLICY_UNSPECIFIED);
}
/** Tests if file creation is atomic. */
@Test
public void createAtomic() throws IOException {
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
OutputStream stream = mUfs.create(testFile);
stream.write(TEST_BYTES);
Assert.assertFalse(mUfs.exists(testFile));
stream.close();
}
/** Tests {@link UnderFileSystem#rename(String, String)} works file to new location. */
@Test
public void renameFile() throws IOException {
String testFileSrc = PathUtils.concatPath(mUnderfsAddress, "testFileSrc");
String testFileDst = PathUtils.concatPath(mUnderfsAddress, "testFileDst");
createEmptyFile(testFileSrc);
mUfs.rename(testFileSrc, testFileDst);
Assert.assertFalse(mUfs.exists(testFileSrc));
Assert.assertTrue(mUfs.exists(testFileDst));
}
/** Tests {@link UnderFileSystem#getFileSize(String)} correctly returns the file size. */
@Test
public void getFileSize() throws IOException {
String testFileEmpty = PathUtils.concatPath(mUnderfsAddress, "testFileEmpty");
String testFileNonEmpty = PathUtils.concatPath(mUnderfsAddress, "testFileNonEmpty");
createEmptyFile(testFileEmpty);
createTestBytesFile(testFileNonEmpty);
Assert.assertEquals(mUfs.getFileSize(testFileEmpty), 0);
Assert.assertEquals(mUfs.getFileSize(testFileNonEmpty), TEST_BYTES.length);
}
/**
* Tests if {@link UnderFileSystem#isFile(String)} correctly returns true for files and false
* otherwise.
*/
@Test
public void isFile() throws IOException {
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
String testDir = PathUtils.concatPath(mUnderfsAddress, "testDir");
Assert.assertFalse(mUfs.isFile(testFile));
createEmptyFile(testFile);
mUfs.mkdirs(testDir, false);
Assert.assertTrue(mUfs.isFile(testFile));
Assert.assertFalse(mUfs.isFile(testDir));
}
/** Tests {@link UnderFileSystem#rename(String, String)} works file to a folder if supported. */
@Test
public void renameFileToFolder() throws IOException {
String testFileSrc = PathUtils.concatPath(mUnderfsAddress, "testFileSrc");
String testFileDst = PathUtils.concatPath(mUnderfsAddress, "testDirDst");
String testFileFinalDst = PathUtils.concatPath(testFileDst, "testFileSrc");
createEmptyFile(testFileSrc);
mUfs.mkdirs(testFileDst, false);
if (mUfs.rename(testFileSrc, testFileDst)) {
Assert.assertFalse(mUfs.exists(testFileSrc));
Assert.assertTrue(mUfs.exists(testFileFinalDst));
}
}
/** Tests if list correctly returns file or folder names for a large directory. */
@Test
public void listLargeDirectory() throws IOException {
LargeDirectoryConfig config = prepareLargeDirectoryTest();
String[] children = config.getChildren();
// Retry for some time to allow list operation eventual consistency for S3 and GCS.
// See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
// https://cloud.google.com/storage/docs/consistency for more details.
// Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
String[] results = new String[] {};
for (int i = 0; i < 20; i++) {
results = mUfs.list(config.getTopLevelDirectory());
if (children.length == results.length) {
break;
}
CommonUtils.sleepMs(500);
}
Assert.assertEquals(children.length, results.length);
Arrays.sort(results);
for (int i = 0; i < children.length; ++i) {
Assert.assertTrue(
results[i].equals(
CommonUtils.stripPrefixIfPresent(
children[i], PathUtils.normalizePath(config.getTopLevelDirectory(), "/"))));
}
}
/**
* Creates a new instance of {@link FileSystemWorker}.
*
* @param blockWorker the block worker handle
* @param workerId a reference to the id of this worker
* @throws IOException if an I/O error occurs
*/
public DefaultFileSystemWorker(BlockWorker blockWorker, AtomicReference<Long> workerId)
throws IOException {
super(
Executors.newFixedThreadPool(
3, ThreadFactoryUtils.build("file-system-worker-heartbeat-%d", true)));
mWorkerId = workerId;
mSessions = new Sessions();
UnderFileSystem ufs = UnderFileSystem.get(Configuration.get(PropertyKey.UNDERFS_ADDRESS));
mFileDataManager =
new FileDataManager(
Preconditions.checkNotNull(blockWorker),
ufs,
RateLimiter.create(Configuration.getBytes(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT)));
mUnderFileSystemManager = new UnderFileSystemManager();
// Setup AbstractMasterClient
mFileSystemMasterWorkerClient =
new FileSystemMasterClient(NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC));
// Setup session cleaner
mSessionCleaner =
new SessionCleaner(
new SessionCleanupCallback() {
/** Cleans up after sessions, to prevent zombie sessions holding ufs resources. */
@Override
public void cleanupSessions() {
for (long session : mSessions.getTimedOutSessions()) {
mSessions.removeSession(session);
mUnderFileSystemManager.cleanupSession(session);
}
}
});
mServiceHandler = new FileSystemWorkerClientServiceHandler(this);
}
/**
* Gets a UFS instance from the cache if exists. Otherwise, creates a new instance and adds that
* to the cache.
*
* @param path the ufs path
* @param ufsConf the ufs configuration
* @return the UFS instance
*/
UnderFileSystem get(String path, Object ufsConf) {
Key key = new Key(new AlluxioURI(path));
UnderFileSystem cachedFs = mUnderFileSystemMap.get(key);
if (cachedFs != null) {
return cachedFs;
}
UnderFileSystem fs = UnderFileSystemRegistry.create(path, ufsConf);
cachedFs = mUnderFileSystemMap.putIfAbsent(key, fs);
if (cachedFs == null) {
return fs;
}
try {
fs.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
return cachedFs;
}
/** Tests that a file can be created and validates the data written to it. */
@Test
public void createOpen() throws IOException {
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
createTestBytesFile(testFile);
byte[] buf = new byte[TEST_BYTES.length];
int bytesRead = mUfs.open(testFile).read(buf);
Assert.assertTrue(bytesRead == TEST_BYTES.length);
Assert.assertTrue(Arrays.equals(buf, TEST_BYTES));
}
/**
* Tests {@link UnderFileSystem#getModificationTimeMs(String)} returns a reasonably accurate time.
*/
@Test
public void getModTime() throws IOException {
long slack = 1000; // Some file systems may report nearest second.
long start = System.currentTimeMillis();
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
createTestBytesFile(testFile);
long end = System.currentTimeMillis();
long modTime = mUfs.getModificationTimeMs(testFile);
Assert.assertTrue(modTime >= start - slack);
Assert.assertTrue(modTime <= end + slack);
}
/** Tests if delete deletes all files or folders for a large directory. */
@Test
public void deleteLargeDirectory() throws IOException {
LargeDirectoryConfig config = prepareLargeDirectoryTest();
mUfs.delete(config.getTopLevelDirectory(), true);
String[] children = config.getChildren();
for (String child : children) {
// Retry for some time to allow list operation eventual consistency for S3 and GCS.
// See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
// https://cloud.google.com/storage/docs/consistency for more details.
// Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
boolean childDeleted = false;
for (int i = 0; i < 20; i++) {
childDeleted = !mUfs.exists(child);
if (childDeleted) {
break;
}
CommonUtils.sleepMs(500);
}
Assert.assertTrue(childDeleted);
}
}
/** Tests if list recursive correctly returns all file names in all subdirectories. */
@Test
public void listRecursive() throws IOException {
String root = mUnderfsAddress;
// TODO(andrew): Should this directory be created in LocalAlluxioCluster creation code?
mUfs.mkdirs(root, true);
// Empty lsr should be empty
Assert.assertEquals(0, mUfs.listRecursive(root).length);
// Create a tree of subdirectories and files
String sub1 = PathUtils.concatPath(root, "sub1");
String sub2 = PathUtils.concatPath(root, "sub2");
String sub11 = PathUtils.concatPath(sub1, "sub11");
String file11 = PathUtils.concatPath(sub11, "file11");
String file2 = PathUtils.concatPath(sub2, "file2");
String file = PathUtils.concatPath(root, "file");
// lsr of nonexistent path should be null
Assert.assertNull(mUfs.listRecursive(sub1));
mUfs.mkdirs(sub1, false);
mUfs.mkdirs(sub2, false);
mUfs.mkdirs(sub11, false);
createEmptyFile(file11);
createEmptyFile(file2);
createEmptyFile(file);
// lsr from root should return paths relative to the root
String[] expectedResRoot = {
"sub1", "sub2", "sub1/sub11", "sub1/sub11/file11", "sub2/file2", "file"
};
String[] actualResRoot = mUfs.listRecursive(root);
Arrays.sort(expectedResRoot);
Arrays.sort(actualResRoot);
Assert.assertArrayEquals(expectedResRoot, actualResRoot);
// lsr from sub1 should return paths relative to sub1
String[] expectedResSub1 = {"sub11", "sub11/file11"};
String[] actualResSub1 = mUfs.listRecursive(sub1);
Arrays.sort(expectedResSub1);
Arrays.sort(actualResSub1);
Assert.assertArrayEquals(expectedResSub1, actualResSub1);
// lsr of file should be null
Assert.assertNull(mUfs.listRecursive(file));
}
/**
* Tests {@link UnderFileSystem#mkdirs(String, boolean)} correctly creates a directory. Tests
* {@link UnderFileSystem#mkdirs(String, boolean)} correctly makes parent directories if
* createParent is specified.
*/
@Test
public void mkdirs() throws IOException {
// make sure the underfs address dir exists already
mUfs.mkdirs(mUnderfsAddress, true);
// empty lsr should be empty
Assert.assertEquals(0, mUfs.listRecursive(mUnderfsAddress).length);
String testDirTop = PathUtils.concatPath(mUnderfsAddress, "testDirTop");
String testDir1 = PathUtils.concatPath(mUnderfsAddress, "1");
String testDir2 = PathUtils.concatPath(testDir1, "2");
String testDir3 = PathUtils.concatPath(testDir2, "3");
String testDirDeep = PathUtils.concatPath(testDir3, "testDirDeep");
mUfs.mkdirs(testDirTop, false);
Assert.assertTrue(mUfs.exists(testDirTop));
mUfs.mkdirs(testDirDeep, true);
Assert.assertTrue(mUfs.exists(testDir1));
Assert.assertTrue(mUfs.exists(testDir2));
Assert.assertTrue(mUfs.exists(testDir3));
Assert.assertTrue(mUfs.exists(testDirDeep));
}
/** Tests load metadata on list. */
@Test
public void loadMetadata() throws Exception {
String dirName = "loadMetaDataRoot";
String rootDir = PathUtils.concatPath(mUnderfsAddress, dirName);
mUfs.mkdirs(rootDir, true);
String rootFile1 = PathUtils.concatPath(rootDir, "file1");
createEmptyFile(rootFile1);
String rootFile2 = PathUtils.concatPath(rootDir, "file2");
createEmptyFile(rootFile2);
AlluxioURI rootAlluxioURI = new AlluxioURI("/" + dirName);
FileSystem client = mLocalAlluxioClusterResource.get().getClient();
client.listStatus(
rootAlluxioURI, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));
try {
client.createDirectory(rootAlluxioURI, CreateDirectoryOptions.defaults());
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(rootAlluxioURI), e.getMessage());
}
AlluxioURI file1URI = rootAlluxioURI.join("file1");
try {
client.createFile(file1URI, CreateFileOptions.defaults()).close();
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file1URI), e.getMessage());
}
AlluxioURI file2URI = rootAlluxioURI.join("file2");
try {
client.createFile(file2URI, CreateFileOptions.defaults()).close();
Assert.fail("create is expected to fail with FileAlreadyExistsException");
} catch (FileAlreadyExistsException e) {
Assert.assertEquals(
ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file2URI), e.getMessage());
}
}
private void createEmptyFile(String path) throws IOException {
OutputStream o = mUfs.create(path);
o.close();
}
@Override
public void close() throws IOException {
if (mClosed) {
return;
}
if (mCurrentBlockOutStream != null) {
mPreviousBlockOutStreams.add(mCurrentBlockOutStream);
}
CompleteFileOptions options = CompleteFileOptions.defaults();
if (mUnderStorageType.isSyncPersist()) {
String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
if (mCanceled) {
// TODO(yupeng): Handle this special case in under storage integrations.
mUnderStorageOutputStream.close();
if (!ufs.exists(tmpPath)) {
// Location of the temporary file has changed, recompute it.
updateUfsPath();
tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
}
ufs.delete(tmpPath, false);
} else {
mUnderStorageOutputStream.flush();
mUnderStorageOutputStream.close();
if (!ufs.exists(tmpPath)) {
// Location of the temporary file has changed, recompute it.
updateUfsPath();
tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
}
if (!ufs.rename(tmpPath, mUfsPath)) {
throw new IOException("Failed to rename " + tmpPath + " to " + mUfsPath);
}
options.setUfsLength(ufs.getFileSize(mUfsPath));
}
}
if (mAlluxioStorageType.isStore()) {
try {
if (mCanceled) {
for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
bos.cancel();
}
} else {
for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
bos.close();
}
}
} catch (IOException e) {
handleCacheWriteException(e);
}
}
// Complete the file if it's ready to be completed.
if (!mCanceled && (mUnderStorageType.isSyncPersist() || mAlluxioStorageType.isStore())) {
FileSystemMasterClient masterClient = mContext.acquireMasterClient();
try {
masterClient.completeFile(mUri, options);
} catch (AlluxioException e) {
throw new IOException(e);
} finally {
mContext.releaseMasterClient(masterClient);
}
}
if (mUnderStorageType.isAsyncPersist()) {
scheduleAsyncPersist();
}
mClosed = true;
}
/** Tests that an empty file can be created. */
@Test
public void createEmpty() throws IOException {
String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
createEmptyFile(testFile);
Assert.assertTrue(mUfs.exists(testFile));
}
@Before
public final void before() throws Exception {
Configuration.set(PropertyKey.UNDERFS_LISTING_LENGTH, 50);
mUnderfsAddress = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
mUfs = UnderFileSystem.get(mUnderfsAddress + AlluxioURI.SEPARATOR);
}
private void createTestBytesFile(String path) throws IOException {
OutputStream o = mUfs.create(path);
o.write(TEST_BYTES);
o.close();
}
/**
* Tests an empty directory can be deleted. Tests a non empty directory will not be deleted if
* recursive is not specified. Tests a non empty directory will be deleted if recursive is
* specified.
*/
@Test
public void deleteDir() throws IOException {
String testDirEmpty = PathUtils.concatPath(mUnderfsAddress, "testDirEmpty");
String testDirNonEmpty = PathUtils.concatPath(mUnderfsAddress, "testDirNonEmpty1");
String testDirNonEmptyChildDir = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmpty2");
String testDirNonEmptyChildFile = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmptyF");
String testDirNonEmptyChildDirFile =
PathUtils.concatPath(testDirNonEmptyChildDir, "testDirNonEmptyChildDirF");
mUfs.mkdirs(testDirEmpty, false);
mUfs.mkdirs(testDirNonEmpty, false);
mUfs.mkdirs(testDirNonEmptyChildDir, false);
createEmptyFile(testDirNonEmptyChildFile);
createEmptyFile(testDirNonEmptyChildDirFile);
mUfs.delete(testDirEmpty, false);
Assert.assertFalse(mUfs.exists(testDirEmpty));
try {
mUfs.delete(testDirNonEmpty, false);
} catch (IOException e) {
// Some File systems may throw IOException
}
Assert.assertTrue(mUfs.exists(testDirNonEmpty));
mUfs.delete(testDirNonEmpty, true);
Assert.assertFalse(mUfs.exists(testDirNonEmpty));
Assert.assertFalse(mUfs.exists(testDirNonEmptyChildDir));
Assert.assertFalse(mUfs.exists(testDirNonEmptyChildFile));
Assert.assertFalse(mUfs.exists(testDirNonEmptyChildDirFile));
}