// Tests that lock block returns the correct path
@Test
public void lockBlockTest() throws Exception {
final int blockSize = (int) WORKER_CAPACITY_BYTES / 2;
CreateFileOptions options =
CreateFileOptions.defaults()
.setBlockSizeBytes(blockSize)
.setWriteType(WriteType.MUST_CACHE);
FileOutStream out = mFileSystem.createFile(new AlluxioURI("/testFile"), options);
URIStatus file = mFileSystem.getStatus(new AlluxioURI("/testFile"));
final long blockId = BlockId.createBlockId(BlockId.getContainerId(file.getFileId()), 0);
out.write(BufferUtils.getIncreasingByteArray(blockSize));
out.close();
String localPath = mBlockWorkerServiceHandler.lockBlock(blockId, SESSION_ID).getBlockPath();
// The local path should exist
Assert.assertNotNull(localPath);
UnderFileSystem ufs = UnderFileSystem.get(localPath, mMasterConfiguration);
byte[] data = new byte[blockSize];
int bytesRead = ufs.open(localPath).read(data);
// The data in the local file should equal the data we wrote earlier
Assert.assertEquals(blockSize, bytesRead);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytesRead, data));
mBlockWorkerServiceHandler.unlockBlock(blockId, SESSION_ID);
}
private void writeTest3Util(AlluxioURI filePath, int len, CreateFileOptions op) throws Exception {
FileOutStream os = mFileSystem.createFile(filePath, op);
os.write(BufferUtils.getIncreasingByteArray(0, len / 2), 0, len / 2);
os.write(BufferUtils.getIncreasingByteArray(len / 2, len / 2), 0, len / 2);
os.close();
checkWrite(filePath, op.getUnderStorageType(), len, len / 2 * 2);
}
private void writeTest1Util(AlluxioURI filePath, int len, CreateFileOptions op) throws Exception {
FileOutStream os = mFileSystem.createFile(filePath, op);
for (int k = 0; k < len; k++) {
os.write((byte) k);
}
os.close();
checkWrite(filePath, op.getUnderStorageType(), len, len);
}
/**
* Tests writing to a file for longer than HEARTBEAT_INTERVAL_MS to make sure the sessionId
* doesn't change. Tracks [ALLUXIO-171].
*/
@Test
public void longWrite() throws Exception {
AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
final int length = 2;
FileOutStream os =
mFileSystem.createFile(
filePath, CreateFileOptions.defaults().setWriteType(WriteType.THROUGH));
os.write((byte) 0);
Thread.sleep(Configuration.getInt(PropertyKey.USER_HEARTBEAT_INTERVAL_MS) * 2);
os.write((byte) 1);
os.close();
checkWrite(filePath, UnderStorageType.SYNC_PERSIST, length, length);
}
/** Tests writing to a file and specify the location to be localhost. */
@Test
public void writeSpecifyLocal() throws Exception {
AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
final int length = 2;
FileOutStream os =
mFileSystem.createFile(
filePath,
CreateFileOptions.defaults()
.setWriteType(WriteType.CACHE_THROUGH)
.setLocationPolicy(new LocalFirstPolicy()));
os.write((byte) 0);
os.write((byte) 1);
os.close();
checkWrite(filePath, UnderStorageType.SYNC_PERSIST, length, length);
}
/**
* Tests if out-of-order writes are possible. Writes could be out-of-order when the following are
* both true: - a "large" write (over half the internal buffer size) follows a smaller write. -
* the "large" write does not cause the internal buffer to overflow.
*/
@Test
public void outOfOrderWrite() throws Exception {
AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
FileOutStream os =
mFileSystem.createFile(
filePath, CreateFileOptions.defaults().setWriteType(WriteType.MUST_CACHE));
// Write something small, so it is written into the buffer, and not directly to the file.
os.write((byte) 0);
// A length greater than 0.5 * BUFFER_BYTES and less than BUFFER_BYTES.
int length = (BUFFER_BYTES * 3) / 4;
// Write a large amount of data (larger than BUFFER_BYTES/2, but will not overflow the buffer.
os.write(BufferUtils.getIncreasingByteArray(1, length));
os.close();
checkWrite(filePath, UnderStorageType.NO_PERSIST, length + 1, length + 1);
}
@Override
public void cancel() throws IOException {
mCanceled = true;
close();
}
/**
* Entry point for the {@link MultiMount} program.
*
* @param args command-line arguments
*/
public static void main(String[] args) {
if (args.length != 1) {
System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
System.exit(-1);
}
AlluxioURI mntPath = new AlluxioURI("/mnt");
AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
AlluxioURI s3Path = new AlluxioURI("s3n://alluxio-demo/");
AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
AlluxioURI hdfsPath = new AlluxioURI(args[0]);
FileSystem fileSystem = FileSystem.Factory.get();
try {
// Make sure mount directory exists.
if (!fileSystem.exists(mntPath)) {
System.out.print("creating " + mntPath + " ... ");
fileSystem.createDirectory(mntPath);
System.out.println("done");
}
// Make sure the S3 mount point does not exist.
if (fileSystem.exists(s3Mount)) {
System.out.print("unmounting " + s3Mount + " ... ");
fileSystem.unmount(s3Mount);
System.out.println("done");
}
// Make sure the HDFS mount point does not exist.
if (fileSystem.exists(hdfsMount)) {
System.out.print("unmounting " + hdfsMount + " ... ");
fileSystem.unmount(hdfsMount);
System.out.println("done");
}
// Mount S3.
System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
fileSystem.mount(s3Mount, s3Path);
System.out.println("done");
// Mount HDFS.
System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
fileSystem.mount(hdfsMount, hdfsPath);
System.out.println("done");
// Make sure output file does not exist.
if (fileSystem.exists(outputPath)) {
System.out.print("deleting " + outputPath + " ... ");
fileSystem.delete(outputPath);
System.out.println("done");
}
// Open the input stream.
System.out.print("opening " + inputPath + " ... ");
FileInStream is = fileSystem.openFile(inputPath);
System.out.println("done");
// Open the output stream, setting the write type to make sure result is persisted.
System.out.print("opening " + outputPath + " ... ");
CreateFileOptions options =
CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH);
FileOutStream os = fileSystem.createFile(outputPath, options);
System.out.println("done");
// Copy the data
System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
IOUtils.copy(is, os);
System.out.println("done");
// Close the input stream.
System.out.print("closing " + inputPath + " ... ");
is.close();
System.out.println("done");
// Close the output stream.
System.out.print("closing " + outputPath + " ... ");
os.close();
System.out.println("done");
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
} finally {
// Make sure the S3 mount point is removed.
try {
if (fileSystem.exists(s3Mount)) {
System.out.print("unmounting " + s3Mount + " ... ");
fileSystem.unmount(s3Mount);
System.out.println("done");
}
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
}
// Make sure the HDFS mount point is removed.
try {
if (fileSystem.exists(hdfsMount)) {
System.out.print("unmounting " + hdfsMount + " ... ");
fileSystem.unmount(hdfsMount);
System.out.println("done");
}
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
}
}
}