@Test
public void copyFromLocalLargeTest() throws IOException, TException {
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
testFile.createNewFile();
FileOutputStream fos = new FileOutputStream(testFile);
byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
fos.write(toWrite);
fos.close();
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
Assert.assertEquals(
getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
mOutput.toString());
TachyonFile tFile = mTfs.open(new TachyonURI("/testFile"));
FileInfo fileInfo = mTfs.getInfo(tFile);
Assert.assertNotNull(fileInfo);
Assert.assertEquals(SIZE_BYTES, fileInfo.length);
ClientOptions options =
new ClientOptions.Builder(new TachyonConf())
.setTachyonStoreType(TachyonStorageType.NO_STORE)
.build();
FileInStream tfis = mTfs.getInStream(tFile, options);
byte[] read = new byte[SIZE_BYTES];
tfis.read(read);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
}
/**
* Copies a file specified by argv from the filesystem to the local filesystem. This is the
* utility function.
*
* @param srcPath The source TachyonURI (has to be a file)
* @param dstFile The destination file in the local filesystem
* @return 0 if command is successful, -1 if an error occurred.
* @throws IOException
*/
public int copyFileToLocal(TachyonURI srcPath, File dstFile) throws IOException {
TachyonFile srcFd;
try {
srcFd = mTfs.open(srcPath);
if (srcFd == null) {
System.out.println(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(srcPath));
return -1;
}
} catch (TachyonException e) {
throw new IOException(e);
}
Closer closer = Closer.create();
try {
InStreamOptions op =
new InStreamOptions.Builder(mTachyonConf)
.setTachyonStorageType(TachyonStorageType.NO_STORE)
.build();
FileInStream is = closer.register(mTfs.getInStream(srcFd, op));
FileOutputStream out = closer.register(new FileOutputStream(dstFile));
byte[] buf = new byte[64 * Constants.MB];
int t = is.read(buf);
while (t != -1) {
out.write(buf, 0, t);
t = is.read(buf);
}
System.out.println("Copied " + srcPath + " to " + dstFile.getPath());
return 0;
} catch (TachyonException e) {
throw new IOException(e.getMessage());
} finally {
closer.close();
}
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read()}. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
for (int i = 0; i < 2; i++) {
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
}
/** Test <code>void read()</code>. Read from underfs. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
if (k == 0) {
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
} else {
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
private byte[] readContent(TachyonFile tFile, int length) throws IOException, TException {
ClientOptions options =
new ClientOptions.Builder(new TachyonConf())
.setTachyonStoreType(TachyonStorageType.NO_STORE)
.build();
FileInStream tfis = mTfs.getInStream(tFile, options);
byte[] read = new byte[length];
tfis.read(read);
return read;
}
/** Test <code>long skip(long len)</code>. */
@Test
public void skipTest() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadCache);
Assert.assertEquals(k / 2, is.skip(k / 2));
Assert.assertEquals(k / 2, is.read());
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
if (k >= 3) {
is = mTfs.getInStream(f, mReadCache);
int t = k / 3;
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(t, is.read());
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(2 * t + 1, is.read());
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
}
/**
* Tests that not reading a file the whole way through then closing it will cause it to not
* recache
*/
@Test
public void incompleteFileReadCancelsRecache() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 2, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
Assert.assertEquals(0, is.read());
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadNoCache);
is.close();
}
/** Tests that reading a file the whole way through with the STORE ReadType will recache it */
@Test
public void completeFileReadTriggersRecache() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
int len = 2;
TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, len, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadCache);
for (int i = 0; i < len; ++i) {
Assert.assertEquals(i, is.read());
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
// Tests that deletes go through despite failing initially due to concurrent read
@Test
public void deleteWhileReadTest() throws Exception {
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test1",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertTrue(mTFS.getInfo(file).getInMemoryPercentage() == 100);
// Open the file
InStreamOptions options =
new InStreamOptions.Builder(new TachyonConf())
.setTachyonStorageType(TachyonStorageType.STORE)
.build();
FileInStream in = mTFS.getInStream(file, options);
Assert.assertEquals(0, in.read());
// Delete the file
mTFS.delete(file);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// After the delete, the master should no longer serve the file
Assert.assertNull(mTFS.open(new TachyonURI("/test1")));
// However, the previous read should still be able to read it as the data still exists
byte[] res = new byte[MEM_CAPACITY_BYTES];
Assert.assertEquals(MEM_CAPACITY_BYTES - 1, in.read(res, 1, MEM_CAPACITY_BYTES - 1));
res[0] = 0;
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MEM_CAPACITY_BYTES, res));
in.close();
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// After the file is closed, the master's delete should go through and new files can be created
TachyonFile newFile =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test2",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertTrue(mTFS.getInfo(newFile).getInMemoryPercentage() == 100);
}
private int loadPath(TachyonFileSystem tachyonClient, TachyonURI filePath) throws IOException {
TachyonFile fd;
FileInfo fInfo;
try {
fd = mTfs.open(filePath);
if (fd == null) {
System.out.print(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(filePath));
return -1;
}
fInfo = mTfs.getInfo(fd);
} catch (IOException ioe) {
return -1;
} catch (TachyonException e) {
throw new IOException(e);
}
try {
if (fInfo.isFolder) {
List<FileInfo> files = tachyonClient.listStatus(fd);
Collections.sort(files);
for (FileInfo file : files) {
TachyonURI newPath = new TachyonURI(file.getPath());
if (loadPath(tachyonClient, newPath) == -1) {
return -1;
}
}
return 0;
} else {
Closer closer = Closer.create();
try {
InStreamOptions op =
new InStreamOptions.Builder(mTachyonConf)
.setTachyonStorageType(TachyonStorageType.STORE)
.build();
FileInStream in = closer.register(mTfs.getInStream(fd, op));
byte[] buf = new byte[8 * Constants.MB];
while (in.read(buf) != -1) {}
return 0;
} catch (Throwable e) {
throw closer.rethrow(e);
} finally {
closer.close();
}
}
} catch (TachyonException e) {
return -1;
}
}
/** Test <code>void read(byte[] b)</code>. Read from underfs. */
@Test
public void readTest7() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
Assert.assertEquals(-1, is.read(ret));
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/**
* Test <code>void seek(long pos)</code>. Validate the expected exception for seeking a position
* that is past block size.
*
* @throws IOException
* @throws TachyonException
*/
@Test
public void seekExceptionTest2() throws IOException, TachyonException {
mThrown.expect(IllegalArgumentException.class);
mThrown.expectMessage("Seek position is past EOF: 1, fileSize = 0");
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
try {
is.seek(k + 1);
} finally {
is.close();
}
}
}
/**
* Prints the file's last 1KB of contents to the console.
*
* @param path The TachyonURI path as the input of the command
* @return 0 if command is successful, -1 if an error occurred.f
* @throws IOException
*/
public int tail(TachyonURI path) throws IOException {
TachyonFile fd;
FileInfo fInfo;
try {
fd = mTfs.open(path);
if (fd == null) {
System.out.println(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path));
return -1;
}
fInfo = mTfs.getInfo(fd);
} catch (TachyonException e) {
throw new IOException(e);
}
if (!fInfo.isFolder) {
InStreamOptions op =
new InStreamOptions.Builder(mTachyonConf)
.setTachyonStorageType(TachyonStorageType.NO_STORE)
.build();
FileInStream is = null;
try {
is = mTfs.getInStream(fd, op);
byte[] buf = new byte[Constants.KB];
long bytesToRead = 0L;
if (fInfo.getLength() > Constants.KB) {
bytesToRead = Constants.KB;
} else {
bytesToRead = fInfo.getLength();
}
is.skip(fInfo.getLength() - bytesToRead);
int read = is.read(buf);
if (read != -1) {
System.out.write(buf, 0, read);
}
return 0;
} catch (TachyonException e) {
throw new IOException(e);
} finally {
is.close();
}
} else {
System.out.println(path + " is not a file.");
return -1;
}
}
/**
* Prints the file's contents to the console.
*
* @param path The TachyonURI path as the input of the command
* @return 0 if command is successful, -1 if an error occurred.
* @throws IOException
*/
public int cat(TachyonURI path) throws IOException {
TachyonFile fd;
FileInfo tFile;
try {
fd = mTfs.open(path);
if (fd == null) {
System.out.print(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path));
return -1;
}
tFile = mTfs.getInfo(fd);
} catch (TachyonException e) {
throw new IOException(e);
}
if (!tFile.isFolder) {
InStreamOptions op =
new InStreamOptions.Builder(mTachyonConf)
.setTachyonStorageType(TachyonStorageType.NO_STORE)
.build();
FileInStream is;
try {
is = mTfs.getInStream(fd, op);
} catch (TachyonException e) {
System.out.print(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path));
return -1;
}
byte[] buf = new byte[512];
try {
int read = is.read(buf);
while (read != -1) {
System.out.write(buf, 0, read);
read = is.read(buf);
}
} finally {
is.close();
}
return 0;
} else {
System.out.println(path + " is not a file.");
return -1;
}
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read(byte[])}. */
@Test
public void readTest2() throws IOException, TachyonException {
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
/** Tests that reading a file consisting of more than one block from the underfs works */
@Test
public void readMultiBlockFile() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
int blockSizeByte = 10;
int numBlocks = 10;
FileOutStream os = mTfs.getOutStream(new TachyonURI(uniqPath), mWriteUnderStore);
for (int i = 0; i < numBlocks; i++) {
for (int j = 0; j < blockSizeByte; j++) {
os.write((byte) (i * blockSizeByte + j));
}
}
os.close();
TachyonFile f = mTfs.open(new TachyonURI(uniqPath));
FileInStream is = mTfs.getInStream(f, mReadCache);
for (int i = 0; i < blockSizeByte * numBlocks; i++) {
Assert.assertEquals((byte) i, is.read());
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
/**
* Test <code>void seek(long pos)</code>.
*
* @throws IOException
* @throws TachyonException
*/
@Test
public void seekTest() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
Assert.assertEquals(0, is.read());
is.seek(k / 3);
Assert.assertEquals(k / 3, is.read());
is.seek(k / 2);
Assert.assertEquals(k / 2, is.read());
is.seek(k / 4);
Assert.assertEquals(k / 4, is.read());
is.close();
}
}
/** Tests that seeking around a file cached locally works. */
@Test
public void seekAroundLocalBlock() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
// The number of bytes per remote block read should be set to 100 in the before function
TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 200, mWriteTachyon);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
Assert.assertEquals(0, is.read());
is.seek(199);
Assert.assertEquals(199, is.read());
is.seek(99);
Assert.assertEquals(99, is.read());
is.close();
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#skip(long)}. */
@Test
public void skipTest() throws IOException, TachyonException {
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
Assert.assertEquals(k / 2, is.skip(k / 2));
Assert.assertEquals(k / 2, is.read());
is.close();
is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
int t = k / 3;
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(t, is.read());
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(2 * t + 1, is.read());
is.close();
}
}
}
@Test
public void promoteBlock() throws Exception {
TachyonFile file1 =
TachyonFSTestUtils.createByteFile(
mTFS,
"/root/test1",
TachyonStorageType.STORE,
UnderStorageType.SYNC_PERSIST,
MEM_CAPACITY_BYTES / 6);
TachyonFile file2 =
TachyonFSTestUtils.createByteFile(
mTFS,
"/root/test2",
TachyonStorageType.STORE,
UnderStorageType.SYNC_PERSIST,
MEM_CAPACITY_BYTES / 2);
TachyonFile file3 =
TachyonFSTestUtils.createByteFile(
mTFS,
"/root/test3",
TachyonStorageType.STORE,
UnderStorageType.SYNC_PERSIST,
MEM_CAPACITY_BYTES / 2);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
TachyonFile toPromote = null;
int toPromoteLen = 0;
FileInfo file1Info = mTFS.getInfo(file1);
FileInfo file2Info = mTFS.getInfo(file2);
FileInfo file3Info = mTFS.getInfo(file3);
// We know some file will not be in memory, but not which one since we do not want to make
// any assumptions on the eviction policy
if (file1Info.getInMemoryPercentage() < 100) {
toPromote = file1;
toPromoteLen = (int) file1Info.getLength();
Assert.assertEquals(100, file2Info.getInMemoryPercentage());
Assert.assertEquals(100, file3Info.getInMemoryPercentage());
} else if (file2Info.getInMemoryPercentage() < 100) {
toPromote = file2;
toPromoteLen = (int) file2Info.getLength();
Assert.assertEquals(100, file1Info.getInMemoryPercentage());
Assert.assertEquals(100, file3Info.getInMemoryPercentage());
} else {
toPromote = file3;
toPromoteLen = (int) file3Info.getLength();
Assert.assertEquals(100, file1Info.getInMemoryPercentage());
Assert.assertEquals(100, file2Info.getInMemoryPercentage());
}
FileInStream is =
mTFS.getInStream(
toPromote,
new InStreamOptions.Builder(mWorkerConf)
.setTachyonStorageType(TachyonStorageType.PROMOTE)
.build());
byte[] buf = new byte[toPromoteLen];
int len = is.read(buf);
is.close();
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertEquals(toPromoteLen, len);
Assert.assertEquals(100, mTFS.getInfo(toPromote).getInMemoryPercentage());
}
