/** Test <code>void read(byte[] b, int off, int len)</code>. Read from underfs. */
@Test
public void readTest3() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k / 2];
Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k / 2, ret));
is.close();
if (k == 0) {
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
} else {
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
Assert.assertEquals(k, is.read(ret, 0, k));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
@Test
public void copyFromLocalTest() throws IOException, TException {
File testDir = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir");
testDir.mkdir();
File testDirInner = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir/testDirInner");
testDirInner.mkdir();
File testFile =
generateFileContent("/testDir/testFile", BufferUtils.getIncreasingByteArray(10));
generateFileContent(
"/testDir/testDirInner/testFile2", BufferUtils.getIncreasingByteArray(10, 20));
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"});
Assert.assertEquals(
getCommandOutput(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"}),
mOutput.toString());
TachyonFile file1 = mTfs.open(new TachyonURI("/testDir/testFile"));
TachyonFile file2 = mTfs.open(new TachyonURI("/testDir/testDirInner/testFile2"));
FileInfo fileInfo1 = mTfs.getInfo(file1);
FileInfo fileInfo2 = mTfs.getInfo(file2);
Assert.assertNotNull(fileInfo1);
Assert.assertNotNull(fileInfo2);
Assert.assertEquals(10, fileInfo1.length);
Assert.assertEquals(20, fileInfo2.length);
byte[] read = readContent(file1, 10);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
read = readContent(file2, 20);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, 20, read));
}
/** Test <code>void read()</code>. Read from underfs. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
if (k == 0) {
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
} else {
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
@Test
public void copyFromLocalLargeTest() throws IOException, TException {
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
testFile.createNewFile();
FileOutputStream fos = new FileOutputStream(testFile);
byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
fos.write(toWrite);
fos.close();
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
Assert.assertEquals(
getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
mOutput.toString());
TachyonFile tFile = mTfs.open(new TachyonURI("/testFile"));
FileInfo fileInfo = mTfs.getInfo(tFile);
Assert.assertNotNull(fileInfo);
Assert.assertEquals(SIZE_BYTES, fileInfo.length);
ClientOptions options =
new ClientOptions.Builder(new TachyonConf())
.setTachyonStoreType(TachyonStorageType.NO_STORE)
.build();
FileInStream tfis = mTfs.getInStream(tFile, options);
byte[] read = new byte[SIZE_BYTES];
tfis.read(read);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read()}. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
for (int i = 0; i < 2; i++) {
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
}
/** Test <code>void read()</code>. Read from remote data server. */
@Test
public void readTest4() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read(byte[])}. */
@Test
public void readTest2() throws IOException, TachyonException {
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
// Tests that deletes go through despite failing initially due to concurrent read
@Test
public void deleteWhileReadTest() throws Exception {
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test1",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertTrue(mTFS.getInfo(file).getInMemoryPercentage() == 100);
// Open the file
InStreamOptions options =
new InStreamOptions.Builder(new TachyonConf())
.setTachyonStorageType(TachyonStorageType.STORE)
.build();
FileInStream in = mTFS.getInStream(file, options);
Assert.assertEquals(0, in.read());
// Delete the file
mTFS.delete(file);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// After the delete, the master should no longer serve the file
Assert.assertNull(mTFS.open(new TachyonURI("/test1")));
// However, the previous read should still be able to read it as the data still exists
byte[] res = new byte[MEM_CAPACITY_BYTES];
Assert.assertEquals(MEM_CAPACITY_BYTES - 1, in.read(res, 1, MEM_CAPACITY_BYTES - 1));
res[0] = 0;
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MEM_CAPACITY_BYTES, res));
in.close();
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// After the file is closed, the master's delete should go through and new files can be created
TachyonFile newFile =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test2",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertTrue(mTFS.getInfo(newFile).getInMemoryPercentage() == 100);
}
private void copyToLocalWithBytes(int bytes) throws IOException {
TachyonFSTestUtils.createByteFile(
mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, bytes);
mFsShell.copyToLocal(
new String[] {
"copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
});
Assert.assertEquals(
getCommandOutput(
new String[] {
"copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
}),
mOutput.toString());
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
FileInputStream fis = new FileInputStream(testFile);
byte[] read = new byte[bytes];
fis.read(read);
fis.close();
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytes, read));
}
@Test
public void copyToLocalTest() throws IOException {
TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
mFsShell.copyToLocal(
new String[] {
"copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
});
Assert.assertEquals(
getCommandOutput(
new String[] {
"copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
}),
mOutput.toString());
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
FileInputStream fis = new FileInputStream(testFile);
byte[] read = new byte[10];
fis.read(read);
fis.close();
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
}
@Test
public void copyFromLocalTestWithFullURI() throws IOException {
File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
String tachyonURI =
"tachyon://"
+ mLocalTachyonCluster.getMasterHostname()
+ ":"
+ mLocalTachyonCluster.getMasterPort()
+ "/destFileURI";
// when
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
String cmdOut =
getCommandOutput(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
// then
Assert.assertThat(cmdOut, CoreMatchers.equalTo(mOutput.toString()));
TachyonFile tFile = mTfs.getFile(new TachyonURI("/destFileURI"));
Assert.assertThat(tFile.length(), CoreMatchers.equalTo(10L));
byte[] read = readContent(tFile, 10);
Assert.assertThat(BufferUtils.equalIncreasingByteArray(10, read), CoreMatchers.equalTo(true));
}
@Test
public void copyFromLocalLargeTest() throws IOException {
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
testFile.createNewFile();
FileOutputStream fos = new FileOutputStream(testFile);
byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
fos.write(toWrite);
fos.close();
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
Assert.assertEquals(
getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
mOutput.toString());
TachyonFile tFile = mTfs.getFile(new TachyonURI("/testFile"));
Assert.assertNotNull(tFile);
Assert.assertEquals(SIZE_BYTES, tFile.length());
InStream tfis = tFile.getInStream(ReadType.NO_CACHE);
byte[] read = new byte[SIZE_BYTES];
tfis.read(read);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
}
@Test
public void copyFromLocalTestWithFullURI() throws IOException, TException {
File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
String tachyonURI =
"tachyon://"
+ mLocalTachyonCluster.getMasterHostname()
+ ":"
+ mLocalTachyonCluster.getMasterPort()
+ "/destFileURI";
// when
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
String cmdOut =
getCommandOutput(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
// then
Assert.assertEquals(cmdOut, mOutput.toString());
TachyonFile file = mTfs.open(new TachyonURI("/destFileURI"));
FileInfo fileInfo = mTfs.getInfo(file);
Assert.assertEquals(10L, fileInfo.length);
byte[] read = readContent(file, 10);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
}
/** Tests creating and opening a store with a number of key. */
@Test
public void createAndOpenStoreWithMultiKeysTest() throws Exception {
final int numKeys = 100;
final int keyLength = 4; // 4Byte key
final int valueLength = 5 * Constants.KB; // 5KB value
mWriter = sKeyValueSystem.createStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
}
mWriter.close();
mReader = sKeyValueSystem.openStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = mReader.get(key);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
}
Assert.assertNull(mReader.get(KEY1));
Assert.assertNull(mReader.get(KEY2));
mReader.close();
}
/** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
@Test
public void readTest6() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k / 2];
int start = 0;
while (start < k / 2) {
int read = is.read(ret, 0, (k / 2) - start);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
start += read;
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/**
* Tests creating and opening a store with a number of keys, while each key-value pair is large
* enough to take a separate key-value partition.
*/
@Test
public void createMultiPartitionsTest() throws Exception {
// TODO(cc): Remove codes using createStoreOfMultiplePartitions.
final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
final int numKeys = 10;
final int keyLength = 4; // 4Byte key
final int valueLength = 500 * Constants.KB; // 500KB value
FileSystem fs = FileSystem.Factory.get();
ClientContext.getConf()
.set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
mWriter = sKeyValueSystem.createStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
}
mWriter.close();
List<URIStatus> files = fs.listStatus(mStoreUri);
Assert.assertEquals(numKeys, files.size());
for (URIStatus info : files) {
Assert.assertTrue(info.getLength() <= maxPartitionSize);
}
mReader = sKeyValueSystem.openStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = mReader.get(key);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
}
Assert.assertNull(mReader.get(KEY1));
Assert.assertNull(mReader.get(KEY2));
mReader.close();
}