/** Test <code>void read(byte[] b, int off, int len)</code>. Read from underfs. */
@Test
public void readTest3() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k / 2];
Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k / 2, ret));
is.close();
if (k == 0) {
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
} else {
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
Assert.assertEquals(k, is.read(ret, 0, k));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read()}. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
for (int i = 0; i < 2; i++) {
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
}
/** Test <code>long skip(long len)</code>. */
@Test
public void skipTest() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadCache);
Assert.assertEquals(k / 2, is.skip(k / 2));
Assert.assertEquals(k / 2, is.read());
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
if (k >= 3) {
is = mTfs.getInStream(f, mReadCache);
int t = k / 3;
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(t, is.read());
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(2 * t + 1, is.read());
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
}
/**
* Creates a store with the specified number of partitions.
*
* <p>NOTE: calling this method will set {@link Constants#KEY_VALUE_PARTITION_SIZE_BYTES_MAX} to
* {@link Constants#MB}.
*
* @param partitionNumber the number of partitions
* @param keyValuePairs the key-value pairs in the store, null if you don't want to know them
* @return the URI to the created store
*/
private TachyonURI createStoreOfMultiplePartitions(
int partitionNumber, List<KeyValuePair> keyValuePairs) throws Exception {
// These sizes are carefully selected, one partition holds only one key-value pair.
final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
ClientContext.getConf()
.set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
final int keyLength = 4; // 4Byte key
final int valueLength = 500 * Constants.KB; // 500KB value
TachyonURI storeUri = new TachyonURI(PathUtils.uniqPath());
mWriter = sKeyValueSystem.createStore(storeUri);
for (int i = 0; i < partitionNumber; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
if (keyValuePairs != null) {
keyValuePairs.add(new KeyValuePair(key, value));
}
}
mWriter.close();
Assert.assertEquals(partitionNumber, getPartitionNumber(storeUri));
return storeUri;
}
/** Test <code>void read()</code>. Read from remote data server. */
@Test
public void readTest4() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/** Test <code>void read()</code>. Read from underfs. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
if (k == 0) {
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
} else {
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/**
* Tests that not reading a file the whole way through then closing it will cause it to not
* recache
*/
@Test
public void incompleteFileReadCancelsRecache() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 2, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
Assert.assertEquals(0, is.read());
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadNoCache);
is.close();
}
/** Tests that reading a file the whole way through with the STORE ReadType will recache it */
@Test
public void completeFileReadTriggersRecache() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
int len = 2;
TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, len, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadCache);
for (int i = 0; i < len; ++i) {
Assert.assertEquals(i, is.read());
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
/** Tests that seeking around a file cached locally works. */
@Test
public void seekAroundLocalBlock() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
// The number of bytes per remote block read should be set to 100 in the before function
TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 200, mWriteTachyon);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
Assert.assertEquals(0, is.read());
is.seek(199);
Assert.assertEquals(199, is.read());
is.seek(99);
Assert.assertEquals(99, is.read());
is.close();
}
/**
* Test <code>void seek(long pos)</code>. Validate the expected exception for seeking a position
* that is past block size.
*
* @throws IOException
* @throws TachyonException
*/
@Test
public void seekExceptionTest2() throws IOException, TachyonException {
mThrown.expect(IllegalArgumentException.class);
mThrown.expectMessage("Seek position is past EOF: 1, fileSize = 0");
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
try {
is.seek(k + 1);
} finally {
is.close();
}
}
}
@BeforeClass
public static final void beforeClass() throws Exception {
sFileSystem = sLocalTachyonClusterResource.get().getClient();
sTachyonConf = sLocalTachyonClusterResource.get().getMasterTachyonConf();
sWriteBoth = StreamOptionUtils.getCreateFileOptionsCacheThrough(sTachyonConf);
sWriteTachyon = StreamOptionUtils.getCreateFileOptionsMustCache(sTachyonConf);
sWriteUnderStore = StreamOptionUtils.getCreateFileOptionsThrough(sTachyonConf);
sTestPath = PathUtils.uniqPath();
// Create files of varying size and write type to later read from
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
}
}
}
/**
* Creates a store with the specified number of key-value pairs. The key-value pairs are in the
* format specified in {@link #genBaseKey(int)} and {@link #genBaseValue(int)} with id starts from
* 0.
*
* <p>The created store's size is {@link Assert}ed before return.
*
* @param size the number of key-value pairs
* @param pairs the key-value pairs in the store, null if you don't want to know them
* @return the URI to the store
* @throws Exception if any error happens
*/
private TachyonURI createStoreOfSize(int size, List<KeyValuePair> pairs) throws Exception {
TachyonURI path = new TachyonURI(PathUtils.uniqPath());
KeyValueStoreWriter writer = sKeyValueSystem.createStore(path);
for (int i = 0; i < size; i++) {
byte[] key = genBaseKey(i).getBytes();
byte[] value = genBaseValue(i).getBytes();
writer.put(key, value);
if (pairs != null) {
pairs.add(new KeyValuePair(key, value));
}
}
writer.close();
Assert.assertEquals(size, sKeyValueSystem.openStore(path).size());
return path;
}
/**
* Test <code>void seek(long pos)</code>.
*
* @throws IOException
* @throws TachyonException
*/
@Test
public void seekTest() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
Assert.assertEquals(0, is.read());
is.seek(k / 3);
Assert.assertEquals(k / 3, is.read());
is.seek(k / 2);
Assert.assertEquals(k / 2, is.read());
is.seek(k / 4);
Assert.assertEquals(k / 4, is.read());
is.close();
}
}
/** Tests that reading a file consisting of more than one block from the underfs works */
@Test
public void readMultiBlockFile() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
int blockSizeByte = 10;
int numBlocks = 10;
FileOutStream os = mTfs.getOutStream(new TachyonURI(uniqPath), mWriteUnderStore);
for (int i = 0; i < numBlocks; i++) {
for (int j = 0; j < blockSizeByte; j++) {
os.write((byte) (i * blockSizeByte + j));
}
}
os.close();
TachyonFile f = mTfs.open(new TachyonURI(uniqPath));
FileInStream is = mTfs.getInStream(f, mReadCache);
for (int i = 0; i < blockSizeByte * numBlocks; i++) {
Assert.assertEquals((byte) i, is.read());
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read(byte[])}. */
@Test
public void readTest2() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
/** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
@Test
public void readTest6() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k / 2];
int start = 0;
while (start < k / 2) {
int read = is.read(ret, 0, (k / 2) - start);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
start += read;
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#skip(long)}. */
@Test
public void skipTest() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
Assert.assertEquals(k / 2, is.skip(k / 2));
Assert.assertEquals(k / 2, is.read());
is.close();
is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
int t = k / 3;
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(t, is.read());
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(2 * t + 1, is.read());
is.close();
}
}
}
@Before
public void before() throws Exception {
mStoreUri = new TachyonURI(PathUtils.uniqPath());
}
