@Test
public void copyFromLocalLargeTest() throws IOException, TException {
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
testFile.createNewFile();
FileOutputStream fos = new FileOutputStream(testFile);
byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
fos.write(toWrite);
fos.close();
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
Assert.assertEquals(
getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
mOutput.toString());
TachyonFile tFile = mTfs.open(new TachyonURI("/testFile"));
FileInfo fileInfo = mTfs.getInfo(tFile);
Assert.assertNotNull(fileInfo);
Assert.assertEquals(SIZE_BYTES, fileInfo.length);
ClientOptions options =
new ClientOptions.Builder(new TachyonConf())
.setTachyonStoreType(TachyonStorageType.NO_STORE)
.build();
FileInStream tfis = mTfs.getInStream(tFile, options);
byte[] read = new byte[SIZE_BYTES];
tfis.read(read);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
}
/**
* Creates a store with the specified number of partitions.
*
* <p>NOTE: calling this method will set {@link Constants#KEY_VALUE_PARTITION_SIZE_BYTES_MAX} to
* {@link Constants#MB}.
*
* @param partitionNumber the number of partitions
* @param keyValuePairs the key-value pairs in the store, null if you don't want to know them
* @return the URI to the created store
*/
private TachyonURI createStoreOfMultiplePartitions(
int partitionNumber, List<KeyValuePair> keyValuePairs) throws Exception {
// These sizes are carefully selected, one partition holds only one key-value pair.
final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
ClientContext.getConf()
.set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
final int keyLength = 4; // 4Byte key
final int valueLength = 500 * Constants.KB; // 500KB value
TachyonURI storeUri = new TachyonURI(PathUtils.uniqPath());
mWriter = sKeyValueSystem.createStore(storeUri);
for (int i = 0; i < partitionNumber; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
if (keyValuePairs != null) {
keyValuePairs.add(new KeyValuePair(key, value));
}
}
mWriter.close();
Assert.assertEquals(partitionNumber, getPartitionNumber(storeUri));
return storeUri;
}
/** Test <code>void read(byte[] b, int off, int len)</code>. Read from underfs. */
@Test
public void readTest3() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k / 2];
Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k / 2, ret));
is.close();
if (k == 0) {
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
} else {
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
Assert.assertEquals(k, is.read(ret, 0, k));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
@Test
public void copyFromLocalTest() throws IOException, TException {
File testDir = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir");
testDir.mkdir();
File testDirInner = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir/testDirInner");
testDirInner.mkdir();
File testFile =
generateFileContent("/testDir/testFile", BufferUtils.getIncreasingByteArray(10));
generateFileContent(
"/testDir/testDirInner/testFile2", BufferUtils.getIncreasingByteArray(10, 20));
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"});
Assert.assertEquals(
getCommandOutput(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"}),
mOutput.toString());
TachyonFile file1 = mTfs.open(new TachyonURI("/testDir/testFile"));
TachyonFile file2 = mTfs.open(new TachyonURI("/testDir/testDirInner/testFile2"));
FileInfo fileInfo1 = mTfs.getInfo(file1);
FileInfo fileInfo2 = mTfs.getInfo(file2);
Assert.assertNotNull(fileInfo1);
Assert.assertNotNull(fileInfo2);
Assert.assertEquals(10, fileInfo1.length);
Assert.assertEquals(20, fileInfo2.length);
byte[] read = readContent(file1, 10);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
read = readContent(file2, 20);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, 20, read));
}
/** Test <code>void read()</code>. Read from underfs. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
if (k == 0) {
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
} else {
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
@Override
public int directRead(byte[] b, int off, int len) throws IOException {
ByteBuffer buf = mLocalFileChannel.map(FileChannel.MapMode.READ_ONLY, getPosition(), len);
buf.get(b, off, len);
BufferUtils.cleanDirectBuffer(buf);
return len;
}
/**
* Tests putting a key-value pair that is larger than the max key-value partition size, expecting
* exception thrown.
*/
@Test
public void putKeyValueTooLargeTest() throws Exception {
final long maxPartitionSize = 500 * Constants.KB; // Each partition is at most 500 KB
final int keyLength = 4; // 4Byte key
final int valueLength = 500 * Constants.KB; // 500KB value
ClientContext.getConf()
.set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
mWriter = sKeyValueSystem.createStore(mStoreUri);
byte[] key = BufferUtils.getIncreasingByteArray(0, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(0, valueLength);
mThrown.expect(IOException.class);
mThrown.expectMessage(ExceptionMessage.KEY_VALUE_TOO_LARGE.getMessage(keyLength, valueLength));
mWriter.put(key, value);
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read()}. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
for (int i = 0; i < 2; i++) {
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
}
@Test
public void catTest() throws IOException {
TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
mFsShell.cat(new String[] {"cat", "/testFile"});
byte[] expect = BufferUtils.getIncreasingByteArray(10);
Assert.assertArrayEquals(expect, mOutput.toByteArray());
}
/** Test <code>void read()</code>. Read from remote data server. */
@Test
public void readTest4() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
@Override
protected void bufferedRead(int len) throws IOException {
if (mBuffer.isDirect()) { // Buffer may not be direct on initialization
BufferUtils.cleanDirectBuffer(mBuffer);
}
mBuffer = mLocalFileChannel.map(FileChannel.MapMode.READ_ONLY, getPosition(), len);
}
/** Asserts that the message back matches the block response protocols. */
private void assertValid(
final DataServerMessage msg,
final int expectedSize,
final long blockId,
final long offset,
final long length) {
assertValid(msg, BufferUtils.getIncreasingByteBuffer(expectedSize), blockId, offset, length);
}
@Test
public void catTest() throws IOException {
TachyonFSTestUtils.createByteFile(
mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
mFsShell.run(new String[] {"cat", "/testFile"});
byte[] expect = BufferUtils.getIncreasingByteArray(10);
Assert.assertArrayEquals(expect, mOutput.toByteArray());
}
@Override
public void write(int b) throws IOException {
checkIfClosed();
Preconditions.checkState(mWrittenBytes + 1 <= mBlockSize, PreconditionMessage.ERR_END_OF_BLOCK);
if (mBuffer.position() >= mBuffer.limit()) {
flush();
}
BufferUtils.putIntByteBuffer(mBuffer, b);
mWrittenBytes++;
}
@Test
public void copyFromLocalTestWithFullURI() throws IOException {
File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
String tachyonURI =
"tachyon://"
+ mLocalTachyonCluster.getMasterHostname()
+ ":"
+ mLocalTachyonCluster.getMasterPort()
+ "/destFileURI";
// when
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
String cmdOut =
getCommandOutput(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
// then
Assert.assertThat(cmdOut, CoreMatchers.equalTo(mOutput.toString()));
TachyonFile tFile = mTfs.getFile(new TachyonURI("/destFileURI"));
Assert.assertThat(tFile.length(), CoreMatchers.equalTo(10L));
byte[] read = readContent(tFile, 10);
Assert.assertThat(BufferUtils.equalIncreasingByteArray(10, read), CoreMatchers.equalTo(true));
}
@Test
public void copyFromLocalLargeTest() throws IOException {
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
testFile.createNewFile();
FileOutputStream fos = new FileOutputStream(testFile);
byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
fos.write(toWrite);
fos.close();
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
Assert.assertEquals(
getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
mOutput.toString());
TachyonFile tFile = mTfs.getFile(new TachyonURI("/testFile"));
Assert.assertNotNull(tFile);
Assert.assertEquals(SIZE_BYTES, tFile.length());
InStream tfis = tFile.getInStream(ReadType.NO_CACHE);
byte[] read = new byte[SIZE_BYTES];
tfis.read(read);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read(byte[])}. */
@Test
public void readTest2() throws IOException, TachyonException {
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
@Test
public void copyFromLocalTestWithFullURI() throws IOException, TException {
File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
String tachyonURI =
"tachyon://"
+ mLocalTachyonCluster.getMasterHostname()
+ ":"
+ mLocalTachyonCluster.getMasterPort()
+ "/destFileURI";
// when
mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
String cmdOut =
getCommandOutput(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
// then
Assert.assertEquals(cmdOut, mOutput.toString());
TachyonFile file = mTfs.open(new TachyonURI("/destFileURI"));
FileInfo fileInfo = mTfs.getInfo(file);
Assert.assertEquals(10L, fileInfo.length);
byte[] read = readContent(file, 10);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
}
@Override
public int read() throws IOException {
checkIfClosed();
if (remaining() == 0) {
close();
return -1;
}
if (!mBufferIsValid || mBuffer.remaining() == 0) {
updateBuffer();
}
mPos++;
return BufferUtils.byteToInt(mBuffer.get());
}
/** Tests creating and opening a store with a number of key. */
@Test
public void createAndOpenStoreWithMultiKeysTest() throws Exception {
final int numKeys = 100;
final int keyLength = 4; // 4Byte key
final int valueLength = 5 * Constants.KB; // 5KB value
mWriter = sKeyValueSystem.createStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
}
mWriter.close();
mReader = sKeyValueSystem.openStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = mReader.get(key);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
}
Assert.assertNull(mReader.get(KEY1));
Assert.assertNull(mReader.get(KEY2));
mReader.close();
}
// Tests that deletes go through despite failing initially due to concurrent read
@Test
public void deleteWhileReadTest() throws Exception {
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test1",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertTrue(mTFS.getInfo(file).getInMemoryPercentage() == 100);
// Open the file
InStreamOptions options =
new InStreamOptions.Builder(new TachyonConf())
.setTachyonStorageType(TachyonStorageType.STORE)
.build();
FileInStream in = mTFS.getInStream(file, options);
Assert.assertEquals(0, in.read());
// Delete the file
mTFS.delete(file);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// After the delete, the master should no longer serve the file
Assert.assertNull(mTFS.open(new TachyonURI("/test1")));
// However, the previous read should still be able to read it as the data still exists
byte[] res = new byte[MEM_CAPACITY_BYTES];
Assert.assertEquals(MEM_CAPACITY_BYTES - 1, in.read(res, 1, MEM_CAPACITY_BYTES - 1));
res[0] = 0;
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MEM_CAPACITY_BYTES, res));
in.close();
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// After the file is closed, the master's delete should go through and new files can be created
TachyonFile newFile =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test2",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertTrue(mTFS.getInfo(newFile).getInMemoryPercentage() == 100);
}
/**
* Tests creating and opening a store with a number of keys, while each key-value pair is large
* enough to take a separate key-value partition.
*/
@Test
public void createMultiPartitionsTest() throws Exception {
// TODO(cc): Remove codes using createStoreOfMultiplePartitions.
final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
final int numKeys = 10;
final int keyLength = 4; // 4Byte key
final int valueLength = 500 * Constants.KB; // 500KB value
FileSystem fs = FileSystem.Factory.get();
ClientContext.getConf()
.set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
mWriter = sKeyValueSystem.createStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
}
mWriter.close();
List<URIStatus> files = fs.listStatus(mStoreUri);
Assert.assertEquals(numKeys, files.size());
for (URIStatus info : files) {
Assert.assertTrue(info.getLength() <= maxPartitionSize);
}
mReader = sKeyValueSystem.openStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = mReader.get(key);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
}
Assert.assertNull(mReader.get(KEY1));
Assert.assertNull(mReader.get(KEY2));
mReader.close();
}
@Test
public void readPartialTest2() throws TException, IOException {
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
BlockInfo block = getFirstBlockInfo(file);
final int offset = 2;
final int length = 6;
DataServerMessage recvMsg = request(block, offset, length);
assertValid(
recvMsg,
BufferUtils.getIncreasingByteBuffer(offset, length),
block.getBlockId(),
offset,
length);
}
private void copyToLocalWithBytes(int bytes) throws IOException {
TachyonFSTestUtils.createByteFile(
mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, bytes);
mFsShell.copyToLocal(
new String[] {
"copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
});
Assert.assertEquals(
getCommandOutput(
new String[] {
"copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
}),
mOutput.toString());
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
FileInputStream fis = new FileInputStream(testFile);
byte[] read = new byte[bytes];
fis.read(read);
fis.close();
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytes, read));
}
@Test
public void copyToLocalTest() throws IOException {
TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
mFsShell.copyToLocal(
new String[] {
"copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
});
Assert.assertEquals(
getCommandOutput(
new String[] {
"copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
}),
mOutput.toString());
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
FileInputStream fis = new FileInputStream(testFile);
byte[] read = new byte[10];
fis.read(read);
fis.close();
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
}
@Test
public void readThroughClientTest() throws IOException, TException {
final int length = 10;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
RemoteBlockReader client =
RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
ByteBuffer result =
client.readRemoteBlock(
new InetSocketAddress(
block.getLocations().get(0).getWorkerAddress().getHost(),
block.getLocations().get(0).getWorkerAddress().getDataPort()),
block.getBlockId(),
0,
length);
Assert.assertEquals(BufferUtils.getIncreasingByteBuffer(length), result);
}
/** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
@Test
public void readTest6() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k / 2];
int start = 0;
while (start < k / 2) {
int read = is.read(ret, 0, (k / 2) - start);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
start += read;
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
@Override
public void close() throws IOException {
if (mClosed) {
return;
}
try {
if (mBlockIsRead) {
mWorkerClient.accessBlock(mBlockId);
ClientContext.getClientMetrics().incBlocksReadLocal(1);
}
mWorkerClient.unlockBlock(mBlockId);
} catch (TachyonException e) {
throw new IOException(e);
} finally {
mContext.releaseWorkerClient(mWorkerClient);
mCloser.close();
if (mBuffer != null && mBuffer.isDirect()) {
BufferUtils.cleanDirectBuffer(mBuffer);
}
}
mClosed = true;
}