/** Test <code>void read()</code>. Read from remote data server. */
@Test
public void readTest4() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
// TODO(calvin): Make this work with the new BlockReader.
// @Test
public void readThroughClientNonExistentTest() throws IOException, TException {
final int length = 10;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
// Get the maximum block id, for use in determining a non-existent block id.
FileInfo fileInfo = mTFS.getInfo(file);
long maxBlockId = block.getBlockId();
for (long blockId : fileInfo.blockIds) {
if (blockId > maxBlockId) {
maxBlockId = blockId;
}
}
RemoteBlockReader client =
RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
ByteBuffer result =
client.readRemoteBlock(
new InetSocketAddress(
block.getLocations().get(0).getWorkerAddress().getHost(),
block.getLocations().get(0).getWorkerAddress().getDataPort()),
maxBlockId + 1,
0,
length);
Assert.assertNull(result);
}
@Test
public void readTest() throws IOException, TException {
final int length = 10;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
DataServerMessage recvMsg = request(block);
assertValid(recvMsg, length, block.getBlockId(), 0, length);
}
@Test
public void readPartialTest1() throws TException, IOException {
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
BlockInfo block = getFirstBlockInfo(file);
final int offset = 0;
final int length = 6;
DataServerMessage recvMsg = request(block, offset, length);
assertValid(recvMsg, length, block.getBlockId(), offset, length);
}
public void write(org.apache.thrift.protocol.TProtocol oprot, BlockInfo struct)
throws org.apache.thrift.TException {
struct.validate();
oprot.writeStructBegin(STRUCT_DESC);
oprot.writeFieldBegin(BLOCK_ID_FIELD_DESC);
oprot.writeI64(struct.blockId);
oprot.writeFieldEnd();
oprot.writeFieldBegin(LENGTH_FIELD_DESC);
oprot.writeI64(struct.length);
oprot.writeFieldEnd();
if (struct.locations != null) {
oprot.writeFieldBegin(LOCATIONS_FIELD_DESC);
{
oprot.writeListBegin(
new org.apache.thrift.protocol.TList(
org.apache.thrift.protocol.TType.STRUCT, struct.locations.size()));
for (BlockLocation _iter3 : struct.locations) {
_iter3.write(oprot);
}
oprot.writeListEnd();
}
oprot.writeFieldEnd();
}
oprot.writeFieldStop();
oprot.writeStructEnd();
}
public boolean equals(BlockInfo that) {
if (that == null) return false;
boolean this_present_blockId = true;
boolean that_present_blockId = true;
if (this_present_blockId || that_present_blockId) {
if (!(this_present_blockId && that_present_blockId)) return false;
if (this.blockId != that.blockId) return false;
}
boolean this_present_length = true;
boolean that_present_length = true;
if (this_present_length || that_present_length) {
if (!(this_present_length && that_present_length)) return false;
if (this.length != that.length) return false;
}
boolean this_present_locations = true && this.isSetLocations();
boolean that_present_locations = true && that.isSetLocations();
if (this_present_locations || that_present_locations) {
if (!(this_present_locations && that_present_locations)) return false;
if (!this.locations.equals(that.locations)) return false;
}
return true;
}
/**
* Create a new socket to the data port and send a block request. The returned value is the
* response from the server.
*/
private DataServerMessage request(final BlockInfo block, final long offset, final long length)
throws IOException, TException {
DataServerMessage sendMsg =
DataServerMessage.createBlockRequestMessage(block.blockId, offset, length);
SocketChannel socketChannel =
SocketChannel.open(
new InetSocketAddress(
block.getLocations().get(0).getWorkerAddress().getHost(),
block.getLocations().get(0).getWorkerAddress().getDataPort()));
try {
while (!sendMsg.finishSending()) {
sendMsg.send(socketChannel);
}
DataServerMessage recvMsg =
DataServerMessage.createBlockResponseMessage(false, block.blockId, offset, length, null);
while (!recvMsg.isMessageReady()) {
int numRead = recvMsg.recv(socketChannel);
if (numRead == -1) {
break;
}
}
return recvMsg;
} finally {
socketChannel.close();
}
}
/** Performs a deep copy on <i>other</i>. */
public BlockInfo(BlockInfo other) {
__isset_bitfield = other.__isset_bitfield;
this.blockId = other.blockId;
this.length = other.length;
if (other.isSetLocations()) {
List<BlockLocation> __this__locations = new ArrayList<BlockLocation>(other.locations.size());
for (BlockLocation other_element : other.locations) {
__this__locations.add(other_element);
}
this.locations = __this__locations;
}
}
@Test
public void readThroughClientTest() throws IOException, TException {
final int length = 10;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
RemoteBlockReader client =
RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
ByteBuffer result =
client.readRemoteBlock(
new InetSocketAddress(
block.getLocations().get(0).getWorkerAddress().getHost(),
block.getLocations().get(0).getWorkerAddress().getDataPort()),
block.getBlockId(),
0,
length);
Assert.assertEquals(BufferUtils.getIncreasingByteBuffer(length), result);
}
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, BlockInfo struct)
throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
BitSet incoming = iprot.readBitSet(3);
if (incoming.get(0)) {
struct.blockId = iprot.readI64();
struct.setBlockIdIsSet(true);
}
if (incoming.get(1)) {
struct.length = iprot.readI64();
struct.setLengthIsSet(true);
}
if (incoming.get(2)) {
{
org.apache.thrift.protocol.TList _list5 =
new org.apache.thrift.protocol.TList(
org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
struct.locations = new ArrayList<BlockLocation>(_list5.size);
BlockLocation _elem6;
for (int _i7 = 0; _i7 < _list5.size; ++_i7) {
_elem6 = new BlockLocation();
_elem6.read(iprot);
struct.locations.add(_elem6);
}
}
struct.setLocationsIsSet(true);
}
}
@Override
public void write(org.apache.thrift.protocol.TProtocol prot, BlockInfo struct)
throws org.apache.thrift.TException {
TTupleProtocol oprot = (TTupleProtocol) prot;
BitSet optionals = new BitSet();
if (struct.isSetBlockId()) {
optionals.set(0);
}
if (struct.isSetLength()) {
optionals.set(1);
}
if (struct.isSetLocations()) {
optionals.set(2);
}
oprot.writeBitSet(optionals, 3);
if (struct.isSetBlockId()) {
oprot.writeI64(struct.blockId);
}
if (struct.isSetLength()) {
oprot.writeI64(struct.length);
}
if (struct.isSetLocations()) {
{
oprot.writeI32(struct.locations.size());
for (BlockLocation _iter4 : struct.locations) {
_iter4.write(oprot);
}
}
}
}
@Override
public int compareTo(BlockInfo other) {
if (!getClass().equals(other.getClass())) {
return getClass().getName().compareTo(other.getClass().getName());
}
int lastComparison = 0;
lastComparison = Boolean.valueOf(isSetBlockId()).compareTo(other.isSetBlockId());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetBlockId()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.blockId, other.blockId);
if (lastComparison != 0) {
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetLength()).compareTo(other.isSetLength());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetLength()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.length, other.length);
if (lastComparison != 0) {
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetLocations()).compareTo(other.isSetLocations());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetLocations()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.locations, other.locations);
if (lastComparison != 0) {
return lastComparison;
}
}
return 0;
}
public void read(org.apache.thrift.protocol.TProtocol iprot, BlockInfo struct)
throws org.apache.thrift.TException {
org.apache.thrift.protocol.TField schemeField;
iprot.readStructBegin();
while (true) {
schemeField = iprot.readFieldBegin();
if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
break;
}
switch (schemeField.id) {
case 1: // BLOCK_ID
if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
struct.blockId = iprot.readI64();
struct.setBlockIdIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 2: // LENGTH
if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
struct.length = iprot.readI64();
struct.setLengthIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 3: // LOCATIONS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
org.apache.thrift.protocol.TList _list0 = iprot.readListBegin();
struct.locations = new ArrayList<BlockLocation>(_list0.size);
BlockLocation _elem1;
for (int _i2 = 0; _i2 < _list0.size; ++_i2) {
_elem1 = new BlockLocation();
_elem1.read(iprot);
struct.locations.add(_elem1);
}
iprot.readListEnd();
}
struct.setLocationsIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
iprot.readFieldEnd();
}
iprot.readStructEnd();
// check for required fields of primitive type, which can't be checked in the validate method
struct.validate();
}
@Test
public void readMultiFiles() throws IOException, TException {
final int length = WORKER_CAPACITY_BYTES / 2 + 1;
TachyonFile file1 =
TachyonFSTestUtils.createByteFile(
mTFS, "/readFile1", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block1 = getFirstBlockInfo(file1);
DataServerMessage recvMsg1 = request(block1);
assertValid(recvMsg1, length, block1.getBlockId(), 0, length);
TachyonFile file2 =
TachyonFSTestUtils.createByteFile(
mTFS, "/readFile2", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block2 = getFirstBlockInfo(file2);
DataServerMessage recvMsg2 = request(block2);
assertValid(recvMsg2, length, block2.getBlockId(), 0, length);
CommonUtils.sleepMs(
mWorkerTachyonConf.getInt(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS) * 2 + 10);
FileInfo fileInfo = mTFS.getInfo(mTFS.open(new TachyonURI("/readFile1")));
Assert.assertEquals(0, fileInfo.inMemoryPercentage);
}
/** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
@Test
public void readTest6() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k / 2];
int start = 0;
while (start < k / 2) {
int read = is.read(ret, 0, (k / 2) - start);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
start += read;
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}