/** Test <code>void read()</code>. Read from remote data server. */
  @Test
  public void readTest4() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);

      long blockId = mTfs.getInfo(f).getBlockIds().get(0);
      BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
      RemoteBlockInStream is =
          new RemoteBlockInStream(
              info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
      byte[] ret = new byte[k];
      int value = is.read();
      int cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
  // TODO(calvin): Make this work with the new BlockReader.
  // @Test
  public void readThroughClientNonExistentTest() throws IOException, TException {
    final int length = 10;
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
    BlockInfo block = getFirstBlockInfo(file);

    // Get the maximum block id, for use in determining a non-existent block id.
    FileInfo fileInfo = mTFS.getInfo(file);
    long maxBlockId = block.getBlockId();
    for (long blockId : fileInfo.blockIds) {
      if (blockId > maxBlockId) {
        maxBlockId = blockId;
      }
    }

    RemoteBlockReader client =
        RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
    ByteBuffer result =
        client.readRemoteBlock(
            new InetSocketAddress(
                block.getLocations().get(0).getWorkerAddress().getHost(),
                block.getLocations().get(0).getWorkerAddress().getDataPort()),
            maxBlockId + 1,
            0,
            length);

    Assert.assertNull(result);
  }
 @Test
 public void readTest() throws IOException, TException {
   final int length = 10;
   TachyonFile file =
       TachyonFSTestUtils.createByteFile(
           mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
   BlockInfo block = getFirstBlockInfo(file);
   DataServerMessage recvMsg = request(block);
   assertValid(recvMsg, length, block.getBlockId(), 0, length);
 }
 @Test
 public void readPartialTest1() throws TException, IOException {
   TachyonFile file =
       TachyonFSTestUtils.createByteFile(
           mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
   BlockInfo block = getFirstBlockInfo(file);
   final int offset = 0;
   final int length = 6;
   DataServerMessage recvMsg = request(block, offset, length);
   assertValid(recvMsg, length, block.getBlockId(), offset, length);
 }
Ejemplo n.º 5
0
    public void write(org.apache.thrift.protocol.TProtocol oprot, BlockInfo struct)
        throws org.apache.thrift.TException {
      struct.validate();

      oprot.writeStructBegin(STRUCT_DESC);
      oprot.writeFieldBegin(BLOCK_ID_FIELD_DESC);
      oprot.writeI64(struct.blockId);
      oprot.writeFieldEnd();
      oprot.writeFieldBegin(LENGTH_FIELD_DESC);
      oprot.writeI64(struct.length);
      oprot.writeFieldEnd();
      if (struct.locations != null) {
        oprot.writeFieldBegin(LOCATIONS_FIELD_DESC);
        {
          oprot.writeListBegin(
              new org.apache.thrift.protocol.TList(
                  org.apache.thrift.protocol.TType.STRUCT, struct.locations.size()));
          for (BlockLocation _iter3 : struct.locations) {
            _iter3.write(oprot);
          }
          oprot.writeListEnd();
        }
        oprot.writeFieldEnd();
      }
      oprot.writeFieldStop();
      oprot.writeStructEnd();
    }
Ejemplo n.º 6
0
  public boolean equals(BlockInfo that) {
    if (that == null) return false;

    boolean this_present_blockId = true;
    boolean that_present_blockId = true;
    if (this_present_blockId || that_present_blockId) {
      if (!(this_present_blockId && that_present_blockId)) return false;
      if (this.blockId != that.blockId) return false;
    }

    boolean this_present_length = true;
    boolean that_present_length = true;
    if (this_present_length || that_present_length) {
      if (!(this_present_length && that_present_length)) return false;
      if (this.length != that.length) return false;
    }

    boolean this_present_locations = true && this.isSetLocations();
    boolean that_present_locations = true && that.isSetLocations();
    if (this_present_locations || that_present_locations) {
      if (!(this_present_locations && that_present_locations)) return false;
      if (!this.locations.equals(that.locations)) return false;
    }

    return true;
  }
 /**
  * Create a new socket to the data port and send a block request. The returned value is the
  * response from the server.
  */
 private DataServerMessage request(final BlockInfo block, final long offset, final long length)
     throws IOException, TException {
   DataServerMessage sendMsg =
       DataServerMessage.createBlockRequestMessage(block.blockId, offset, length);
   SocketChannel socketChannel =
       SocketChannel.open(
           new InetSocketAddress(
               block.getLocations().get(0).getWorkerAddress().getHost(),
               block.getLocations().get(0).getWorkerAddress().getDataPort()));
   try {
     while (!sendMsg.finishSending()) {
       sendMsg.send(socketChannel);
     }
     DataServerMessage recvMsg =
         DataServerMessage.createBlockResponseMessage(false, block.blockId, offset, length, null);
     while (!recvMsg.isMessageReady()) {
       int numRead = recvMsg.recv(socketChannel);
       if (numRead == -1) {
         break;
       }
     }
     return recvMsg;
   } finally {
     socketChannel.close();
   }
 }
Ejemplo n.º 8
0
 /** Performs a deep copy on <i>other</i>. */
 public BlockInfo(BlockInfo other) {
   __isset_bitfield = other.__isset_bitfield;
   this.blockId = other.blockId;
   this.length = other.length;
   if (other.isSetLocations()) {
     List<BlockLocation> __this__locations = new ArrayList<BlockLocation>(other.locations.size());
     for (BlockLocation other_element : other.locations) {
       __this__locations.add(other_element);
     }
     this.locations = __this__locations;
   }
 }
  @Test
  public void readThroughClientTest() throws IOException, TException {
    final int length = 10;
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
    BlockInfo block = getFirstBlockInfo(file);

    RemoteBlockReader client =
        RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
    ByteBuffer result =
        client.readRemoteBlock(
            new InetSocketAddress(
                block.getLocations().get(0).getWorkerAddress().getHost(),
                block.getLocations().get(0).getWorkerAddress().getDataPort()),
            block.getBlockId(),
            0,
            length);

    Assert.assertEquals(BufferUtils.getIncreasingByteBuffer(length), result);
  }
Ejemplo n.º 10
0
 @Override
 public void read(org.apache.thrift.protocol.TProtocol prot, BlockInfo struct)
     throws org.apache.thrift.TException {
   TTupleProtocol iprot = (TTupleProtocol) prot;
   BitSet incoming = iprot.readBitSet(3);
   if (incoming.get(0)) {
     struct.blockId = iprot.readI64();
     struct.setBlockIdIsSet(true);
   }
   if (incoming.get(1)) {
     struct.length = iprot.readI64();
     struct.setLengthIsSet(true);
   }
   if (incoming.get(2)) {
     {
       org.apache.thrift.protocol.TList _list5 =
           new org.apache.thrift.protocol.TList(
               org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
       struct.locations = new ArrayList<BlockLocation>(_list5.size);
       BlockLocation _elem6;
       for (int _i7 = 0; _i7 < _list5.size; ++_i7) {
         _elem6 = new BlockLocation();
         _elem6.read(iprot);
         struct.locations.add(_elem6);
       }
     }
     struct.setLocationsIsSet(true);
   }
 }
Ejemplo n.º 11
0
 @Override
 public void write(org.apache.thrift.protocol.TProtocol prot, BlockInfo struct)
     throws org.apache.thrift.TException {
   TTupleProtocol oprot = (TTupleProtocol) prot;
   BitSet optionals = new BitSet();
   if (struct.isSetBlockId()) {
     optionals.set(0);
   }
   if (struct.isSetLength()) {
     optionals.set(1);
   }
   if (struct.isSetLocations()) {
     optionals.set(2);
   }
   oprot.writeBitSet(optionals, 3);
   if (struct.isSetBlockId()) {
     oprot.writeI64(struct.blockId);
   }
   if (struct.isSetLength()) {
     oprot.writeI64(struct.length);
   }
   if (struct.isSetLocations()) {
     {
       oprot.writeI32(struct.locations.size());
       for (BlockLocation _iter4 : struct.locations) {
         _iter4.write(oprot);
       }
     }
   }
 }
Ejemplo n.º 12
0
  @Override
  public int compareTo(BlockInfo other) {
    if (!getClass().equals(other.getClass())) {
      return getClass().getName().compareTo(other.getClass().getName());
    }

    int lastComparison = 0;

    lastComparison = Boolean.valueOf(isSetBlockId()).compareTo(other.isSetBlockId());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (isSetBlockId()) {
      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.blockId, other.blockId);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    lastComparison = Boolean.valueOf(isSetLength()).compareTo(other.isSetLength());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (isSetLength()) {
      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.length, other.length);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    lastComparison = Boolean.valueOf(isSetLocations()).compareTo(other.isSetLocations());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (isSetLocations()) {
      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.locations, other.locations);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    return 0;
  }
Ejemplo n.º 13
0
    public void read(org.apache.thrift.protocol.TProtocol iprot, BlockInfo struct)
        throws org.apache.thrift.TException {
      org.apache.thrift.protocol.TField schemeField;
      iprot.readStructBegin();
      while (true) {
        schemeField = iprot.readFieldBegin();
        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
          break;
        }
        switch (schemeField.id) {
          case 1: // BLOCK_ID
            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
              struct.blockId = iprot.readI64();
              struct.setBlockIdIsSet(true);
            } else {
              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
            }
            break;
          case 2: // LENGTH
            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
              struct.length = iprot.readI64();
              struct.setLengthIsSet(true);
            } else {
              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
            }
            break;
          case 3: // LOCATIONS
            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
              {
                org.apache.thrift.protocol.TList _list0 = iprot.readListBegin();
                struct.locations = new ArrayList<BlockLocation>(_list0.size);
                BlockLocation _elem1;
                for (int _i2 = 0; _i2 < _list0.size; ++_i2) {
                  _elem1 = new BlockLocation();
                  _elem1.read(iprot);
                  struct.locations.add(_elem1);
                }
                iprot.readListEnd();
              }
              struct.setLocationsIsSet(true);
            } else {
              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
            }
            break;
          default:
            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
        }
        iprot.readFieldEnd();
      }
      iprot.readStructEnd();

      // check for required fields of primitive type, which can't be checked in the validate method
      struct.validate();
    }
  @Test
  public void readMultiFiles() throws IOException, TException {
    final int length = WORKER_CAPACITY_BYTES / 2 + 1;
    TachyonFile file1 =
        TachyonFSTestUtils.createByteFile(
            mTFS, "/readFile1", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
    BlockInfo block1 = getFirstBlockInfo(file1);
    DataServerMessage recvMsg1 = request(block1);
    assertValid(recvMsg1, length, block1.getBlockId(), 0, length);

    TachyonFile file2 =
        TachyonFSTestUtils.createByteFile(
            mTFS, "/readFile2", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
    BlockInfo block2 = getFirstBlockInfo(file2);
    DataServerMessage recvMsg2 = request(block2);
    assertValid(recvMsg2, length, block2.getBlockId(), 0, length);

    CommonUtils.sleepMs(
        mWorkerTachyonConf.getInt(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS) * 2 + 10);

    FileInfo fileInfo = mTFS.getInfo(mTFS.open(new TachyonURI("/readFile1")));
    Assert.assertEquals(0, fileInfo.inMemoryPercentage);
  }
  /** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
  @Test
  public void readTest6() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);

      long blockId = mTfs.getInfo(f).getBlockIds().get(0);
      BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
      RemoteBlockInStream is =
          new RemoteBlockInStream(
              info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
      byte[] ret = new byte[k / 2];
      int start = 0;
      while (start < k / 2) {
        int read = is.read(ret, 0, (k / 2) - start);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
        start += read;
      }
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }