Exemple #1
0
  @Test
  public void copyFromLocalLargeTest() throws IOException, TException {
    File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
    testFile.createNewFile();
    FileOutputStream fos = new FileOutputStream(testFile);
    byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
    fos.write(toWrite);
    fos.close();
    mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
    Assert.assertEquals(
        getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
        mOutput.toString());
    TachyonFile tFile = mTfs.open(new TachyonURI("/testFile"));
    FileInfo fileInfo = mTfs.getInfo(tFile);
    Assert.assertNotNull(fileInfo);
    Assert.assertEquals(SIZE_BYTES, fileInfo.length);

    ClientOptions options =
        new ClientOptions.Builder(new TachyonConf())
            .setTachyonStoreType(TachyonStorageType.NO_STORE)
            .build();
    FileInStream tfis = mTfs.getInStream(tFile, options);
    byte[] read = new byte[SIZE_BYTES];
    tfis.read(read);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
  }
  /**
   * Creates a store with the specified number of partitions.
   *
   * <p>NOTE: calling this method will set {@link Constants#KEY_VALUE_PARTITION_SIZE_BYTES_MAX} to
   * {@link Constants#MB}.
   *
   * @param partitionNumber the number of partitions
   * @param keyValuePairs the key-value pairs in the store, null if you don't want to know them
   * @return the URI to the created store
   */
  private TachyonURI createStoreOfMultiplePartitions(
      int partitionNumber, List<KeyValuePair> keyValuePairs) throws Exception {
    // These sizes are carefully selected, one partition holds only one key-value pair.
    final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
    ClientContext.getConf()
        .set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
    final int keyLength = 4; // 4Byte key
    final int valueLength = 500 * Constants.KB; // 500KB value

    TachyonURI storeUri = new TachyonURI(PathUtils.uniqPath());
    mWriter = sKeyValueSystem.createStore(storeUri);
    for (int i = 0; i < partitionNumber; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
      mWriter.put(key, value);
      if (keyValuePairs != null) {
        keyValuePairs.add(new KeyValuePair(key, value));
      }
    }
    mWriter.close();

    Assert.assertEquals(partitionNumber, getPartitionNumber(storeUri));

    return storeUri;
  }
  /** Test <code>void read(byte[] b, int off, int len)</code>. Read from underfs. */
  @Test
  public void readTest3() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      byte[] ret = new byte[k / 2];
      Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k / 2, ret));
      is.close();
      if (k == 0) {
        Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      } else {
        Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      }

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      Assert.assertEquals(k, is.read(ret, 0, k));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      Assert.assertEquals(k, is.read(ret));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
Exemple #4
0
 @Test
 public void copyFromLocalTest() throws IOException, TException {
   File testDir = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir");
   testDir.mkdir();
   File testDirInner = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir/testDirInner");
   testDirInner.mkdir();
   File testFile =
       generateFileContent("/testDir/testFile", BufferUtils.getIncreasingByteArray(10));
   generateFileContent(
       "/testDir/testDirInner/testFile2", BufferUtils.getIncreasingByteArray(10, 20));
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"});
   Assert.assertEquals(
       getCommandOutput(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"}),
       mOutput.toString());
   TachyonFile file1 = mTfs.open(new TachyonURI("/testDir/testFile"));
   TachyonFile file2 = mTfs.open(new TachyonURI("/testDir/testDirInner/testFile2"));
   FileInfo fileInfo1 = mTfs.getInfo(file1);
   FileInfo fileInfo2 = mTfs.getInfo(file2);
   Assert.assertNotNull(fileInfo1);
   Assert.assertNotNull(fileInfo2);
   Assert.assertEquals(10, fileInfo1.length);
   Assert.assertEquals(20, fileInfo2.length);
   byte[] read = readContent(file1, 10);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
   read = readContent(file2, 20);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, 20, read));
 }
  /** Test <code>void read()</code>. Read from underfs. */
  @Test
  public void readTest1() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      byte[] ret = new byte[k];
      int value = is.read();
      int cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      if (k == 0) {
        Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      } else {
        Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      }

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      value = is.read();
      cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      value = is.read();
      cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
 @Override
 public int directRead(byte[] b, int off, int len) throws IOException {
   ByteBuffer buf = mLocalFileChannel.map(FileChannel.MapMode.READ_ONLY, getPosition(), len);
   buf.get(b, off, len);
   BufferUtils.cleanDirectBuffer(buf);
   return len;
 }
  /**
   * Tests putting a key-value pair that is larger than the max key-value partition size, expecting
   * exception thrown.
   */
  @Test
  public void putKeyValueTooLargeTest() throws Exception {
    final long maxPartitionSize = 500 * Constants.KB; // Each partition is at most 500 KB
    final int keyLength = 4; // 4Byte key
    final int valueLength = 500 * Constants.KB; // 500KB value

    ClientContext.getConf()
        .set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
    mWriter = sKeyValueSystem.createStore(mStoreUri);
    byte[] key = BufferUtils.getIncreasingByteArray(0, keyLength);
    byte[] value = BufferUtils.getIncreasingByteArray(0, valueLength);

    mThrown.expect(IOException.class);
    mThrown.expectMessage(ExceptionMessage.KEY_VALUE_TOO_LARGE.getMessage(keyLength, valueLength));
    mWriter.put(key, value);
  }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#read()}. */
  @Test
  public void readTest1() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
        FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);

        for (int i = 0; i < 2; i++) {
          FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
          byte[] ret = new byte[k];
          int value = is.read();
          int cnt = 0;
          while (value != -1) {
            Assert.assertTrue(value >= 0);
            Assert.assertTrue(value < 256);
            ret[cnt++] = (byte) value;
            value = is.read();
          }
          Assert.assertEquals(cnt, k);
          Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
          is.close();
        }
      }
    }
  }
Exemple #9
0
 @Test
 public void catTest() throws IOException {
   TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.cat(new String[] {"cat", "/testFile"});
   byte[] expect = BufferUtils.getIncreasingByteArray(10);
   Assert.assertArrayEquals(expect, mOutput.toByteArray());
 }
  /** Test <code>void read()</code>. Read from remote data server. */
  @Test
  public void readTest4() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);

      long blockId = mTfs.getInfo(f).getBlockIds().get(0);
      BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
      RemoteBlockInStream is =
          new RemoteBlockInStream(
              info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
      byte[] ret = new byte[k];
      int value = is.read();
      int cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
 @Override
 protected void bufferedRead(int len) throws IOException {
   if (mBuffer.isDirect()) { // Buffer may not be direct on initialization
     BufferUtils.cleanDirectBuffer(mBuffer);
   }
   mBuffer = mLocalFileChannel.map(FileChannel.MapMode.READ_ONLY, getPosition(), len);
 }
 /** Asserts that the message back matches the block response protocols. */
 private void assertValid(
     final DataServerMessage msg,
     final int expectedSize,
     final long blockId,
     final long offset,
     final long length) {
   assertValid(msg, BufferUtils.getIncreasingByteBuffer(expectedSize), blockId, offset, length);
 }
Exemple #13
0
 @Test
 public void catTest() throws IOException {
   TachyonFSTestUtils.createByteFile(
       mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
   mFsShell.run(new String[] {"cat", "/testFile"});
   byte[] expect = BufferUtils.getIncreasingByteArray(10);
   Assert.assertArrayEquals(expect, mOutput.toByteArray());
 }
 @Override
 public void write(int b) throws IOException {
   checkIfClosed();
   Preconditions.checkState(mWrittenBytes + 1 <= mBlockSize, PreconditionMessage.ERR_END_OF_BLOCK);
   if (mBuffer.position() >= mBuffer.limit()) {
     flush();
   }
   BufferUtils.putIntByteBuffer(mBuffer, b);
   mWrittenBytes++;
 }
Exemple #15
0
 @Test
 public void copyFromLocalTestWithFullURI() throws IOException {
   File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
   String tachyonURI =
       "tachyon://"
           + mLocalTachyonCluster.getMasterHostname()
           + ":"
           + mLocalTachyonCluster.getMasterPort()
           + "/destFileURI";
   // when
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   String cmdOut =
       getCommandOutput(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   // then
   Assert.assertThat(cmdOut, CoreMatchers.equalTo(mOutput.toString()));
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/destFileURI"));
   Assert.assertThat(tFile.length(), CoreMatchers.equalTo(10L));
   byte[] read = readContent(tFile, 10);
   Assert.assertThat(BufferUtils.equalIncreasingByteArray(10, read), CoreMatchers.equalTo(true));
 }
Exemple #16
0
 @Test
 public void copyFromLocalLargeTest() throws IOException {
   File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
   testFile.createNewFile();
   FileOutputStream fos = new FileOutputStream(testFile);
   byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
   fos.write(toWrite);
   fos.close();
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
   Assert.assertEquals(
       getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
       mOutput.toString());
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/testFile"));
   Assert.assertNotNull(tFile);
   Assert.assertEquals(SIZE_BYTES, tFile.length());
   InStream tfis = tFile.getInStream(ReadType.NO_CACHE);
   byte[] read = new byte[SIZE_BYTES];
   tfis.read(read);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
 }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#read(byte[])}. */
  @Test
  public void readTest2() throws IOException, TachyonException {
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());

        FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        byte[] ret = new byte[k];
        Assert.assertEquals(k, is.read(ret));
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();

        is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        ret = new byte[k];
        Assert.assertEquals(k, is.read(ret));
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();
      }
    }
  }
Exemple #18
0
 @Test
 public void copyFromLocalTestWithFullURI() throws IOException, TException {
   File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
   String tachyonURI =
       "tachyon://"
           + mLocalTachyonCluster.getMasterHostname()
           + ":"
           + mLocalTachyonCluster.getMasterPort()
           + "/destFileURI";
   // when
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   String cmdOut =
       getCommandOutput(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   // then
   Assert.assertEquals(cmdOut, mOutput.toString());
   TachyonFile file = mTfs.open(new TachyonURI("/destFileURI"));
   FileInfo fileInfo = mTfs.getInfo(file);
   Assert.assertEquals(10L, fileInfo.length);
   byte[] read = readContent(file, 10);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
 }
 @Override
 public int read() throws IOException {
   checkIfClosed();
   if (remaining() == 0) {
     close();
     return -1;
   }
   if (!mBufferIsValid || mBuffer.remaining() == 0) {
     updateBuffer();
   }
   mPos++;
   return BufferUtils.byteToInt(mBuffer.get());
 }
  /** Tests creating and opening a store with a number of key. */
  @Test
  public void createAndOpenStoreWithMultiKeysTest() throws Exception {
    final int numKeys = 100;
    final int keyLength = 4; // 4Byte key
    final int valueLength = 5 * Constants.KB; // 5KB value
    mWriter = sKeyValueSystem.createStore(mStoreUri);
    for (int i = 0; i < numKeys; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
      mWriter.put(key, value);
    }
    mWriter.close();

    mReader = sKeyValueSystem.openStore(mStoreUri);
    for (int i = 0; i < numKeys; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = mReader.get(key);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
    }
    Assert.assertNull(mReader.get(KEY1));
    Assert.assertNull(mReader.get(KEY2));
    mReader.close();
  }
  // Tests that deletes go through despite failing initially due to concurrent read
  @Test
  public void deleteWhileReadTest() throws Exception {
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test1",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    Assert.assertTrue(mTFS.getInfo(file).getInMemoryPercentage() == 100);
    // Open the file
    InStreamOptions options =
        new InStreamOptions.Builder(new TachyonConf())
            .setTachyonStorageType(TachyonStorageType.STORE)
            .build();
    FileInStream in = mTFS.getInStream(file, options);
    Assert.assertEquals(0, in.read());

    // Delete the file
    mTFS.delete(file);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the delete, the master should no longer serve the file
    Assert.assertNull(mTFS.open(new TachyonURI("/test1")));

    // However, the previous read should still be able to read it as the data still exists
    byte[] res = new byte[MEM_CAPACITY_BYTES];
    Assert.assertEquals(MEM_CAPACITY_BYTES - 1, in.read(res, 1, MEM_CAPACITY_BYTES - 1));
    res[0] = 0;
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MEM_CAPACITY_BYTES, res));
    in.close();

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the file is closed, the master's delete should go through and new files can be created
    TachyonFile newFile =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test2",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
    Assert.assertTrue(mTFS.getInfo(newFile).getInMemoryPercentage() == 100);
  }
  /**
   * Tests creating and opening a store with a number of keys, while each key-value pair is large
   * enough to take a separate key-value partition.
   */
  @Test
  public void createMultiPartitionsTest() throws Exception {
    // TODO(cc): Remove codes using createStoreOfMultiplePartitions.
    final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
    final int numKeys = 10;
    final int keyLength = 4; // 4Byte key
    final int valueLength = 500 * Constants.KB; // 500KB value

    FileSystem fs = FileSystem.Factory.get();

    ClientContext.getConf()
        .set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
    mWriter = sKeyValueSystem.createStore(mStoreUri);
    for (int i = 0; i < numKeys; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
      mWriter.put(key, value);
    }
    mWriter.close();

    List<URIStatus> files = fs.listStatus(mStoreUri);
    Assert.assertEquals(numKeys, files.size());
    for (URIStatus info : files) {
      Assert.assertTrue(info.getLength() <= maxPartitionSize);
    }

    mReader = sKeyValueSystem.openStore(mStoreUri);
    for (int i = 0; i < numKeys; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = mReader.get(key);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
    }
    Assert.assertNull(mReader.get(KEY1));
    Assert.assertNull(mReader.get(KEY2));
    mReader.close();
  }
 @Test
 public void readPartialTest2() throws TException, IOException {
   TachyonFile file =
       TachyonFSTestUtils.createByteFile(
           mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
   BlockInfo block = getFirstBlockInfo(file);
   final int offset = 2;
   final int length = 6;
   DataServerMessage recvMsg = request(block, offset, length);
   assertValid(
       recvMsg,
       BufferUtils.getIncreasingByteBuffer(offset, length),
       block.getBlockId(),
       offset,
       length);
 }
Exemple #24
0
 private void copyToLocalWithBytes(int bytes) throws IOException {
   TachyonFSTestUtils.createByteFile(
       mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, bytes);
   mFsShell.copyToLocal(
       new String[] {
         "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
       });
   Assert.assertEquals(
       getCommandOutput(
           new String[] {
             "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
           }),
       mOutput.toString());
   File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
   FileInputStream fis = new FileInputStream(testFile);
   byte[] read = new byte[bytes];
   fis.read(read);
   fis.close();
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytes, read));
 }
Exemple #25
0
 @Test
 public void copyToLocalTest() throws IOException {
   TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.copyToLocal(
       new String[] {
         "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
       });
   Assert.assertEquals(
       getCommandOutput(
           new String[] {
             "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
           }),
       mOutput.toString());
   File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
   FileInputStream fis = new FileInputStream(testFile);
   byte[] read = new byte[10];
   fis.read(read);
   fis.close();
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
 }
  @Test
  public void readThroughClientTest() throws IOException, TException {
    final int length = 10;
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
    BlockInfo block = getFirstBlockInfo(file);

    RemoteBlockReader client =
        RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
    ByteBuffer result =
        client.readRemoteBlock(
            new InetSocketAddress(
                block.getLocations().get(0).getWorkerAddress().getHost(),
                block.getLocations().get(0).getWorkerAddress().getDataPort()),
            block.getBlockId(),
            0,
            length);

    Assert.assertEquals(BufferUtils.getIncreasingByteBuffer(length), result);
  }
  /** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
  @Test
  public void readTest6() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);

      long blockId = mTfs.getInfo(f).getBlockIds().get(0);
      BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
      RemoteBlockInStream is =
          new RemoteBlockInStream(
              info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
      byte[] ret = new byte[k / 2];
      int start = 0;
      while (start < k / 2) {
        int read = is.read(ret, 0, (k / 2) - start);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
        start += read;
      }
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
  @Override
  public void close() throws IOException {
    if (mClosed) {
      return;
    }
    try {
      if (mBlockIsRead) {
        mWorkerClient.accessBlock(mBlockId);
        ClientContext.getClientMetrics().incBlocksReadLocal(1);
      }
      mWorkerClient.unlockBlock(mBlockId);
    } catch (TachyonException e) {
      throw new IOException(e);
    } finally {
      mContext.releaseWorkerClient(mWorkerClient);
      mCloser.close();
      if (mBuffer != null && mBuffer.isDirect()) {
        BufferUtils.cleanDirectBuffer(mBuffer);
      }
    }

    mClosed = true;
  }