/** Test <code>void read(byte[] b, int off, int len)</code>. Read from underfs. */
  @Test
  public void readTest3() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      byte[] ret = new byte[k / 2];
      Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k / 2, ret));
      is.close();
      if (k == 0) {
        Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      } else {
        Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      }

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      Assert.assertEquals(k, is.read(ret, 0, k));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      Assert.assertEquals(k, is.read(ret));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
Example #2
0
 @Test
 public void copyFromLocalTest() throws IOException, TException {
   File testDir = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir");
   testDir.mkdir();
   File testDirInner = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir/testDirInner");
   testDirInner.mkdir();
   File testFile =
       generateFileContent("/testDir/testFile", BufferUtils.getIncreasingByteArray(10));
   generateFileContent(
       "/testDir/testDirInner/testFile2", BufferUtils.getIncreasingByteArray(10, 20));
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"});
   Assert.assertEquals(
       getCommandOutput(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"}),
       mOutput.toString());
   TachyonFile file1 = mTfs.open(new TachyonURI("/testDir/testFile"));
   TachyonFile file2 = mTfs.open(new TachyonURI("/testDir/testDirInner/testFile2"));
   FileInfo fileInfo1 = mTfs.getInfo(file1);
   FileInfo fileInfo2 = mTfs.getInfo(file2);
   Assert.assertNotNull(fileInfo1);
   Assert.assertNotNull(fileInfo2);
   Assert.assertEquals(10, fileInfo1.length);
   Assert.assertEquals(20, fileInfo2.length);
   byte[] read = readContent(file1, 10);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
   read = readContent(file2, 20);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, 20, read));
 }
  /** Test <code>void read()</code>. Read from underfs. */
  @Test
  public void readTest1() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      byte[] ret = new byte[k];
      int value = is.read();
      int cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      if (k == 0) {
        Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      } else {
        Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      }

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      value = is.read();
      cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      value = is.read();
      cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
Example #4
0
  @Test
  public void copyFromLocalLargeTest() throws IOException, TException {
    File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
    testFile.createNewFile();
    FileOutputStream fos = new FileOutputStream(testFile);
    byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
    fos.write(toWrite);
    fos.close();
    mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
    Assert.assertEquals(
        getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
        mOutput.toString());
    TachyonFile tFile = mTfs.open(new TachyonURI("/testFile"));
    FileInfo fileInfo = mTfs.getInfo(tFile);
    Assert.assertNotNull(fileInfo);
    Assert.assertEquals(SIZE_BYTES, fileInfo.length);

    ClientOptions options =
        new ClientOptions.Builder(new TachyonConf())
            .setTachyonStoreType(TachyonStorageType.NO_STORE)
            .build();
    FileInStream tfis = mTfs.getInStream(tFile, options);
    byte[] read = new byte[SIZE_BYTES];
    tfis.read(read);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
  }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#read()}. */
  @Test
  public void readTest1() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
        FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);

        for (int i = 0; i < 2; i++) {
          FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
          byte[] ret = new byte[k];
          int value = is.read();
          int cnt = 0;
          while (value != -1) {
            Assert.assertTrue(value >= 0);
            Assert.assertTrue(value < 256);
            ret[cnt++] = (byte) value;
            value = is.read();
          }
          Assert.assertEquals(cnt, k);
          Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
          is.close();
        }
      }
    }
  }
  /** Test <code>void read()</code>. Read from remote data server. */
  @Test
  public void readTest4() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);

      long blockId = mTfs.getInfo(f).getBlockIds().get(0);
      BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
      RemoteBlockInStream is =
          new RemoteBlockInStream(
              info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
      byte[] ret = new byte[k];
      int value = is.read();
      int cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#read(byte[])}. */
  @Test
  public void readTest2() throws IOException, TachyonException {
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());

        FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        byte[] ret = new byte[k];
        Assert.assertEquals(k, is.read(ret));
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();

        is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        ret = new byte[k];
        Assert.assertEquals(k, is.read(ret));
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();
      }
    }
  }
  // Tests that deletes go through despite failing initially due to concurrent read
  @Test
  public void deleteWhileReadTest() throws Exception {
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test1",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    Assert.assertTrue(mTFS.getInfo(file).getInMemoryPercentage() == 100);
    // Open the file
    InStreamOptions options =
        new InStreamOptions.Builder(new TachyonConf())
            .setTachyonStorageType(TachyonStorageType.STORE)
            .build();
    FileInStream in = mTFS.getInStream(file, options);
    Assert.assertEquals(0, in.read());

    // Delete the file
    mTFS.delete(file);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the delete, the master should no longer serve the file
    Assert.assertNull(mTFS.open(new TachyonURI("/test1")));

    // However, the previous read should still be able to read it as the data still exists
    byte[] res = new byte[MEM_CAPACITY_BYTES];
    Assert.assertEquals(MEM_CAPACITY_BYTES - 1, in.read(res, 1, MEM_CAPACITY_BYTES - 1));
    res[0] = 0;
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MEM_CAPACITY_BYTES, res));
    in.close();

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the file is closed, the master's delete should go through and new files can be created
    TachyonFile newFile =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test2",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
    Assert.assertTrue(mTFS.getInfo(newFile).getInMemoryPercentage() == 100);
  }
Example #9
0
 private void copyToLocalWithBytes(int bytes) throws IOException {
   TachyonFSTestUtils.createByteFile(
       mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, bytes);
   mFsShell.copyToLocal(
       new String[] {
         "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
       });
   Assert.assertEquals(
       getCommandOutput(
           new String[] {
             "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
           }),
       mOutput.toString());
   File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
   FileInputStream fis = new FileInputStream(testFile);
   byte[] read = new byte[bytes];
   fis.read(read);
   fis.close();
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytes, read));
 }
Example #10
0
 @Test
 public void copyToLocalTest() throws IOException {
   TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.copyToLocal(
       new String[] {
         "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
       });
   Assert.assertEquals(
       getCommandOutput(
           new String[] {
             "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
           }),
       mOutput.toString());
   File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
   FileInputStream fis = new FileInputStream(testFile);
   byte[] read = new byte[10];
   fis.read(read);
   fis.close();
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
 }
Example #11
0
 @Test
 public void copyFromLocalTestWithFullURI() throws IOException {
   File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
   String tachyonURI =
       "tachyon://"
           + mLocalTachyonCluster.getMasterHostname()
           + ":"
           + mLocalTachyonCluster.getMasterPort()
           + "/destFileURI";
   // when
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   String cmdOut =
       getCommandOutput(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   // then
   Assert.assertThat(cmdOut, CoreMatchers.equalTo(mOutput.toString()));
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/destFileURI"));
   Assert.assertThat(tFile.length(), CoreMatchers.equalTo(10L));
   byte[] read = readContent(tFile, 10);
   Assert.assertThat(BufferUtils.equalIncreasingByteArray(10, read), CoreMatchers.equalTo(true));
 }
Example #12
0
 @Test
 public void copyFromLocalLargeTest() throws IOException {
   File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
   testFile.createNewFile();
   FileOutputStream fos = new FileOutputStream(testFile);
   byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
   fos.write(toWrite);
   fos.close();
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
   Assert.assertEquals(
       getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
       mOutput.toString());
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/testFile"));
   Assert.assertNotNull(tFile);
   Assert.assertEquals(SIZE_BYTES, tFile.length());
   InStream tfis = tFile.getInStream(ReadType.NO_CACHE);
   byte[] read = new byte[SIZE_BYTES];
   tfis.read(read);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
 }
Example #13
0
 @Test
 public void copyFromLocalTestWithFullURI() throws IOException, TException {
   File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
   String tachyonURI =
       "tachyon://"
           + mLocalTachyonCluster.getMasterHostname()
           + ":"
           + mLocalTachyonCluster.getMasterPort()
           + "/destFileURI";
   // when
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   String cmdOut =
       getCommandOutput(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   // then
   Assert.assertEquals(cmdOut, mOutput.toString());
   TachyonFile file = mTfs.open(new TachyonURI("/destFileURI"));
   FileInfo fileInfo = mTfs.getInfo(file);
   Assert.assertEquals(10L, fileInfo.length);
   byte[] read = readContent(file, 10);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
 }
  /** Tests creating and opening a store with a number of key. */
  @Test
  public void createAndOpenStoreWithMultiKeysTest() throws Exception {
    final int numKeys = 100;
    final int keyLength = 4; // 4Byte key
    final int valueLength = 5 * Constants.KB; // 5KB value
    mWriter = sKeyValueSystem.createStore(mStoreUri);
    for (int i = 0; i < numKeys; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
      mWriter.put(key, value);
    }
    mWriter.close();

    mReader = sKeyValueSystem.openStore(mStoreUri);
    for (int i = 0; i < numKeys; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = mReader.get(key);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
    }
    Assert.assertNull(mReader.get(KEY1));
    Assert.assertNull(mReader.get(KEY2));
    mReader.close();
  }
  /** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
  @Test
  public void readTest6() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);

      long blockId = mTfs.getInfo(f).getBlockIds().get(0);
      BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
      RemoteBlockInStream is =
          new RemoteBlockInStream(
              info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
      byte[] ret = new byte[k / 2];
      int start = 0;
      while (start < k / 2) {
        int read = is.read(ret, 0, (k / 2) - start);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
        start += read;
      }
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
  /**
   * Tests creating and opening a store with a number of keys, while each key-value pair is large
   * enough to take a separate key-value partition.
   */
  @Test
  public void createMultiPartitionsTest() throws Exception {
    // TODO(cc): Remove codes using createStoreOfMultiplePartitions.
    final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
    final int numKeys = 10;
    final int keyLength = 4; // 4Byte key
    final int valueLength = 500 * Constants.KB; // 500KB value

    FileSystem fs = FileSystem.Factory.get();

    ClientContext.getConf()
        .set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
    mWriter = sKeyValueSystem.createStore(mStoreUri);
    for (int i = 0; i < numKeys; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
      mWriter.put(key, value);
    }
    mWriter.close();

    List<URIStatus> files = fs.listStatus(mStoreUri);
    Assert.assertEquals(numKeys, files.size());
    for (URIStatus info : files) {
      Assert.assertTrue(info.getLength() <= maxPartitionSize);
    }

    mReader = sKeyValueSystem.openStore(mStoreUri);
    for (int i = 0; i < numKeys; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = mReader.get(key);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
    }
    Assert.assertNull(mReader.get(KEY1));
    Assert.assertNull(mReader.get(KEY2));
    mReader.close();
  }