Esempio n. 1
0
  @Test
  public void duTest() throws IOException {
    TachyonFSTestUtils.createByteFile(
        mTfs, "/testRoot/testFileA", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
    TachyonFSTestUtils.createByteFile(
        mTfs,
        "/testRoot/testDir/testFileB",
        TachyonStorageType.STORE,
        UnderStorageType.NO_PERSIST,
        20);
    TachyonFSTestUtils.createByteFile(
        mTfs,
        "/testRoot/testDir/testDir/testFileC",
        TachyonStorageType.STORE,
        UnderStorageType.NO_PERSIST,
        30);

    String expected = "";
    // du a non-existing file
    mFsShell.run(new String[] {"du", "/testRoot/noneExisting"});
    expected += "Could not find path: /testRoot/noneExisting\n";
    // du a file
    mFsShell.run(new String[] {"du", "/testRoot/testFileA"});
    expected += "/testRoot/testFileA is 10 bytes\n";
    // du a folder
    mFsShell.run(new String[] {"du", "/testRoot/testDir"});
    expected += "/testRoot/testDir is 50 bytes\n";

    Assert.assertEquals(expected, mOutput.toString());
  }
Esempio n. 2
0
  /*
   * @Test public void locationTest() throws IOException { TachyonFile file =
   * TachyonFSTestUtils.createByteFile(mTfs, "/testFile", CacheType.STORE,
   * UnderStorageType.NO_PERSIST, 10); mFsShell.run(new String[] {"location", "/testFile"});
   *
   * FileInfo fileInfo = mTfs.getInfo(file); Assert.assertNotNull(fileInfo); List<String>
   * locationsList = tFile.getLocationHosts(); String[] commandParameters = new String[3 +
   * locationsList.size()]; commandParameters[0] = "location"; commandParameters[1] = "/testFile";
   * commandParameters[2] = String.valueOf(file.getFileId()); Iterator<String> iter =
   * locationsList.iterator(); int i = 3; while (iter.hasNext()) { commandParameters[i ++] =
   * iter.next(); } Assert.assertEquals(getCommandOutput(commandParameters), mOutput.toString()); }
   */
  @Test
  public void lsrTest() throws IOException, TException {
    FileInfo[] files = new FileInfo[4];

    TachyonFile fileA =
        TachyonFSTestUtils.createByteFile(
            mTfs, "/testRoot/testFileA", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
    files[0] = mTfs.getInfo(fileA);
    TachyonFSTestUtils.createByteFile(
        mTfs,
        "/testRoot/testDir/testFileB",
        TachyonStorageType.STORE,
        UnderStorageType.NO_PERSIST,
        20);
    files[1] = mTfs.getInfo(mTfs.open(new TachyonURI("/testRoot/testDir")));
    files[2] = mTfs.getInfo(mTfs.open(new TachyonURI("/testRoot/testDir/testFileB")));
    TachyonFile fileC =
        TachyonFSTestUtils.createByteFile(
            mTfs, "/testRoot/testFileC", TachyonStorageType.NO_STORE, UnderStorageType.PERSIST, 30);
    files[3] = mTfs.getInfo(fileC);
    mFsShell.run(new String[] {"lsr", "/testRoot"});
    String expected = "";
    String format = "%-10s%-25s%-15s%-5s\n";
    expected +=
        String.format(
            format,
            FormatUtils.getSizeFromBytes(10),
            TFsShell.convertMsToDate(files[0].getCreationTimeMs()),
            "In Memory",
            "/testRoot/testFileA");
    expected +=
        String.format(
            format,
            FormatUtils.getSizeFromBytes(0),
            TFsShell.convertMsToDate(files[1].getCreationTimeMs()),
            "",
            "/testRoot/testDir");
    expected +=
        String.format(
            format,
            FormatUtils.getSizeFromBytes(20),
            TFsShell.convertMsToDate(files[2].getCreationTimeMs()),
            "In Memory",
            "/testRoot/testDir/testFileB");
    expected +=
        String.format(
            format,
            FormatUtils.getSizeFromBytes(30),
            TFsShell.convertMsToDate(files[3].getCreationTimeMs()),
            "Not In Memory",
            "/testRoot/testFileC");
    Assert.assertEquals(expected, mOutput.toString());
  }
Esempio n. 3
0
 @Test
 public void countTest() throws IOException {
   TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileA", WriteType.MUST_CACHE, 10);
   TachyonFSTestUtils.createByteFile(
       mTfs, "/testRoot/testDir/testFileB", WriteType.MUST_CACHE, 20);
   TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileB", WriteType.MUST_CACHE, 30);
   mFsShell.count(new String[] {"count", "/testRoot"});
   String expected = "";
   String format = "%-25s%-25s%-15s\n";
   expected += String.format(format, "File Count", "Folder Count", "Total Bytes");
   expected += String.format(format, 3, 2, 60);
   Assert.assertEquals(expected, mOutput.toString());
 }
  // Tests that deletes go through despite failing initially due to concurrent read
  @Test
  public void deleteWhileReadTest() throws Exception {
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test1",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    Assert.assertTrue(mTFS.getInfo(file).getInMemoryPercentage() == 100);
    // Open the file
    InStreamOptions options =
        new InStreamOptions.Builder(new TachyonConf())
            .setTachyonStorageType(TachyonStorageType.STORE)
            .build();
    FileInStream in = mTFS.getInStream(file, options);
    Assert.assertEquals(0, in.read());

    // Delete the file
    mTFS.delete(file);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the delete, the master should no longer serve the file
    Assert.assertNull(mTFS.open(new TachyonURI("/test1")));

    // However, the previous read should still be able to read it as the data still exists
    byte[] res = new byte[MEM_CAPACITY_BYTES];
    Assert.assertEquals(MEM_CAPACITY_BYTES - 1, in.read(res, 1, MEM_CAPACITY_BYTES - 1));
    res[0] = 0;
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MEM_CAPACITY_BYTES, res));
    in.close();

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the file is closed, the master's delete should go through and new files can be created
    TachyonFile newFile =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test2",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
    Assert.assertTrue(mTFS.getInfo(newFile).getInMemoryPercentage() == 100);
  }
Esempio n. 5
0
 @Test
 public void catTest() throws IOException {
   TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.cat(new String[] {"cat", "/testFile"});
   byte[] expect = BufferUtils.getIncreasingByteArray(10);
   Assert.assertArrayEquals(expect, mOutput.toByteArray());
 }
  // TODO(calvin): Make this work with the new BlockReader.
  // @Test
  public void readThroughClientNonExistentTest() throws IOException, TException {
    final int length = 10;
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
    BlockInfo block = getFirstBlockInfo(file);

    // Get the maximum block id, for use in determining a non-existent block id.
    FileInfo fileInfo = mTFS.getInfo(file);
    long maxBlockId = block.getBlockId();
    for (long blockId : fileInfo.blockIds) {
      if (blockId > maxBlockId) {
        maxBlockId = blockId;
      }
    }

    RemoteBlockReader client =
        RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
    ByteBuffer result =
        client.readRemoteBlock(
            new InetSocketAddress(
                block.getLocations().get(0).getWorkerAddress().getHost(),
                block.getLocations().get(0).getWorkerAddress().getDataPort()),
            maxBlockId + 1,
            0,
            length);

    Assert.assertNull(result);
  }
Esempio n. 7
0
 @Test
 public void tailEmptyFileTest() throws IOException {
   TachyonFSTestUtils.createByteFile(
       mTfs, "/emptyFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 0);
   int ret = mFsShell.run(new String[] {"tail", "/emptyFile"});
   Assert.assertEquals(0, ret);
 }
Esempio n. 8
0
 @Test
 public void freeTest() throws IOException {
   TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.free(new String[] {"free", "/testFile"});
   TachyonConf tachyonConf = mLocalTachyonCluster.getMasterTachyonConf();
   CommonUtils.sleepMs(null, CommonUtils.getToMasterHeartBeatIntervalMs(tachyonConf) * 2 + 10);
   Assert.assertFalse(mTfs.getFile(new TachyonURI("/testFile")).isInMemory());
 }
Esempio n. 9
0
 @Test
 public void catTest() throws IOException {
   TachyonFSTestUtils.createByteFile(
       mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
   mFsShell.run(new String[] {"cat", "/testFile"});
   byte[] expect = BufferUtils.getIncreasingByteArray(10);
   Assert.assertArrayEquals(expect, mOutput.toByteArray());
 }
 @Test
 public void tooLargeOffset() throws IOException, TException {
   final int length = 10;
   TachyonFile file =
       TachyonFSTestUtils.createByteFile(
           mTFS, "/readTooLarge", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
   BlockInfo block = getFirstBlockInfo(file);
   DataServerMessage recvMsg = request(block, length * 2, 1);
   assertError(recvMsg, block.blockId);
 }
Esempio n. 11
0
 @Test
 public void loadDirTest() throws IOException, TException {
   TachyonFile fileA =
       TachyonFSTestUtils.createByteFile(
           mTfs, "/testRoot/testFileA", TachyonStorageType.NO_STORE, UnderStorageType.PERSIST, 10);
   TachyonFile fileB =
       TachyonFSTestUtils.createByteFile(
           mTfs, "/testRoot/testFileB", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
   FileInfo fileInfoA = mTfs.getInfo(fileA);
   FileInfo fileInfoB = mTfs.getInfo(fileB);
   Assert.assertFalse(fileInfoA.getInMemoryPercentage() == 100);
   Assert.assertTrue(fileInfoB.getInMemoryPercentage() == 100);
   // Testing loading of a directory
   mFsShell.run(new String[] {"load", "/testRoot"});
   fileInfoA = mTfs.getInfo(fileA);
   fileInfoB = mTfs.getInfo(fileB);
   Assert.assertTrue(fileInfoA.getInMemoryPercentage() == 100);
   Assert.assertTrue(fileInfoB.getInMemoryPercentage() == 100);
 }
Esempio n. 12
0
 @Test
 public void freeTest() throws IOException, TException {
   TachyonFile file =
       TachyonFSTestUtils.createByteFile(
           mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
   mFsShell.run(new String[] {"free", "/testFile"});
   TachyonConf tachyonConf = mLocalTachyonCluster.getMasterTachyonConf();
   CommonUtils.sleepMs(tachyonConf.getInt(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS));
   Assert.assertFalse(mTfs.getInfo(file).getInMemoryPercentage() == 100);
 }
 @Test
 public void readTest() throws IOException, TException {
   final int length = 10;
   TachyonFile file =
       TachyonFSTestUtils.createByteFile(
           mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
   BlockInfo block = getFirstBlockInfo(file);
   DataServerMessage recvMsg = request(block);
   assertValid(recvMsg, length, block.getBlockId(), 0, length);
 }
 @Test
 public void readPartialTest1() throws TException, IOException {
   TachyonFile file =
       TachyonFSTestUtils.createByteFile(
           mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
   BlockInfo block = getFirstBlockInfo(file);
   final int offset = 0;
   final int length = 6;
   DataServerMessage recvMsg = request(block, offset, length);
   assertValid(recvMsg, length, block.getBlockId(), offset, length);
 }
  @BeforeClass
  public static void beforeClass() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.tachyon.impl", TFS.class.getName());

    TachyonFileSystem tachyonFS = sLocalTachyonClusterResource.get().getClient();
    TachyonFSTestUtils.createByteFile(
        tachyonFS, "/testFile1", TachyonStorageType.STORE, UnderStorageType.SYNC_PERSIST, FILE_LEN);

    URI uri = URI.create(sLocalTachyonClusterResource.get().getMasterUri());
    sTFS = FileSystem.get(uri, conf);
  }
Esempio n. 16
0
 @Test
 public void loadFileTest() throws IOException, TException {
   TachyonFile file =
       TachyonFSTestUtils.createByteFile(
           mTfs, "/testFile", TachyonStorageType.NO_STORE, UnderStorageType.PERSIST, 10);
   FileInfo fileInfo = mTfs.getInfo(file);
   Assert.assertFalse(fileInfo.getInMemoryPercentage() == 100);
   // Testing loading of a single file
   mFsShell.run(new String[] {"load", "/testFile"});
   fileInfo = mTfs.getInfo(file);
   Assert.assertTrue(fileInfo.getInMemoryPercentage() == 100);
 }
Esempio n. 17
0
  @Test
  public void duTest() throws IOException {
    TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileA", WriteType.MUST_CACHE, 10);
    TachyonFSTestUtils.createByteFile(
        mTfs, "/testRoot/testDir/testFileB", WriteType.MUST_CACHE, 20);
    TachyonFSTestUtils.createByteFile(
        mTfs, "/testRoot/testDir/testDir/testFileC", WriteType.MUST_CACHE, 30);

    String expected = "";
    // du a non-existing file
    mFsShell.du(new String[] {"du", "/testRoot/noneExisting"});
    expected += "/testRoot/noneExisting does not exist\n";
    // du a file
    mFsShell.du(new String[] {"du", "/testRoot/testFileA"});
    expected += "/testRoot/testFileA is 10 bytes\n";
    // du a folder
    mFsShell.du(new String[] {"du", "/testRoot/testDir"});
    expected += "/testRoot/testDir is 50 bytes\n";

    Assert.assertEquals(expected, mOutput.toString());
  }
  // Tests that pinning a file and then unpinning
  @Test
  public void unpinFileTest() throws Exception {
    // Create a file that fills the entire Tachyon store
    TachyonFile file1 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test1",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    // Pin the file
    mTFS.setState(file1, mSetPinned);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // Confirm the pin with master
    Assert.assertTrue(mTFS.getInfo(file1).isIsPinned());

    // Unpin the file
    mTFS.setState(file1, mSetUnpinned);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // Confirm the unpin
    Assert.assertFalse(mTFS.getInfo(file1).isIsPinned());

    // Try to create a file that cannot be stored unless the previous file is evicted, this
    // should succeed
    TachyonFile file2 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test2",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    // File 2 should be in memory and File 1 should be evicted
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
    Assert.assertFalse(mTFS.getInfo(file1).getInMemoryPercentage() == 100);
    Assert.assertTrue(mTFS.getInfo(file2).getInMemoryPercentage() == 100);
  }
Esempio n. 19
0
 @Test
 public void lsrTest() throws IOException {
   int fileIdA =
       TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileA", WriteType.MUST_CACHE, 10);
   TachyonFile[] files = new TachyonFile[4];
   files[0] = mTfs.getFile(fileIdA);
   TachyonFSTestUtils.createByteFile(
       mTfs, "/testRoot/testDir/testFileB", WriteType.MUST_CACHE, 20);
   files[1] = mTfs.getFile(new TachyonURI("/testRoot/testDir"));
   files[2] = mTfs.getFile(new TachyonURI("/testRoot/testDir/testFileB"));
   int fileIdC =
       TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileC", WriteType.THROUGH, 30);
   files[3] = mTfs.getFile(fileIdC);
   mFsShell.ls(new String[] {"count", "/testRoot"});
   String expected = "";
   String format = "%-10s%-25s%-15s%-5s\n";
   expected +=
       String.format(
           format,
           FormatUtils.getSizeFromBytes(10),
           TFsShell.convertMsToDate(files[0].getCreationTimeMs()),
           "In Memory",
           "/testRoot/testFileA");
   expected +=
       String.format(
           format,
           FormatUtils.getSizeFromBytes(0),
           TFsShell.convertMsToDate(files[1].getCreationTimeMs()),
           "",
           "/testRoot/testDir");
   expected +=
       String.format(
           format,
           FormatUtils.getSizeFromBytes(30),
           TFsShell.convertMsToDate(files[3].getCreationTimeMs()),
           "Not In Memory",
           "/testRoot/testFileC");
   Assert.assertEquals(expected, mOutput.toString());
 }
  @Test
  public void readMultiFiles() throws IOException, TException {
    final int length = WORKER_CAPACITY_BYTES / 2 + 1;
    TachyonFile file1 =
        TachyonFSTestUtils.createByteFile(
            mTFS, "/readFile1", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
    BlockInfo block1 = getFirstBlockInfo(file1);
    DataServerMessage recvMsg1 = request(block1);
    assertValid(recvMsg1, length, block1.getBlockId(), 0, length);

    TachyonFile file2 =
        TachyonFSTestUtils.createByteFile(
            mTFS, "/readFile2", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
    BlockInfo block2 = getFirstBlockInfo(file2);
    DataServerMessage recvMsg2 = request(block2);
    assertValid(recvMsg2, length, block2.getBlockId(), 0, length);

    CommonUtils.sleepMs(
        mWorkerTachyonConf.getInt(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS) * 2 + 10);

    FileInfo fileInfo = mTFS.getInfo(mTFS.open(new TachyonURI("/readFile1")));
    Assert.assertEquals(0, fileInfo.inMemoryPercentage);
  }
  // Tests that pinning a file prevents it from being evicted.
  @Test
  public void pinFileTest() throws Exception {
    // Create a file that fills the entire Tachyon store
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test1",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    // Pin the file
    mTFS.setState(file, mSetPinned);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // Confirm the pin with master
    Assert.assertTrue(mTFS.getInfo(file).isIsPinned());

    // Try to create a file that cannot be stored unless the previous file is evicted, expect an
    // exception since worker cannot serve the request
    mThrown.expect(IOException.class);
    TachyonFSTestUtils.createByteFile(
        mTFS, "/test2", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, MEM_CAPACITY_BYTES);
  }
Esempio n. 22
0
 @Test
 public void fileinfoTest() throws IOException {
   int fileId = TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.fileinfo(new String[] {"fileinfo", "/testFile"});
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/testFile"));
   Assert.assertNotNull(tFile);
   List<ClientBlockInfo> blocks = mTfs.getFileBlocks(fileId);
   String[] commandParameters = new String[3 + blocks.size()];
   commandParameters[0] = "fileinfo";
   commandParameters[1] = "/testFile";
   commandParameters[2] = String.valueOf(fileId);
   Iterator<ClientBlockInfo> iter = blocks.iterator();
   int i = 3;
   while (iter.hasNext()) {
     commandParameters[i++] = iter.next().toString();
   }
   Assert.assertEquals(getCommandOutput(commandParameters), mOutput.toString());
 }
Esempio n. 23
0
 @Test
 public void locationTest() throws IOException {
   int fileId = TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.location(new String[] {"location", "/testFile"});
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/testFile"));
   Assert.assertNotNull(tFile);
   List<String> locationsList = tFile.getLocationHosts();
   String[] commandParameters = new String[3 + locationsList.size()];
   commandParameters[0] = "location";
   commandParameters[1] = "/testFile";
   commandParameters[2] = String.valueOf(fileId);
   Iterator<String> iter = locationsList.iterator();
   int i = 3;
   while (iter.hasNext()) {
     commandParameters[i++] = iter.next();
   }
   Assert.assertEquals(getCommandOutput(commandParameters), mOutput.toString());
 }
Esempio n. 24
0
 @Test
 public void copyToLocalTest() throws IOException {
   TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.copyToLocal(
       new String[] {
         "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
       });
   Assert.assertEquals(
       getCommandOutput(
           new String[] {
             "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
           }),
       mOutput.toString());
   File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
   FileInputStream fis = new FileInputStream(testFile);
   byte[] read = new byte[10];
   fis.read(read);
   fis.close();
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
 }
Esempio n. 25
0
 private void copyToLocalWithBytes(int bytes) throws IOException {
   TachyonFSTestUtils.createByteFile(
       mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, bytes);
   mFsShell.copyToLocal(
       new String[] {
         "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
       });
   Assert.assertEquals(
       getCommandOutput(
           new String[] {
             "copyToLocal", "/testFile", mLocalTachyonCluster.getTachyonHome() + "/testFile"
           }),
       mOutput.toString());
   File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
   FileInputStream fis = new FileInputStream(testFile);
   byte[] read = new byte[bytes];
   fis.read(read);
   fis.close();
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytes, read));
 }
  @Test
  public void readThroughClientTest() throws IOException, TException {
    final int length = 10;
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
    BlockInfo block = getFirstBlockInfo(file);

    RemoteBlockReader client =
        RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
    ByteBuffer result =
        client.readRemoteBlock(
            new InetSocketAddress(
                block.getLocations().get(0).getWorkerAddress().getHost(),
                block.getLocations().get(0).getWorkerAddress().getDataPort()),
            block.getBlockId(),
            0,
            length);

    Assert.assertEquals(BufferUtils.getIncreasingByteBuffer(length), result);
  }
  @Test
  public void promoteBlock() throws Exception {
    TachyonFile file1 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/root/test1",
            TachyonStorageType.STORE,
            UnderStorageType.SYNC_PERSIST,
            MEM_CAPACITY_BYTES / 6);
    TachyonFile file2 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/root/test2",
            TachyonStorageType.STORE,
            UnderStorageType.SYNC_PERSIST,
            MEM_CAPACITY_BYTES / 2);
    TachyonFile file3 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/root/test3",
            TachyonStorageType.STORE,
            UnderStorageType.SYNC_PERSIST,
            MEM_CAPACITY_BYTES / 2);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    TachyonFile toPromote = null;
    int toPromoteLen = 0;
    FileInfo file1Info = mTFS.getInfo(file1);
    FileInfo file2Info = mTFS.getInfo(file2);
    FileInfo file3Info = mTFS.getInfo(file3);

    // We know some file will not be in memory, but not which one since we do not want to make
    // any assumptions on the eviction policy
    if (file1Info.getInMemoryPercentage() < 100) {
      toPromote = file1;
      toPromoteLen = (int) file1Info.getLength();
      Assert.assertEquals(100, file2Info.getInMemoryPercentage());
      Assert.assertEquals(100, file3Info.getInMemoryPercentage());
    } else if (file2Info.getInMemoryPercentage() < 100) {
      toPromote = file2;
      toPromoteLen = (int) file2Info.getLength();
      Assert.assertEquals(100, file1Info.getInMemoryPercentage());
      Assert.assertEquals(100, file3Info.getInMemoryPercentage());
    } else {
      toPromote = file3;
      toPromoteLen = (int) file3Info.getLength();
      Assert.assertEquals(100, file1Info.getInMemoryPercentage());
      Assert.assertEquals(100, file2Info.getInMemoryPercentage());
    }

    FileInStream is =
        mTFS.getInStream(
            toPromote,
            new InStreamOptions.Builder(mWorkerConf)
                .setTachyonStorageType(TachyonStorageType.PROMOTE)
                .build());
    byte[] buf = new byte[toPromoteLen];
    int len = is.read(buf);
    is.close();

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    Assert.assertEquals(toPromoteLen, len);
    Assert.assertEquals(100, mTFS.getInfo(toPromote).getInMemoryPercentage());
  }
Esempio n. 28
0
 @Test
 public void tailEmptyFileTest() throws IOException {
   TachyonFSTestUtils.createByteFile(mTfs, "/emptyFile", WriteType.MUST_CACHE, 0);
   int ret = mFsShell.tail(new String[] {"tail", "/emptyFile"});
   Assert.assertEquals(0, ret);
 }