Пример #1
0
  @Test
  public void copyFromLocalLargeTest() throws IOException, TException {
    File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
    testFile.createNewFile();
    FileOutputStream fos = new FileOutputStream(testFile);
    byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
    fos.write(toWrite);
    fos.close();
    mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
    Assert.assertEquals(
        getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
        mOutput.toString());
    TachyonFile tFile = mTfs.open(new TachyonURI("/testFile"));
    FileInfo fileInfo = mTfs.getInfo(tFile);
    Assert.assertNotNull(fileInfo);
    Assert.assertEquals(SIZE_BYTES, fileInfo.length);

    ClientOptions options =
        new ClientOptions.Builder(new TachyonConf())
            .setTachyonStoreType(TachyonStorageType.NO_STORE)
            .build();
    FileInStream tfis = mTfs.getInStream(tFile, options);
    byte[] read = new byte[SIZE_BYTES];
    tfis.read(read);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
  }
Пример #2
0
  /**
   * Copies a file specified by argv from the filesystem to the local filesystem. This is the
   * utility function.
   *
   * @param srcPath The source TachyonURI (has to be a file)
   * @param dstFile The destination file in the local filesystem
   * @return 0 if command is successful, -1 if an error occurred.
   * @throws IOException
   */
  public int copyFileToLocal(TachyonURI srcPath, File dstFile) throws IOException {
    TachyonFile srcFd;
    try {
      srcFd = mTfs.open(srcPath);
      if (srcFd == null) {
        System.out.println(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(srcPath));
        return -1;
      }
    } catch (TachyonException e) {
      throw new IOException(e);
    }

    Closer closer = Closer.create();
    try {
      InStreamOptions op =
          new InStreamOptions.Builder(mTachyonConf)
              .setTachyonStorageType(TachyonStorageType.NO_STORE)
              .build();
      FileInStream is = closer.register(mTfs.getInStream(srcFd, op));
      FileOutputStream out = closer.register(new FileOutputStream(dstFile));
      byte[] buf = new byte[64 * Constants.MB];
      int t = is.read(buf);
      while (t != -1) {
        out.write(buf, 0, t);
        t = is.read(buf);
      }
      System.out.println("Copied " + srcPath + " to " + dstFile.getPath());
      return 0;
    } catch (TachyonException e) {
      throw new IOException(e.getMessage());
    } finally {
      closer.close();
    }
  }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#read()}. */
  @Test
  public void readTest1() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
        FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);

        for (int i = 0; i < 2; i++) {
          FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
          byte[] ret = new byte[k];
          int value = is.read();
          int cnt = 0;
          while (value != -1) {
            Assert.assertTrue(value >= 0);
            Assert.assertTrue(value < 256);
            ret[cnt++] = (byte) value;
            value = is.read();
          }
          Assert.assertEquals(cnt, k);
          Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
          is.close();
        }
      }
    }
  }
  /** Test <code>void read()</code>. Read from underfs. */
  @Test
  public void readTest1() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      byte[] ret = new byte[k];
      int value = is.read();
      int cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      if (k == 0) {
        Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      } else {
        Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      }

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      value = is.read();
      cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      value = is.read();
      cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
Пример #5
0
 private byte[] readContent(TachyonFile tFile, int length) throws IOException, TException {
   ClientOptions options =
       new ClientOptions.Builder(new TachyonConf())
           .setTachyonStoreType(TachyonStorageType.NO_STORE)
           .build();
   FileInStream tfis = mTfs.getInStream(tFile, options);
   byte[] read = new byte[length];
   tfis.read(read);
   return read;
 }
  /** Test <code>long skip(long len)</code>. */
  @Test
  public void skipTest() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadCache);
      Assert.assertEquals(k / 2, is.skip(k / 2));
      Assert.assertEquals(k / 2, is.read());
      is.close();
      Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);

      if (k >= 3) {
        is = mTfs.getInStream(f, mReadCache);
        int t = k / 3;
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(t, is.read());
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(2 * t + 1, is.read());
        is.close();
        Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      }
    }
  }
  /**
   * Tests that not reading a file the whole way through then closing it will cause it to not
   * recache
   */
  @Test
  public void incompleteFileReadCancelsRecache() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 2, mWriteUnderStore);

    FileInStream is = mTfs.getInStream(f, mReadNoCache);
    Assert.assertEquals(0, is.read());
    is.close();
    Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    is = mTfs.getInStream(f, mReadNoCache);
    is.close();
  }
  /** Tests that reading a file the whole way through with the STORE ReadType will recache it */
  @Test
  public void completeFileReadTriggersRecache() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    int len = 2;
    TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, len, mWriteUnderStore);

    FileInStream is = mTfs.getInStream(f, mReadCache);
    for (int i = 0; i < len; ++i) {
      Assert.assertEquals(i, is.read());
    }
    is.close();
    Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
  }
  // Tests that deletes go through despite failing initially due to concurrent read
  @Test
  public void deleteWhileReadTest() throws Exception {
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test1",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    Assert.assertTrue(mTFS.getInfo(file).getInMemoryPercentage() == 100);
    // Open the file
    InStreamOptions options =
        new InStreamOptions.Builder(new TachyonConf())
            .setTachyonStorageType(TachyonStorageType.STORE)
            .build();
    FileInStream in = mTFS.getInStream(file, options);
    Assert.assertEquals(0, in.read());

    // Delete the file
    mTFS.delete(file);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the delete, the master should no longer serve the file
    Assert.assertNull(mTFS.open(new TachyonURI("/test1")));

    // However, the previous read should still be able to read it as the data still exists
    byte[] res = new byte[MEM_CAPACITY_BYTES];
    Assert.assertEquals(MEM_CAPACITY_BYTES - 1, in.read(res, 1, MEM_CAPACITY_BYTES - 1));
    res[0] = 0;
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MEM_CAPACITY_BYTES, res));
    in.close();

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the file is closed, the master's delete should go through and new files can be created
    TachyonFile newFile =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test2",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
    Assert.assertTrue(mTFS.getInfo(newFile).getInMemoryPercentage() == 100);
  }
Пример #10
0
  private int loadPath(TachyonFileSystem tachyonClient, TachyonURI filePath) throws IOException {
    TachyonFile fd;
    FileInfo fInfo;
    try {
      fd = mTfs.open(filePath);
      if (fd == null) {
        System.out.print(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(filePath));
        return -1;
      }
      fInfo = mTfs.getInfo(fd);
    } catch (IOException ioe) {
      return -1;
    } catch (TachyonException e) {
      throw new IOException(e);
    }

    try {
      if (fInfo.isFolder) {
        List<FileInfo> files = tachyonClient.listStatus(fd);
        Collections.sort(files);
        for (FileInfo file : files) {
          TachyonURI newPath = new TachyonURI(file.getPath());
          if (loadPath(tachyonClient, newPath) == -1) {
            return -1;
          }
        }
        return 0;
      } else {
        Closer closer = Closer.create();
        try {
          InStreamOptions op =
              new InStreamOptions.Builder(mTachyonConf)
                  .setTachyonStorageType(TachyonStorageType.STORE)
                  .build();
          FileInStream in = closer.register(mTfs.getInStream(fd, op));
          byte[] buf = new byte[8 * Constants.MB];
          while (in.read(buf) != -1) {}
          return 0;
        } catch (Throwable e) {
          throw closer.rethrow(e);
        } finally {
          closer.close();
        }
      }
    } catch (TachyonException e) {
      return -1;
    }
  }
  /** Test <code>void read(byte[] b)</code>. Read from underfs. */
  @Test
  public void readTest7() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      byte[] ret = new byte[k];
      Assert.assertEquals(k, is.read(ret));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      Assert.assertEquals(-1, is.read(ret));
      is.close();
      Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
  /**
   * Test <code>void seek(long pos)</code>. Validate the expected exception for seeking a position
   * that is past block size.
   *
   * @throws IOException
   * @throws TachyonException
   */
  @Test
  public void seekExceptionTest2() throws IOException, TachyonException {
    mThrown.expect(IllegalArgumentException.class);
    mThrown.expectMessage("Seek position is past EOF: 1, fileSize = 0");
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      try {
        is.seek(k + 1);
      } finally {
        is.close();
      }
    }
  }
Пример #13
0
  /**
   * Prints the file's last 1KB of contents to the console.
   *
   * @param path The TachyonURI path as the input of the command
   * @return 0 if command is successful, -1 if an error occurred.f
   * @throws IOException
   */
  public int tail(TachyonURI path) throws IOException {
    TachyonFile fd;
    FileInfo fInfo;
    try {
      fd = mTfs.open(path);
      if (fd == null) {
        System.out.println(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path));
        return -1;
      }
      fInfo = mTfs.getInfo(fd);
    } catch (TachyonException e) {
      throw new IOException(e);
    }

    if (!fInfo.isFolder) {
      InStreamOptions op =
          new InStreamOptions.Builder(mTachyonConf)
              .setTachyonStorageType(TachyonStorageType.NO_STORE)
              .build();
      FileInStream is = null;
      try {
        is = mTfs.getInStream(fd, op);
        byte[] buf = new byte[Constants.KB];
        long bytesToRead = 0L;
        if (fInfo.getLength() > Constants.KB) {
          bytesToRead = Constants.KB;
        } else {
          bytesToRead = fInfo.getLength();
        }
        is.skip(fInfo.getLength() - bytesToRead);
        int read = is.read(buf);
        if (read != -1) {
          System.out.write(buf, 0, read);
        }
        return 0;
      } catch (TachyonException e) {
        throw new IOException(e);
      } finally {
        is.close();
      }
    } else {
      System.out.println(path + " is not a file.");
      return -1;
    }
  }
Пример #14
0
  /**
   * Prints the file's contents to the console.
   *
   * @param path The TachyonURI path as the input of the command
   * @return 0 if command is successful, -1 if an error occurred.
   * @throws IOException
   */
  public int cat(TachyonURI path) throws IOException {
    TachyonFile fd;
    FileInfo tFile;
    try {
      fd = mTfs.open(path);
      if (fd == null) {
        System.out.print(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path));
        return -1;
      }
      tFile = mTfs.getInfo(fd);
    } catch (TachyonException e) {
      throw new IOException(e);
    }

    if (!tFile.isFolder) {
      InStreamOptions op =
          new InStreamOptions.Builder(mTachyonConf)
              .setTachyonStorageType(TachyonStorageType.NO_STORE)
              .build();
      FileInStream is;
      try {
        is = mTfs.getInStream(fd, op);
      } catch (TachyonException e) {
        System.out.print(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path));
        return -1;
      }
      byte[] buf = new byte[512];
      try {
        int read = is.read(buf);
        while (read != -1) {
          System.out.write(buf, 0, read);
          read = is.read(buf);
        }
      } finally {
        is.close();
      }
      return 0;
    } else {
      System.out.println(path + " is not a file.");
      return -1;
    }
  }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#read(byte[])}. */
  @Test
  public void readTest2() throws IOException, TachyonException {
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());

        FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        byte[] ret = new byte[k];
        Assert.assertEquals(k, is.read(ret));
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();

        is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        ret = new byte[k];
        Assert.assertEquals(k, is.read(ret));
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();
      }
    }
  }
  /** Tests that reading a file consisting of more than one block from the underfs works */
  @Test
  public void readMultiBlockFile() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    int blockSizeByte = 10;
    int numBlocks = 10;
    FileOutStream os = mTfs.getOutStream(new TachyonURI(uniqPath), mWriteUnderStore);
    for (int i = 0; i < numBlocks; i++) {
      for (int j = 0; j < blockSizeByte; j++) {
        os.write((byte) (i * blockSizeByte + j));
      }
    }
    os.close();

    TachyonFile f = mTfs.open(new TachyonURI(uniqPath));
    FileInStream is = mTfs.getInStream(f, mReadCache);
    for (int i = 0; i < blockSizeByte * numBlocks; i++) {
      Assert.assertEquals((byte) i, is.read());
    }
    is.close();
    Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
  }
  /**
   * Test <code>void seek(long pos)</code>.
   *
   * @throws IOException
   * @throws TachyonException
   */
  @Test
  public void seekTest() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);

      Assert.assertEquals(0, is.read());
      is.seek(k / 3);
      Assert.assertEquals(k / 3, is.read());
      is.seek(k / 2);
      Assert.assertEquals(k / 2, is.read());
      is.seek(k / 4);
      Assert.assertEquals(k / 4, is.read());
      is.close();
    }
  }
 /** Tests that seeking around a file cached locally works. */
 @Test
 public void seekAroundLocalBlock() throws IOException, TachyonException {
   String uniqPath = PathUtils.uniqPath();
   // The number of bytes per remote block read should be set to 100 in the before function
   TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 200, mWriteTachyon);
   FileInStream is = mTfs.getInStream(f, mReadNoCache);
   Assert.assertEquals(0, is.read());
   is.seek(199);
   Assert.assertEquals(199, is.read());
   is.seek(99);
   Assert.assertEquals(99, is.read());
   is.close();
 }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#skip(long)}. */
  @Test
  public void skipTest() throws IOException, TachyonException {
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());

        FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));

        Assert.assertEquals(k / 2, is.skip(k / 2));
        Assert.assertEquals(k / 2, is.read());
        is.close();

        is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        int t = k / 3;
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(t, is.read());
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(2 * t + 1, is.read());
        is.close();
      }
    }
  }
  @Test
  public void promoteBlock() throws Exception {
    TachyonFile file1 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/root/test1",
            TachyonStorageType.STORE,
            UnderStorageType.SYNC_PERSIST,
            MEM_CAPACITY_BYTES / 6);
    TachyonFile file2 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/root/test2",
            TachyonStorageType.STORE,
            UnderStorageType.SYNC_PERSIST,
            MEM_CAPACITY_BYTES / 2);
    TachyonFile file3 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/root/test3",
            TachyonStorageType.STORE,
            UnderStorageType.SYNC_PERSIST,
            MEM_CAPACITY_BYTES / 2);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    TachyonFile toPromote = null;
    int toPromoteLen = 0;
    FileInfo file1Info = mTFS.getInfo(file1);
    FileInfo file2Info = mTFS.getInfo(file2);
    FileInfo file3Info = mTFS.getInfo(file3);

    // We know some file will not be in memory, but not which one since we do not want to make
    // any assumptions on the eviction policy
    if (file1Info.getInMemoryPercentage() < 100) {
      toPromote = file1;
      toPromoteLen = (int) file1Info.getLength();
      Assert.assertEquals(100, file2Info.getInMemoryPercentage());
      Assert.assertEquals(100, file3Info.getInMemoryPercentage());
    } else if (file2Info.getInMemoryPercentage() < 100) {
      toPromote = file2;
      toPromoteLen = (int) file2Info.getLength();
      Assert.assertEquals(100, file1Info.getInMemoryPercentage());
      Assert.assertEquals(100, file3Info.getInMemoryPercentage());
    } else {
      toPromote = file3;
      toPromoteLen = (int) file3Info.getLength();
      Assert.assertEquals(100, file1Info.getInMemoryPercentage());
      Assert.assertEquals(100, file2Info.getInMemoryPercentage());
    }

    FileInStream is =
        mTFS.getInStream(
            toPromote,
            new InStreamOptions.Builder(mWorkerConf)
                .setTachyonStorageType(TachyonStorageType.PROMOTE)
                .build());
    byte[] buf = new byte[toPromoteLen];
    int len = is.read(buf);
    is.close();

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    Assert.assertEquals(toPromoteLen, len);
    Assert.assertEquals(100, mTFS.getInfo(toPromote).getInMemoryPercentage());
  }