// Tests that files are evicted when there is not enough space in the worker.
  @Test
  public void evictionTest() throws Exception {
    final int blockSize = (int) WORKER_CAPACITY_BYTES / 2;
    AlluxioURI file1 = new AlluxioURI("/file1");
    FileSystemTestUtils.createByteFile(mFileSystem, file1, WriteType.MUST_CACHE, blockSize);

    // File should be in memory after it is written with MUST_CACHE
    URIStatus fileInfo1 = mFileSystem.getStatus(file1);
    Assert.assertEquals(100, fileInfo1.getInMemoryPercentage());

    AlluxioURI file2 = new AlluxioURI("/file2");
    FileSystemTestUtils.createByteFile(mFileSystem, file2, WriteType.MUST_CACHE, blockSize);

    // Both file 1 and 2 should be in memory since the combined size is not larger than worker space
    fileInfo1 = mFileSystem.getStatus(file1);
    URIStatus fileInfo2 = mFileSystem.getStatus(file2);
    Assert.assertEquals(100, fileInfo1.getInMemoryPercentage());
    Assert.assertEquals(100, fileInfo2.getInMemoryPercentage());

    AlluxioURI file3 = new AlluxioURI("/file3");
    FileSystemTestUtils.createByteFile(mFileSystem, file3, WriteType.MUST_CACHE, blockSize);

    waitForHeartbeat();

    fileInfo1 = mFileSystem.getStatus(file1);
    fileInfo2 = mFileSystem.getStatus(file2);
    URIStatus fileInfo3 = mFileSystem.getStatus(file3);

    // File 3 should be in memory and one of file 1 or 2 should be in memory
    Assert.assertEquals(100, fileInfo3.getInMemoryPercentage());
    Assert.assertTrue(
        "Exactly one of file1 and file2 should be 100% in memory",
        fileInfo1.getInMemoryPercentage() == 100 ^ fileInfo2.getInMemoryPercentage() == 100);
  }
Esempio n. 2
0
 /**
  * Constructs a new stream for reading a file from HDFS.
  *
  * @param uri the Alluxio file URI
  * @param conf Hadoop configuration
  * @param bufferSize the buffer size
  * @param stats filesystem statistics
  * @throws IOException if the underlying file does not exist or its stream cannot be created
  */
 public HdfsFileInputStream(
     AlluxioURI uri,
     org.apache.hadoop.conf.Configuration conf,
     int bufferSize,
     org.apache.hadoop.fs.FileSystem.Statistics stats)
     throws IOException {
   LOG.debug("HdfsFileInputStream({}, {}, {}, {}, {})", uri, conf, bufferSize, stats);
   long bufferBytes = Configuration.getBytes(Constants.USER_FILE_BUFFER_BYTES);
   mBuffer = new byte[Ints.checkedCast(bufferBytes) * 4];
   mCurrentPosition = 0;
   FileSystem fs = FileSystem.Factory.get();
   mHadoopConf = conf;
   mHadoopBufferSize = bufferSize;
   mStatistics = stats;
   try {
     mFileInfo = fs.getStatus(uri);
     mHdfsPath = new Path(mFileInfo.getUfsPath());
     mAlluxioFileInputStream = fs.openFile(uri, OpenFileOptions.defaults());
   } catch (FileDoesNotExistException e) {
     throw new FileNotFoundException(
         ExceptionMessage.HDFS_FILE_NOT_FOUND.getMessage(mHdfsPath, uri));
   } catch (AlluxioException e) {
     throw new IOException(e);
   }
 }
  // Tests that caching a block successfully persists the block if the block exists
  @Test
  public void cacheBlockTest() throws Exception {
    mFileSystem.createFile(new AlluxioURI("/testFile"));
    URIStatus file = mFileSystem.getStatus(new AlluxioURI("/testFile"));

    final int blockSize = (int) WORKER_CAPACITY_BYTES / 10;
    // Construct the block ids for the file.
    final long blockId0 = BlockId.createBlockId(BlockId.getContainerId(file.getFileId()), 0);
    final long blockId1 = BlockId.createBlockId(BlockId.getContainerId(file.getFileId()), 1);

    String filename =
        mBlockWorkerServiceHandler.requestBlockLocation(SESSION_ID, blockId0, blockSize);
    createBlockFile(filename, blockSize);
    mBlockWorkerServiceHandler.cacheBlock(SESSION_ID, blockId0);

    // The master should be immediately updated with the persisted block
    Assert.assertEquals(blockSize, mBlockMasterClient.getUsedBytes());

    // Attempting to cache a non existent block should throw an exception
    Exception exception = null;
    try {
      mBlockWorkerServiceHandler.cacheBlock(SESSION_ID, blockId1);
    } catch (TException e) {
      exception = e;
    }
    Assert.assertNotNull(exception);
  }
Esempio n. 4
0
 private List<FileBlockInfo> getFileBlocks(AlluxioURI path) throws IOException {
   try {
     return mFileSystem.getStatus(path).getFileBlockInfos();
   } catch (AlluxioException e) {
     throw new IOException(e);
   }
 }
  // Tests that lock block returns the correct path
  @Test
  public void lockBlockTest() throws Exception {
    final int blockSize = (int) WORKER_CAPACITY_BYTES / 2;

    CreateFileOptions options =
        CreateFileOptions.defaults()
            .setBlockSizeBytes(blockSize)
            .setWriteType(WriteType.MUST_CACHE);
    FileOutStream out = mFileSystem.createFile(new AlluxioURI("/testFile"), options);
    URIStatus file = mFileSystem.getStatus(new AlluxioURI("/testFile"));

    final long blockId = BlockId.createBlockId(BlockId.getContainerId(file.getFileId()), 0);

    out.write(BufferUtils.getIncreasingByteArray(blockSize));
    out.close();

    String localPath = mBlockWorkerServiceHandler.lockBlock(blockId, SESSION_ID).getBlockPath();

    // The local path should exist
    Assert.assertNotNull(localPath);

    UnderFileSystem ufs = UnderFileSystem.get(localPath, mMasterConfiguration);
    byte[] data = new byte[blockSize];
    int bytesRead = ufs.open(localPath).read(data);

    // The data in the local file should equal the data we wrote earlier
    Assert.assertEquals(blockSize, bytesRead);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytesRead, data));

    mBlockWorkerServiceHandler.unlockBlock(blockId, SESSION_ID);
  }
Esempio n. 6
0
 /**
  * Convenience method which ensures the given path exists, wrapping any {@link AlluxioException}
  * in {@link IOException}.
  *
  * @param path the path to look up
  * @throws IOException if an Alluxio exception occurs
  */
 private void ensureExists(AlluxioURI path) throws IOException {
   try {
     mFileSystem.getStatus(path);
   } catch (AlluxioException e) {
     throw new IOException(e);
   }
 }
Esempio n. 7
0
  @Override
  public boolean rename(Path src, Path dst) throws IOException {
    LOG.info("rename({}, {})", src, dst);
    if (mStatistics != null) {
      mStatistics.incrementWriteOps(1);
    }

    AlluxioURI srcPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(src));
    AlluxioURI dstPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(dst));
    ensureExists(srcPath);
    URIStatus dstStatus;
    try {
      dstStatus = mFileSystem.getStatus(dstPath);
    } catch (IOException | AlluxioException e) {
      dstStatus = null;
    }
    // If the destination is an existing folder, try to move the src into the folder
    if (dstStatus != null && dstStatus.isFolder()) {
      dstPath = dstPath.join(srcPath.getName());
    }
    try {
      mFileSystem.rename(srcPath, dstPath);
      return true;
    } catch (IOException | AlluxioException e) {
      LOG.error("Failed to rename {} to {}", src, dst, e);
      return false;
    }
  }
Esempio n. 8
0
  /**
   * {@inheritDoc}
   *
   * <p>If the file does not exist in Alluxio, query it from HDFS.
   */
  @Override
  public FileStatus getFileStatus(Path path) throws IOException {
    LOG.info("getFileStatus({})", path);

    if (mStatistics != null) {
      mStatistics.incrementReadOps(1);
    }
    AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
    URIStatus fileStatus;
    try {
      fileStatus = mFileSystem.getStatus(uri);
    } catch (FileDoesNotExistException e) {
      throw new FileNotFoundException(e.getMessage());
    } catch (AlluxioException e) {
      throw new IOException(e);
    }

    return new FileStatus(
        fileStatus.getLength(),
        fileStatus.isFolder(),
        BLOCK_REPLICATION_CONSTANT,
        fileStatus.getBlockSizeBytes(),
        fileStatus.getLastModificationTimeMs(),
        fileStatus.getCreationTimeMs(),
        new FsPermission((short) fileStatus.getMode()),
        fileStatus.getOwner(),
        fileStatus.getGroup(),
        new Path(mAlluxioHeader + uri));
  }
Esempio n. 9
0
  /**
   * Attempts to create a file. Overwrite will not succeed if the path exists and is a folder.
   *
   * @param path path to create
   * @param permission permissions of the created file/folder
   * @param overwrite overwrite if file exists
   * @param bufferSize the size in bytes of the buffer to be used
   * @param replication under filesystem replication factor
   * @param blockSize block size in bytes
   * @param progress queryable progress
   * @return an {@link FSDataOutputStream} created at the indicated path of a file
   * @throws IOException if overwrite is not specified and the path already exists or if the path is
   *     a folder
   */
  @Override
  public FSDataOutputStream create(
      Path path,
      FsPermission permission,
      boolean overwrite,
      int bufferSize,
      short replication,
      long blockSize,
      Progressable progress)
      throws IOException {
    LOG.info(
        "create({}, {}, {}, {}, {}, {}, {})",
        path,
        permission,
        overwrite,
        bufferSize,
        replication,
        blockSize,
        progress);
    if (mStatistics != null) {
      mStatistics.incrementWriteOps(1);
    }

    // Check whether the file already exists, and delete it if overwrite is true
    AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
    try {
      if (mFileSystem.exists(uri)) {
        if (!overwrite) {
          throw new IOException(ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(uri));
        }
        if (mFileSystem.getStatus(uri).isFolder()) {
          throw new IOException(ExceptionMessage.FILE_CREATE_IS_DIRECTORY.getMessage(uri));
        }
        mFileSystem.delete(uri);
      }
    } catch (AlluxioException e) {
      throw new IOException(e);
    }

    // The file no longer exists at this point, so we can create it
    CreateFileOptions options =
        CreateFileOptions.defaults()
            .setBlockSizeBytes(blockSize)
            .setMode(new Mode(permission.toShort()));
    try {
      FileOutStream outStream = mFileSystem.createFile(uri, options);
      return new FSDataOutputStream(outStream, mStatistics);
    } catch (AlluxioException e) {
      throw new IOException(e);
    }
  }
  // Tests that lock block returns error on failure
  @Test
  public void lockBlockFailureTest() throws Exception {
    mFileSystem.createFile(new AlluxioURI("/testFile"));
    URIStatus file = mFileSystem.getStatus(new AlluxioURI("/testFile"));
    final long blockId = BlockId.createBlockId(BlockId.getContainerId(file.getFileId()), 0);

    Exception exception = null;
    try {
      mBlockWorkerServiceHandler.lockBlock(blockId, SESSION_ID);
    } catch (AlluxioTException e) {
      exception = e;
    }

    // A file does not exist exception should have been thrown
    Assert.assertNotNull(exception);
  }
  // Tests that cancelling a block will remove the temporary file
  @Test
  public void cancelBlockTest() throws Exception {
    mFileSystem.createFile(new AlluxioURI("/testFile"));
    URIStatus file = mFileSystem.getStatus(new AlluxioURI("/testFile"));

    final int blockSize = (int) WORKER_CAPACITY_BYTES / 2;
    final long blockId = BlockId.createBlockId(BlockId.getContainerId(file.getFileId()), 0);

    String filename =
        mBlockWorkerServiceHandler.requestBlockLocation(SESSION_ID, blockId, blockSize);
    createBlockFile(filename, blockSize);
    mBlockWorkerServiceHandler.cancelBlock(SESSION_ID, blockId);

    // The block should not exist after being cancelled
    Assert.assertFalse(new File(filename).exists());

    // The master should not have recorded any used space after the block is cancelled
    waitForHeartbeat();
    Assert.assertEquals(0, mBlockMasterClient.getUsedBytes());
  }