예제 #1
0
 /** @return the remaining capacity (in bytes) */
 public long getAvailableBytes() {
   long availableBytes = 0;
   for (StorageDir dir : mDirs) {
     availableBytes += dir.getAvailableBytes();
   }
   return availableBytes;
 }
예제 #2
0
 /**
  * Cleans up the meta data of the given temp block ids
  *
  * @param userId the ID of the client associated with the temp blocks
  * @param tempBlockIds the list of temporary block ids to be cleaned up, non temporary block ids
  *     will be ignored.
  */
 public synchronized void cleanupUserTempBlocks(long userId, List<Long> tempBlockIds) {
   for (StorageTier tier : mTiers) {
     for (StorageDir dir : tier.getStorageDirs()) {
       dir.cleanupUserTempBlocks(userId, tempBlockIds);
     }
   }
 }
예제 #3
0
 /**
  * Commits a temp block.
  *
  * @param tempBlockMeta the meta data of the temp block to commit
  * @throws OutOfSpaceException when no more space left to hold the block
  * @throws AlreadyExistsException when the block already exists in committed blocks
  * @throws NotFoundException when temp block can not be found
  */
 public synchronized void commitTempBlockMeta(TempBlockMeta tempBlockMeta)
     throws OutOfSpaceException, AlreadyExistsException, NotFoundException {
   BlockMeta block = new BlockMeta(Preconditions.checkNotNull(tempBlockMeta));
   StorageDir dir = tempBlockMeta.getParentDir();
   dir.removeTempBlockMeta(tempBlockMeta);
   dir.addBlockMeta(block);
 }
예제 #4
0
 /**
  * Gets all the temporary blocks associated with a user, empty list is returned if the user has no
  * temporary blocks.
  *
  * @param userId the ID of the user
  * @return A list of temp blocks associated with the user
  */
 public synchronized List<TempBlockMeta> getUserTempBlocks(long userId) {
   List<TempBlockMeta> userTempBlocks = new ArrayList<TempBlockMeta>();
   for (StorageTier tier : mTiers) {
     for (StorageDir dir : tier.getStorageDirs()) {
       userTempBlocks.addAll(dir.getUserTempBlocks(userId));
     }
   }
   return userTempBlocks;
 }
예제 #5
0
 /**
  * Checks if the storage has a given temp block.
  *
  * @param blockId the temp block ID
  * @return true if the block is contained, false otherwise
  */
 public synchronized boolean hasTempBlockMeta(long blockId) {
   for (StorageTier tier : mTiers) {
     for (StorageDir dir : tier.getStorageDirs()) {
       if (dir.hasTempBlockMeta(blockId)) {
         return true;
       }
     }
   }
   return false;
 }
예제 #6
0
 /**
  * Gets the metadata of a block given its blockId.
  *
  * @param blockId the block ID
  * @return metadata of the block
  * @throws NotFoundException if no BlockMeta for this blockId is found
  */
 public synchronized BlockMeta getBlockMeta(long blockId) throws NotFoundException {
   for (StorageTier tier : mTiers) {
     for (StorageDir dir : tier.getStorageDirs()) {
       if (dir.hasBlockMeta(blockId)) {
         return dir.getBlockMeta(blockId);
       }
     }
   }
   throw new NotFoundException("Failed to get BlockMeta: blockId " + blockId + " not found");
 }
예제 #7
0
  private void initStorageTier() throws AlreadyExistsException, IOException, OutOfSpaceException {
    String workerDataFolder =
        WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER, Constants.DEFAULT_DATA_FOLDER);
    String tierDirPathConf =
        String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, mTierLevel);
    String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf, "/mnt/ramdisk").split(",");

    // Add the worker data folder path after each storage directory, the final path will be like
    // /mnt/ramdisk/tachyonworker
    for (int i = 0; i < dirPaths.length; i++) {
      dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder);
    }

    String tierDirCapacityConf =
        String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel);
    String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf, "0").split(",");

    mDirs = new ArrayList<StorageDir>(dirPaths.length);

    long totalCapacity = 0;
    for (int i = 0; i < dirPaths.length; i++) {
      int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i;
      long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]);
      totalCapacity += capacity;
      mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i]));
    }
    mCapacityBytes = totalCapacity;
  }
예제 #8
0
 public BlockStoreMeta(BlockMetadataManager manager) {
   Preconditions.checkNotNull(manager);
   for (StorageTier tier : manager.getTiers()) {
     int aliasIndex = tier.getTierAlias() - 1;
     mCapacityBytesOnTiers.set(
         aliasIndex, mCapacityBytesOnTiers.get(aliasIndex) + tier.getCapacityBytes());
     mUsedBytesOnTiers.set(
         aliasIndex,
         mUsedBytesOnTiers.get(aliasIndex) + (tier.getCapacityBytes() - tier.getAvailableBytes()));
     for (StorageDir dir : tier.getStorageDirs()) {
       mBlockIdsOnDirs.put(dir.getStorageDirId(), dir.getBlockIds());
       mCapacityBytesOnDirs.put(dir.getStorageDirId(), dir.getCapacityBytes());
       mUsedBytesOnDirs.put(
           dir.getStorageDirId(), dir.getCapacityBytes() - dir.getAvailableBytes());
       mDirPaths.put(dir.getStorageDirId(), dir.getDirPath());
     }
   }
 }
예제 #9
0
 /**
  * Get evictable bytes for this dir, i.e., the total bytes of total evictable blocks
  *
  * @return evictable bytes for this dir
  */
 public long getEvitableBytes() {
   long bytes = 0;
   for (BlockMeta blockMeta : mDir.getBlocks()) {
     long blockId = blockMeta.getBlockId();
     if (mManagerView.isBlockEvictable(blockId)) {
       bytes += blockMeta.getBlockSize();
     }
   }
   return bytes;
 }
예제 #10
0
  /**
   * Get a filtered list of block metadata, for blocks that are neither pinned or being blocked.
   *
   * @return a list of metadata for all evictable blocks
   */
  public List<BlockMeta> getEvictableBlocks() {
    List<BlockMeta> filteredList = new ArrayList<BlockMeta>();

    for (BlockMeta blockMeta : mDir.getBlocks()) {
      long blockId = blockMeta.getBlockId();
      if (mManagerView.isBlockEvictable(blockId)) {
        filteredList.add(blockMeta);
      }
    }
    return filteredList;
  }
예제 #11
0
  /**
   * Gets the amount of available space of given location in bytes. Master queries the total number
   * of bytes available on each tier of the worker, and Evictor/Allocator often cares about the
   * bytes at a {@link StorageDir}.
   *
   * @param location location the check available bytes
   * @return available bytes
   * @throws IllegalArgumentException when location does not belong to tiered storage
   */
  public synchronized long getAvailableBytes(BlockStoreLocation location) {
    long spaceAvailable = 0;

    if (location.equals(BlockStoreLocation.anyTier())) {
      for (StorageTier tier : mTiers) {
        spaceAvailable += tier.getAvailableBytes();
      }
      return spaceAvailable;
    }

    int tierAlias = location.tierAlias();
    StorageTier tier = getTier(tierAlias);
    // TODO: This should probably be max of the capacity bytes in the dirs?
    if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
      return tier.getAvailableBytes();
    }

    int dirIndex = location.dir();
    StorageDir dir = tier.getDir(dirIndex);
    return dir.getAvailableBytes();
  }
예제 #12
0
  /**
   * Moves the metadata of an existing block to another location or throws IOExceptions.
   *
   * @param blockMeta the meta data of the block to move
   * @param newLocation new location of the block
   * @return the new block metadata if success, absent otherwise
   * @throws IllegalArgumentException when the newLocation is not in the tiered storage
   * @throws NotFoundException when the block to move is not found
   * @throws AlreadyExistsException when the block to move already exists in the destination
   * @throws OutOfSpaceException when destination have no extra space to hold the block to move
   */
  public synchronized BlockMeta moveBlockMeta(BlockMeta blockMeta, BlockStoreLocation newLocation)
      throws NotFoundException, AlreadyExistsException, OutOfSpaceException {
    // If existing location belongs to the target location, simply return the current block meta.
    BlockStoreLocation oldLocation = blockMeta.getBlockLocation();
    if (oldLocation.belongTo(newLocation)) {
      LOG.info("moveBlockMeta: moving {} to {} is a noop", oldLocation, newLocation);
      return blockMeta;
    }

    long blockSize = blockMeta.getBlockSize();
    int newTierAlias = newLocation.tierAlias();
    StorageTier newTier = getTier(newTierAlias);
    StorageDir newDir = null;
    if (newLocation.equals(BlockStoreLocation.anyDirInTier(newTierAlias))) {
      for (StorageDir dir : newTier.getStorageDirs()) {
        if (dir.getAvailableBytes() >= blockSize) {
          newDir = dir;
          break;
        }
      }
    } else {
      StorageDir dir = newTier.getDir(newLocation.dir());
      if (dir.getAvailableBytes() >= blockSize) {
        newDir = dir;
      }
    }

    if (newDir == null) {
      throw new OutOfSpaceException(
          "Failed to move BlockMeta: newLocation "
              + newLocation
              + " does not have enough space for "
              + blockSize
              + " bytes");
    }
    StorageDir oldDir = blockMeta.getParentDir();
    oldDir.removeBlockMeta(blockMeta);
    BlockMeta newBlockMeta = new BlockMeta(blockMeta.getBlockId(), blockSize, newDir);
    newDir.addBlockMeta(newBlockMeta);
    return newBlockMeta;
  }
예제 #13
0
 /**
  * Adds a temp block.
  *
  * @param tempBlockMeta the meta data of the temp block to add
  * @throws OutOfSpaceException when no more space left to hold the block
  * @throws AlreadyExistsException when the block already exists
  */
 public synchronized void addTempBlockMeta(TempBlockMeta tempBlockMeta)
     throws OutOfSpaceException, AlreadyExistsException {
   StorageDir dir = tempBlockMeta.getParentDir();
   dir.addTempBlockMeta(tempBlockMeta);
 }
예제 #14
0
 /**
  * Get capacity bytes for this dir
  *
  * @return capacity bytes for this dir
  */
 public long getCapacityBytes() {
   return mDir.getCapacityBytes();
 }
예제 #15
0
 /**
  * Get the index of this Dir
  *
  * @return index of the dir
  */
 public int getDirViewIndex() {
   return mDir.getDirIndex();
 }
예제 #16
0
 /**
  * Create a BlockStoraLocation for this directory view. Redirecting to {@link
  * StorageDir#toBlockStoreLocation}
  *
  * @return BlockStoreLocation created
  */
 public BlockStoreLocation toBlockStoreLocation() {
   return mDir.toBlockStoreLocation();
 }
예제 #17
0
 /**
  * Get committed bytes for this dir. This includes all blocks, locked, pinned, committed etc.
  *
  * @return committed bytes for this dir
  */
 public long getCommittedBytes() {
   return mDir.getCommittedBytes();
 }
예제 #18
0
 /**
  * Get available bytes for this dir
  *
  * @return available bytes for this dir
  */
 public long getAvailableBytes() {
   return mDir.getAvailableBytes() + mBlocksToMoveOutSize - mBlocksToMoveInSize;
 }
예제 #19
0
 /**
  * Factory method to create {@link StorageDir}
  *
  * <p>It will load meta data of existing committed blocks in the dirPath specified. Only files
  * with directory depth 1 under dirPath and whose file name can be parsed into {@code long} will
  * be considered as existing committed blocks, these files will be preserved, others files or
  * directories will be deleted.
  *
  * @param tier the {@link StorageTier} this dir belongs to
  * @param dirIndex the index of this dir in its tier
  * @param capacityBytes the initial capacity of this dir, can not be modified later
  * @param dirPath filesystem path of this dir for actual storage
  * @return the new created StorageDir
  * @throws IOException when meta data of existing committed blocks can not be loaded
  */
 public static StorageDir newStorageDir(
     StorageTier tier, int dirIndex, long capacityBytes, String dirPath) throws IOException {
   StorageDir dir = new StorageDir(tier, dirIndex, capacityBytes, dirPath);
   dir.initializeMeta();
   return dir;
 }
예제 #20
0
 /**
  * Aborts a temp block.
  *
  * @param tempBlockMeta the meta data of the temp block to add
  * @throws NotFoundException when block can not be found
  */
 public synchronized void abortTempBlockMeta(TempBlockMeta tempBlockMeta)
     throws NotFoundException {
   StorageDir dir = tempBlockMeta.getParentDir();
   dir.removeTempBlockMeta(tempBlockMeta);
 }
예제 #21
0
 /**
  * Modifies the size of a temp block
  *
  * @param tempBlockMeta the temp block to modify
  * @param newSize new size in bytes
  * @throws InvalidStateException when newSize is smaller than current size
  */
 public synchronized void resizeTempBlockMeta(TempBlockMeta tempBlockMeta, long newSize)
     throws InvalidStateException {
   StorageDir dir = tempBlockMeta.getParentDir();
   dir.resizeTempBlockMeta(tempBlockMeta, newSize);
 }
예제 #22
0
 /**
  * Remove the metadata of a specific block.
  *
  * @param block the meta data of the block to remove
  * @throws NotFoundException when block is not found
  */
 public synchronized void removeBlockMeta(BlockMeta block) throws NotFoundException {
   StorageDir dir = block.getParentDir();
   dir.removeBlockMeta(block);
 }