/** @return the remaining capacity (in bytes) */
public long getAvailableBytes() {
long availableBytes = 0;
for (StorageDir dir : mDirs) {
availableBytes += dir.getAvailableBytes();
}
return availableBytes;
}
/**
* Cleans up the meta data of the given temp block ids
*
* @param userId the ID of the client associated with the temp blocks
* @param tempBlockIds the list of temporary block ids to be cleaned up, non temporary block ids
* will be ignored.
*/
public synchronized void cleanupUserTempBlocks(long userId, List<Long> tempBlockIds) {
for (StorageTier tier : mTiers) {
for (StorageDir dir : tier.getStorageDirs()) {
dir.cleanupUserTempBlocks(userId, tempBlockIds);
}
}
}
/**
* Commits a temp block.
*
* @param tempBlockMeta the meta data of the temp block to commit
* @throws OutOfSpaceException when no more space left to hold the block
* @throws AlreadyExistsException when the block already exists in committed blocks
* @throws NotFoundException when temp block can not be found
*/
public synchronized void commitTempBlockMeta(TempBlockMeta tempBlockMeta)
throws OutOfSpaceException, AlreadyExistsException, NotFoundException {
BlockMeta block = new BlockMeta(Preconditions.checkNotNull(tempBlockMeta));
StorageDir dir = tempBlockMeta.getParentDir();
dir.removeTempBlockMeta(tempBlockMeta);
dir.addBlockMeta(block);
}
/**
* Gets all the temporary blocks associated with a user, empty list is returned if the user has no
* temporary blocks.
*
* @param userId the ID of the user
* @return A list of temp blocks associated with the user
*/
public synchronized List<TempBlockMeta> getUserTempBlocks(long userId) {
List<TempBlockMeta> userTempBlocks = new ArrayList<TempBlockMeta>();
for (StorageTier tier : mTiers) {
for (StorageDir dir : tier.getStorageDirs()) {
userTempBlocks.addAll(dir.getUserTempBlocks(userId));
}
}
return userTempBlocks;
}
/**
* Checks if the storage has a given temp block.
*
* @param blockId the temp block ID
* @return true if the block is contained, false otherwise
*/
public synchronized boolean hasTempBlockMeta(long blockId) {
for (StorageTier tier : mTiers) {
for (StorageDir dir : tier.getStorageDirs()) {
if (dir.hasTempBlockMeta(blockId)) {
return true;
}
}
}
return false;
}
/**
* Gets the metadata of a block given its blockId.
*
* @param blockId the block ID
* @return metadata of the block
* @throws NotFoundException if no BlockMeta for this blockId is found
*/
public synchronized BlockMeta getBlockMeta(long blockId) throws NotFoundException {
for (StorageTier tier : mTiers) {
for (StorageDir dir : tier.getStorageDirs()) {
if (dir.hasBlockMeta(blockId)) {
return dir.getBlockMeta(blockId);
}
}
}
throw new NotFoundException("Failed to get BlockMeta: blockId " + blockId + " not found");
}
private void initStorageTier() throws AlreadyExistsException, IOException, OutOfSpaceException {
String workerDataFolder =
WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER, Constants.DEFAULT_DATA_FOLDER);
String tierDirPathConf =
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, mTierLevel);
String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf, "/mnt/ramdisk").split(",");
// Add the worker data folder path after each storage directory, the final path will be like
// /mnt/ramdisk/tachyonworker
for (int i = 0; i < dirPaths.length; i++) {
dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder);
}
String tierDirCapacityConf =
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel);
String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf, "0").split(",");
mDirs = new ArrayList<StorageDir>(dirPaths.length);
long totalCapacity = 0;
for (int i = 0; i < dirPaths.length; i++) {
int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i;
long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]);
totalCapacity += capacity;
mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i]));
}
mCapacityBytes = totalCapacity;
}
public BlockStoreMeta(BlockMetadataManager manager) {
Preconditions.checkNotNull(manager);
for (StorageTier tier : manager.getTiers()) {
int aliasIndex = tier.getTierAlias() - 1;
mCapacityBytesOnTiers.set(
aliasIndex, mCapacityBytesOnTiers.get(aliasIndex) + tier.getCapacityBytes());
mUsedBytesOnTiers.set(
aliasIndex,
mUsedBytesOnTiers.get(aliasIndex) + (tier.getCapacityBytes() - tier.getAvailableBytes()));
for (StorageDir dir : tier.getStorageDirs()) {
mBlockIdsOnDirs.put(dir.getStorageDirId(), dir.getBlockIds());
mCapacityBytesOnDirs.put(dir.getStorageDirId(), dir.getCapacityBytes());
mUsedBytesOnDirs.put(
dir.getStorageDirId(), dir.getCapacityBytes() - dir.getAvailableBytes());
mDirPaths.put(dir.getStorageDirId(), dir.getDirPath());
}
}
}
/**
* Get evictable bytes for this dir, i.e., the total bytes of total evictable blocks
*
* @return evictable bytes for this dir
*/
public long getEvitableBytes() {
long bytes = 0;
for (BlockMeta blockMeta : mDir.getBlocks()) {
long blockId = blockMeta.getBlockId();
if (mManagerView.isBlockEvictable(blockId)) {
bytes += blockMeta.getBlockSize();
}
}
return bytes;
}
/**
* Get a filtered list of block metadata, for blocks that are neither pinned or being blocked.
*
* @return a list of metadata for all evictable blocks
*/
public List<BlockMeta> getEvictableBlocks() {
List<BlockMeta> filteredList = new ArrayList<BlockMeta>();
for (BlockMeta blockMeta : mDir.getBlocks()) {
long blockId = blockMeta.getBlockId();
if (mManagerView.isBlockEvictable(blockId)) {
filteredList.add(blockMeta);
}
}
return filteredList;
}
/**
* Gets the amount of available space of given location in bytes. Master queries the total number
* of bytes available on each tier of the worker, and Evictor/Allocator often cares about the
* bytes at a {@link StorageDir}.
*
* @param location location the check available bytes
* @return available bytes
* @throws IllegalArgumentException when location does not belong to tiered storage
*/
public synchronized long getAvailableBytes(BlockStoreLocation location) {
long spaceAvailable = 0;
if (location.equals(BlockStoreLocation.anyTier())) {
for (StorageTier tier : mTiers) {
spaceAvailable += tier.getAvailableBytes();
}
return spaceAvailable;
}
int tierAlias = location.tierAlias();
StorageTier tier = getTier(tierAlias);
// TODO: This should probably be max of the capacity bytes in the dirs?
if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
return tier.getAvailableBytes();
}
int dirIndex = location.dir();
StorageDir dir = tier.getDir(dirIndex);
return dir.getAvailableBytes();
}
/**
* Moves the metadata of an existing block to another location or throws IOExceptions.
*
* @param blockMeta the meta data of the block to move
* @param newLocation new location of the block
* @return the new block metadata if success, absent otherwise
* @throws IllegalArgumentException when the newLocation is not in the tiered storage
* @throws NotFoundException when the block to move is not found
* @throws AlreadyExistsException when the block to move already exists in the destination
* @throws OutOfSpaceException when destination have no extra space to hold the block to move
*/
public synchronized BlockMeta moveBlockMeta(BlockMeta blockMeta, BlockStoreLocation newLocation)
throws NotFoundException, AlreadyExistsException, OutOfSpaceException {
// If existing location belongs to the target location, simply return the current block meta.
BlockStoreLocation oldLocation = blockMeta.getBlockLocation();
if (oldLocation.belongTo(newLocation)) {
LOG.info("moveBlockMeta: moving {} to {} is a noop", oldLocation, newLocation);
return blockMeta;
}
long blockSize = blockMeta.getBlockSize();
int newTierAlias = newLocation.tierAlias();
StorageTier newTier = getTier(newTierAlias);
StorageDir newDir = null;
if (newLocation.equals(BlockStoreLocation.anyDirInTier(newTierAlias))) {
for (StorageDir dir : newTier.getStorageDirs()) {
if (dir.getAvailableBytes() >= blockSize) {
newDir = dir;
break;
}
}
} else {
StorageDir dir = newTier.getDir(newLocation.dir());
if (dir.getAvailableBytes() >= blockSize) {
newDir = dir;
}
}
if (newDir == null) {
throw new OutOfSpaceException(
"Failed to move BlockMeta: newLocation "
+ newLocation
+ " does not have enough space for "
+ blockSize
+ " bytes");
}
StorageDir oldDir = blockMeta.getParentDir();
oldDir.removeBlockMeta(blockMeta);
BlockMeta newBlockMeta = new BlockMeta(blockMeta.getBlockId(), blockSize, newDir);
newDir.addBlockMeta(newBlockMeta);
return newBlockMeta;
}
/**
* Adds a temp block.
*
* @param tempBlockMeta the meta data of the temp block to add
* @throws OutOfSpaceException when no more space left to hold the block
* @throws AlreadyExistsException when the block already exists
*/
public synchronized void addTempBlockMeta(TempBlockMeta tempBlockMeta)
throws OutOfSpaceException, AlreadyExistsException {
StorageDir dir = tempBlockMeta.getParentDir();
dir.addTempBlockMeta(tempBlockMeta);
}
/**
* Get capacity bytes for this dir
*
* @return capacity bytes for this dir
*/
public long getCapacityBytes() {
return mDir.getCapacityBytes();
}
/**
* Get the index of this Dir
*
* @return index of the dir
*/
public int getDirViewIndex() {
return mDir.getDirIndex();
}
/**
* Create a BlockStoraLocation for this directory view. Redirecting to {@link
* StorageDir#toBlockStoreLocation}
*
* @return BlockStoreLocation created
*/
public BlockStoreLocation toBlockStoreLocation() {
return mDir.toBlockStoreLocation();
}
/**
* Get committed bytes for this dir. This includes all blocks, locked, pinned, committed etc.
*
* @return committed bytes for this dir
*/
public long getCommittedBytes() {
return mDir.getCommittedBytes();
}
/**
* Get available bytes for this dir
*
* @return available bytes for this dir
*/
public long getAvailableBytes() {
return mDir.getAvailableBytes() + mBlocksToMoveOutSize - mBlocksToMoveInSize;
}
/**
* Factory method to create {@link StorageDir}
*
* <p>It will load meta data of existing committed blocks in the dirPath specified. Only files
* with directory depth 1 under dirPath and whose file name can be parsed into {@code long} will
* be considered as existing committed blocks, these files will be preserved, others files or
* directories will be deleted.
*
* @param tier the {@link StorageTier} this dir belongs to
* @param dirIndex the index of this dir in its tier
* @param capacityBytes the initial capacity of this dir, can not be modified later
* @param dirPath filesystem path of this dir for actual storage
* @return the new created StorageDir
* @throws IOException when meta data of existing committed blocks can not be loaded
*/
public static StorageDir newStorageDir(
StorageTier tier, int dirIndex, long capacityBytes, String dirPath) throws IOException {
StorageDir dir = new StorageDir(tier, dirIndex, capacityBytes, dirPath);
dir.initializeMeta();
return dir;
}
/**
* Aborts a temp block.
*
* @param tempBlockMeta the meta data of the temp block to add
* @throws NotFoundException when block can not be found
*/
public synchronized void abortTempBlockMeta(TempBlockMeta tempBlockMeta)
throws NotFoundException {
StorageDir dir = tempBlockMeta.getParentDir();
dir.removeTempBlockMeta(tempBlockMeta);
}
/**
* Modifies the size of a temp block
*
* @param tempBlockMeta the temp block to modify
* @param newSize new size in bytes
* @throws InvalidStateException when newSize is smaller than current size
*/
public synchronized void resizeTempBlockMeta(TempBlockMeta tempBlockMeta, long newSize)
throws InvalidStateException {
StorageDir dir = tempBlockMeta.getParentDir();
dir.resizeTempBlockMeta(tempBlockMeta, newSize);
}
/**
* Remove the metadata of a specific block.
*
* @param block the meta data of the block to remove
* @throws NotFoundException when block is not found
*/
public synchronized void removeBlockMeta(BlockMeta block) throws NotFoundException {
StorageDir dir = block.getParentDir();
dir.removeBlockMeta(block);
}