/** @return the remaining capacity (in bytes) */ public long getAvailableBytes() { long availableBytes = 0; for (StorageDir dir : mDirs) { availableBytes += dir.getAvailableBytes(); } return availableBytes; }
/** * Cleans up the meta data of the given temp block ids * * @param userId the ID of the client associated with the temp blocks * @param tempBlockIds the list of temporary block ids to be cleaned up, non temporary block ids * will be ignored. */ public synchronized void cleanupUserTempBlocks(long userId, List<Long> tempBlockIds) { for (StorageTier tier : mTiers) { for (StorageDir dir : tier.getStorageDirs()) { dir.cleanupUserTempBlocks(userId, tempBlockIds); } } }
/** * Commits a temp block. * * @param tempBlockMeta the meta data of the temp block to commit * @throws OutOfSpaceException when no more space left to hold the block * @throws AlreadyExistsException when the block already exists in committed blocks * @throws NotFoundException when temp block can not be found */ public synchronized void commitTempBlockMeta(TempBlockMeta tempBlockMeta) throws OutOfSpaceException, AlreadyExistsException, NotFoundException { BlockMeta block = new BlockMeta(Preconditions.checkNotNull(tempBlockMeta)); StorageDir dir = tempBlockMeta.getParentDir(); dir.removeTempBlockMeta(tempBlockMeta); dir.addBlockMeta(block); }
/** * Gets all the temporary blocks associated with a user, empty list is returned if the user has no * temporary blocks. * * @param userId the ID of the user * @return A list of temp blocks associated with the user */ public synchronized List<TempBlockMeta> getUserTempBlocks(long userId) { List<TempBlockMeta> userTempBlocks = new ArrayList<TempBlockMeta>(); for (StorageTier tier : mTiers) { for (StorageDir dir : tier.getStorageDirs()) { userTempBlocks.addAll(dir.getUserTempBlocks(userId)); } } return userTempBlocks; }
/** * Checks if the storage has a given temp block. * * @param blockId the temp block ID * @return true if the block is contained, false otherwise */ public synchronized boolean hasTempBlockMeta(long blockId) { for (StorageTier tier : mTiers) { for (StorageDir dir : tier.getStorageDirs()) { if (dir.hasTempBlockMeta(blockId)) { return true; } } } return false; }
/** * Gets the metadata of a block given its blockId. * * @param blockId the block ID * @return metadata of the block * @throws NotFoundException if no BlockMeta for this blockId is found */ public synchronized BlockMeta getBlockMeta(long blockId) throws NotFoundException { for (StorageTier tier : mTiers) { for (StorageDir dir : tier.getStorageDirs()) { if (dir.hasBlockMeta(blockId)) { return dir.getBlockMeta(blockId); } } } throw new NotFoundException("Failed to get BlockMeta: blockId " + blockId + " not found"); }
private void initStorageTier() throws AlreadyExistsException, IOException, OutOfSpaceException { String workerDataFolder = WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER, Constants.DEFAULT_DATA_FOLDER); String tierDirPathConf = String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, mTierLevel); String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf, "/mnt/ramdisk").split(","); // Add the worker data folder path after each storage directory, the final path will be like // /mnt/ramdisk/tachyonworker for (int i = 0; i < dirPaths.length; i++) { dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder); } String tierDirCapacityConf = String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel); String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf, "0").split(","); mDirs = new ArrayList<StorageDir>(dirPaths.length); long totalCapacity = 0; for (int i = 0; i < dirPaths.length; i++) { int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i; long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]); totalCapacity += capacity; mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i])); } mCapacityBytes = totalCapacity; }
public BlockStoreMeta(BlockMetadataManager manager) { Preconditions.checkNotNull(manager); for (StorageTier tier : manager.getTiers()) { int aliasIndex = tier.getTierAlias() - 1; mCapacityBytesOnTiers.set( aliasIndex, mCapacityBytesOnTiers.get(aliasIndex) + tier.getCapacityBytes()); mUsedBytesOnTiers.set( aliasIndex, mUsedBytesOnTiers.get(aliasIndex) + (tier.getCapacityBytes() - tier.getAvailableBytes())); for (StorageDir dir : tier.getStorageDirs()) { mBlockIdsOnDirs.put(dir.getStorageDirId(), dir.getBlockIds()); mCapacityBytesOnDirs.put(dir.getStorageDirId(), dir.getCapacityBytes()); mUsedBytesOnDirs.put( dir.getStorageDirId(), dir.getCapacityBytes() - dir.getAvailableBytes()); mDirPaths.put(dir.getStorageDirId(), dir.getDirPath()); } } }
/** * Get evictable bytes for this dir, i.e., the total bytes of total evictable blocks * * @return evictable bytes for this dir */ public long getEvitableBytes() { long bytes = 0; for (BlockMeta blockMeta : mDir.getBlocks()) { long blockId = blockMeta.getBlockId(); if (mManagerView.isBlockEvictable(blockId)) { bytes += blockMeta.getBlockSize(); } } return bytes; }
/** * Get a filtered list of block metadata, for blocks that are neither pinned or being blocked. * * @return a list of metadata for all evictable blocks */ public List<BlockMeta> getEvictableBlocks() { List<BlockMeta> filteredList = new ArrayList<BlockMeta>(); for (BlockMeta blockMeta : mDir.getBlocks()) { long blockId = blockMeta.getBlockId(); if (mManagerView.isBlockEvictable(blockId)) { filteredList.add(blockMeta); } } return filteredList; }
/** * Gets the amount of available space of given location in bytes. Master queries the total number * of bytes available on each tier of the worker, and Evictor/Allocator often cares about the * bytes at a {@link StorageDir}. * * @param location location the check available bytes * @return available bytes * @throws IllegalArgumentException when location does not belong to tiered storage */ public synchronized long getAvailableBytes(BlockStoreLocation location) { long spaceAvailable = 0; if (location.equals(BlockStoreLocation.anyTier())) { for (StorageTier tier : mTiers) { spaceAvailable += tier.getAvailableBytes(); } return spaceAvailable; } int tierAlias = location.tierAlias(); StorageTier tier = getTier(tierAlias); // TODO: This should probably be max of the capacity bytes in the dirs? if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) { return tier.getAvailableBytes(); } int dirIndex = location.dir(); StorageDir dir = tier.getDir(dirIndex); return dir.getAvailableBytes(); }
/** * Moves the metadata of an existing block to another location or throws IOExceptions. * * @param blockMeta the meta data of the block to move * @param newLocation new location of the block * @return the new block metadata if success, absent otherwise * @throws IllegalArgumentException when the newLocation is not in the tiered storage * @throws NotFoundException when the block to move is not found * @throws AlreadyExistsException when the block to move already exists in the destination * @throws OutOfSpaceException when destination have no extra space to hold the block to move */ public synchronized BlockMeta moveBlockMeta(BlockMeta blockMeta, BlockStoreLocation newLocation) throws NotFoundException, AlreadyExistsException, OutOfSpaceException { // If existing location belongs to the target location, simply return the current block meta. BlockStoreLocation oldLocation = blockMeta.getBlockLocation(); if (oldLocation.belongTo(newLocation)) { LOG.info("moveBlockMeta: moving {} to {} is a noop", oldLocation, newLocation); return blockMeta; } long blockSize = blockMeta.getBlockSize(); int newTierAlias = newLocation.tierAlias(); StorageTier newTier = getTier(newTierAlias); StorageDir newDir = null; if (newLocation.equals(BlockStoreLocation.anyDirInTier(newTierAlias))) { for (StorageDir dir : newTier.getStorageDirs()) { if (dir.getAvailableBytes() >= blockSize) { newDir = dir; break; } } } else { StorageDir dir = newTier.getDir(newLocation.dir()); if (dir.getAvailableBytes() >= blockSize) { newDir = dir; } } if (newDir == null) { throw new OutOfSpaceException( "Failed to move BlockMeta: newLocation " + newLocation + " does not have enough space for " + blockSize + " bytes"); } StorageDir oldDir = blockMeta.getParentDir(); oldDir.removeBlockMeta(blockMeta); BlockMeta newBlockMeta = new BlockMeta(blockMeta.getBlockId(), blockSize, newDir); newDir.addBlockMeta(newBlockMeta); return newBlockMeta; }
/** * Adds a temp block. * * @param tempBlockMeta the meta data of the temp block to add * @throws OutOfSpaceException when no more space left to hold the block * @throws AlreadyExistsException when the block already exists */ public synchronized void addTempBlockMeta(TempBlockMeta tempBlockMeta) throws OutOfSpaceException, AlreadyExistsException { StorageDir dir = tempBlockMeta.getParentDir(); dir.addTempBlockMeta(tempBlockMeta); }
/** * Get capacity bytes for this dir * * @return capacity bytes for this dir */ public long getCapacityBytes() { return mDir.getCapacityBytes(); }
/** * Get the index of this Dir * * @return index of the dir */ public int getDirViewIndex() { return mDir.getDirIndex(); }
/** * Create a BlockStoraLocation for this directory view. Redirecting to {@link * StorageDir#toBlockStoreLocation} * * @return BlockStoreLocation created */ public BlockStoreLocation toBlockStoreLocation() { return mDir.toBlockStoreLocation(); }
/** * Get committed bytes for this dir. This includes all blocks, locked, pinned, committed etc. * * @return committed bytes for this dir */ public long getCommittedBytes() { return mDir.getCommittedBytes(); }
/** * Get available bytes for this dir * * @return available bytes for this dir */ public long getAvailableBytes() { return mDir.getAvailableBytes() + mBlocksToMoveOutSize - mBlocksToMoveInSize; }
/** * Factory method to create {@link StorageDir} * * <p>It will load meta data of existing committed blocks in the dirPath specified. Only files * with directory depth 1 under dirPath and whose file name can be parsed into {@code long} will * be considered as existing committed blocks, these files will be preserved, others files or * directories will be deleted. * * @param tier the {@link StorageTier} this dir belongs to * @param dirIndex the index of this dir in its tier * @param capacityBytes the initial capacity of this dir, can not be modified later * @param dirPath filesystem path of this dir for actual storage * @return the new created StorageDir * @throws IOException when meta data of existing committed blocks can not be loaded */ public static StorageDir newStorageDir( StorageTier tier, int dirIndex, long capacityBytes, String dirPath) throws IOException { StorageDir dir = new StorageDir(tier, dirIndex, capacityBytes, dirPath); dir.initializeMeta(); return dir; }
/** * Aborts a temp block. * * @param tempBlockMeta the meta data of the temp block to add * @throws NotFoundException when block can not be found */ public synchronized void abortTempBlockMeta(TempBlockMeta tempBlockMeta) throws NotFoundException { StorageDir dir = tempBlockMeta.getParentDir(); dir.removeTempBlockMeta(tempBlockMeta); }
/** * Modifies the size of a temp block * * @param tempBlockMeta the temp block to modify * @param newSize new size in bytes * @throws InvalidStateException when newSize is smaller than current size */ public synchronized void resizeTempBlockMeta(TempBlockMeta tempBlockMeta, long newSize) throws InvalidStateException { StorageDir dir = tempBlockMeta.getParentDir(); dir.resizeTempBlockMeta(tempBlockMeta, newSize); }
/** * Remove the metadata of a specific block. * * @param block the meta data of the block to remove * @throws NotFoundException when block is not found */ public synchronized void removeBlockMeta(BlockMeta block) throws NotFoundException { StorageDir dir = block.getParentDir(); dir.removeBlockMeta(block); }