/**
* Cleans up the meta data of the given temp block ids
*
* @param userId the ID of the client associated with the temp blocks
* @param tempBlockIds the list of temporary block ids to be cleaned up, non temporary block ids
* will be ignored.
*/
public synchronized void cleanupUserTempBlocks(long userId, List<Long> tempBlockIds) {
for (StorageTier tier : mTiers) {
for (StorageDir dir : tier.getStorageDirs()) {
dir.cleanupUserTempBlocks(userId, tempBlockIds);
}
}
}
/**
* Gets all the temporary blocks associated with a user, empty list is returned if the user has no
* temporary blocks.
*
* @param userId the ID of the user
* @return A list of temp blocks associated with the user
*/
public synchronized List<TempBlockMeta> getUserTempBlocks(long userId) {
List<TempBlockMeta> userTempBlocks = new ArrayList<TempBlockMeta>();
for (StorageTier tier : mTiers) {
for (StorageDir dir : tier.getStorageDirs()) {
userTempBlocks.addAll(dir.getUserTempBlocks(userId));
}
}
return userTempBlocks;
}
/**
* Checks if the storage has a given temp block.
*
* @param blockId the temp block ID
* @return true if the block is contained, false otherwise
*/
public synchronized boolean hasTempBlockMeta(long blockId) {
for (StorageTier tier : mTiers) {
for (StorageDir dir : tier.getStorageDirs()) {
if (dir.hasTempBlockMeta(blockId)) {
return true;
}
}
}
return false;
}
/**
* Gets the metadata of a block given its blockId.
*
* @param blockId the block ID
* @return metadata of the block
* @throws NotFoundException if no BlockMeta for this blockId is found
*/
public synchronized BlockMeta getBlockMeta(long blockId) throws NotFoundException {
for (StorageTier tier : mTiers) {
for (StorageDir dir : tier.getStorageDirs()) {
if (dir.hasBlockMeta(blockId)) {
return dir.getBlockMeta(blockId);
}
}
}
throw new NotFoundException("Failed to get BlockMeta: blockId " + blockId + " not found");
}
private void initBlockMetadataManager(TachyonConf tachyonConf)
throws AlreadyExistsException, IOException, OutOfSpaceException {
// Initialize storage tiers
int totalTiers = tachyonConf.getInt(Constants.WORKER_MAX_TIERED_STORAGE_LEVEL, 1);
mAliasToTiers = new HashMap<Integer, StorageTier>(totalTiers);
mTiers = new ArrayList<StorageTier>(totalTiers);
for (int level = 0; level < totalTiers; level++) {
StorageTier tier = StorageTier.newStorageTier(tachyonConf, level);
mTiers.add(tier);
mAliasToTiers.put(tier.getTierAlias(), tier);
}
}
/**
* Moves the metadata of an existing block to another location or throws IOExceptions.
*
* @param blockMeta the meta data of the block to move
* @param newLocation new location of the block
* @return the new block metadata if success, absent otherwise
* @throws IllegalArgumentException when the newLocation is not in the tiered storage
* @throws NotFoundException when the block to move is not found
* @throws AlreadyExistsException when the block to move already exists in the destination
* @throws OutOfSpaceException when destination have no extra space to hold the block to move
*/
public synchronized BlockMeta moveBlockMeta(BlockMeta blockMeta, BlockStoreLocation newLocation)
throws NotFoundException, AlreadyExistsException, OutOfSpaceException {
// If existing location belongs to the target location, simply return the current block meta.
BlockStoreLocation oldLocation = blockMeta.getBlockLocation();
if (oldLocation.belongTo(newLocation)) {
LOG.info("moveBlockMeta: moving {} to {} is a noop", oldLocation, newLocation);
return blockMeta;
}
long blockSize = blockMeta.getBlockSize();
int newTierAlias = newLocation.tierAlias();
StorageTier newTier = getTier(newTierAlias);
StorageDir newDir = null;
if (newLocation.equals(BlockStoreLocation.anyDirInTier(newTierAlias))) {
for (StorageDir dir : newTier.getStorageDirs()) {
if (dir.getAvailableBytes() >= blockSize) {
newDir = dir;
break;
}
}
} else {
StorageDir dir = newTier.getDir(newLocation.dir());
if (dir.getAvailableBytes() >= blockSize) {
newDir = dir;
}
}
if (newDir == null) {
throw new OutOfSpaceException(
"Failed to move BlockMeta: newLocation "
+ newLocation
+ " does not have enough space for "
+ blockSize
+ " bytes");
}
StorageDir oldDir = blockMeta.getParentDir();
oldDir.removeBlockMeta(blockMeta);
BlockMeta newBlockMeta = new BlockMeta(blockMeta.getBlockId(), blockSize, newDir);
newDir.addBlockMeta(newBlockMeta);
return newBlockMeta;
}
/**
* Gets the amount of available space of given location in bytes. Master queries the total number
* of bytes available on each tier of the worker, and Evictor/Allocator often cares about the
* bytes at a {@link StorageDir}.
*
* @param location location the check available bytes
* @return available bytes
* @throws IllegalArgumentException when location does not belong to tiered storage
*/
public synchronized long getAvailableBytes(BlockStoreLocation location) {
long spaceAvailable = 0;
if (location.equals(BlockStoreLocation.anyTier())) {
for (StorageTier tier : mTiers) {
spaceAvailable += tier.getAvailableBytes();
}
return spaceAvailable;
}
int tierAlias = location.tierAlias();
StorageTier tier = getTier(tierAlias);
// TODO: This should probably be max of the capacity bytes in the dirs?
if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
return tier.getAvailableBytes();
}
int dirIndex = location.dir();
StorageDir dir = tier.getDir(dirIndex);
return dir.getAvailableBytes();
}
public BlockStoreMeta(BlockMetadataManager manager) {
Preconditions.checkNotNull(manager);
for (StorageTier tier : manager.getTiers()) {
int aliasIndex = tier.getTierAlias() - 1;
mCapacityBytesOnTiers.set(
aliasIndex, mCapacityBytesOnTiers.get(aliasIndex) + tier.getCapacityBytes());
mUsedBytesOnTiers.set(
aliasIndex,
mUsedBytesOnTiers.get(aliasIndex) + (tier.getCapacityBytes() - tier.getAvailableBytes()));
for (StorageDir dir : tier.getStorageDirs()) {
mBlockIdsOnDirs.put(dir.getStorageDirId(), dir.getBlockIds());
mCapacityBytesOnDirs.put(dir.getStorageDirId(), dir.getCapacityBytes());
mUsedBytesOnDirs.put(
dir.getStorageDirId(), dir.getCapacityBytes() - dir.getAvailableBytes());
mDirPaths.put(dir.getStorageDirId(), dir.getDirPath());
}
}
}
/**
* @param tierLevel the tier level
* @return a new storage tier
* @throws AlreadyExistsException if the tier already exists
* @throws IOException if an I/O error occurred
* @throws OutOfSpaceException if there is not enough space available
*/
public static StorageTier newStorageTier(int tierLevel)
throws AlreadyExistsException, IOException, OutOfSpaceException {
StorageTier ret = new StorageTier(tierLevel);
ret.initStorageTier();
return ret;
}
public BlockStoreLocation toBlockStoreLocation() {
return new BlockStoreLocation(mTier.getTierAlias(), mTier.getTierLevel(), mDirIndex);
}
// TODO: deprecate this method.
public long getStorageDirId() {
int level = mTier.getTierLevel();
int storageLevelAliasValue = mTier.getTierAlias();
return StorageDirId.getStorageDirId(level, storageLevelAliasValue, mDirIndex);
}