/**
* Aborts a temp block.
*
* @param sessionId the id of session
* @param blockId the id of block
* @throws BlockDoesNotExistException if block id can not be found in temporary blocks
* @throws BlockAlreadyExistsException if block id already exists in committed blocks
* @throws InvalidWorkerStateException if block id is not owned by session id
* @throws IOException if I/O errors occur when deleting the block file
*/
private void abortBlockInternal(long sessionId, long blockId)
throws BlockDoesNotExistException, BlockAlreadyExistsException, InvalidWorkerStateException,
IOException {
long lockId = mLockManager.lockBlock(sessionId, blockId, BlockLockType.WRITE);
try {
String path;
TempBlockMeta tempBlockMeta;
mMetadataReadLock.lock();
try {
checkTempBlockOwnedBySession(sessionId, blockId);
tempBlockMeta = mMetaManager.getTempBlockMeta(blockId);
path = tempBlockMeta.getPath();
} finally {
mMetadataReadLock.unlock();
}
// Heavy IO is guarded by block lock but not metadata lock. This may throw IOException.
Files.delete(Paths.get(path));
mMetadataWriteLock.lock();
try {
mMetaManager.abortTempBlockMeta(tempBlockMeta);
} catch (BlockDoesNotExistException e) {
throw Throwables.propagate(e); // We shall never reach here
} finally {
mMetadataWriteLock.unlock();
}
} finally {
mLockManager.unlockBlock(lockId);
}
}
/**
* Checks if a block id is available for a new temp block. This method must be enclosed by {@link
* #mMetadataLock}.
*
* @param blockId the id of block
* @throws BlockAlreadyExistsException if block id already exists
*/
private void checkTempBlockIdAvailable(long blockId) throws BlockAlreadyExistsException {
if (mMetaManager.hasTempBlockMeta(blockId)) {
throw new BlockAlreadyExistsException(ExceptionMessage.TEMP_BLOCK_ID_EXISTS, blockId);
}
if (mMetaManager.hasBlockMeta(blockId)) {
throw new BlockAlreadyExistsException(ExceptionMessage.TEMP_BLOCK_ID_COMMITTED, blockId);
}
}
/**
* Checks if block id is a temporary block and owned by session id. This method must be enclosed
* by {@link #mMetadataLock}.
*
* @param sessionId the id of session
* @param blockId the id of block
* @throws BlockDoesNotExistException if block id can not be found in temporary blocks
* @throws BlockAlreadyExistsException if block id already exists in committed blocks
* @throws InvalidWorkerStateException if block id is not owned by session id
*/
private void checkTempBlockOwnedBySession(long sessionId, long blockId)
throws BlockDoesNotExistException, BlockAlreadyExistsException, InvalidWorkerStateException {
if (mMetaManager.hasBlockMeta(blockId)) {
throw new BlockAlreadyExistsException(ExceptionMessage.TEMP_BLOCK_ID_COMMITTED, blockId);
}
TempBlockMeta tempBlockMeta = mMetaManager.getTempBlockMeta(blockId);
long ownerSessionId = tempBlockMeta.getSessionId();
if (ownerSessionId != sessionId) {
throw new InvalidWorkerStateException(
ExceptionMessage.BLOCK_ID_FOR_DIFFERENT_SESSION, blockId, ownerSessionId, sessionId);
}
}
@Override
public boolean hasBlockMeta(long blockId) {
mMetadataReadLock.lock();
boolean hasBlock = mMetaManager.hasBlockMeta(blockId);
mMetadataReadLock.unlock();
return hasBlock;
}
// TODO(bin): Make this method to return a snapshot.
@Override
public BlockMeta getVolatileBlockMeta(long blockId) throws BlockDoesNotExistException {
mMetadataReadLock.lock();
try {
return mMetaManager.getBlockMeta(blockId);
} finally {
mMetadataReadLock.unlock();
}
}
/**
* Commits a temp block.
*
* @param sessionId the id of session
* @param blockId the id of block
* @return destination location to move the block
* @throws BlockDoesNotExistException if block id can not be found in temporary blocks
* @throws BlockAlreadyExistsException if block id already exists in committed blocks
* @throws InvalidWorkerStateException if block id is not owned by session id
* @throws IOException if I/O errors occur when deleting the block file
*/
private BlockStoreLocation commitBlockInternal(long sessionId, long blockId)
throws BlockAlreadyExistsException, InvalidWorkerStateException, BlockDoesNotExistException,
IOException {
long lockId = mLockManager.lockBlock(sessionId, blockId, BlockLockType.WRITE);
try {
// When committing TempBlockMeta, the final BlockMeta calculates the block size according to
// the actual file size of this TempBlockMeta. Therefore, commitTempBlockMeta must happen
// after moving actual block file to its committed path.
BlockStoreLocation loc;
String srcPath;
String dstPath;
TempBlockMeta tempBlockMeta;
mMetadataReadLock.lock();
try {
checkTempBlockOwnedBySession(sessionId, blockId);
tempBlockMeta = mMetaManager.getTempBlockMeta(blockId);
srcPath = tempBlockMeta.getPath();
dstPath = tempBlockMeta.getCommitPath();
loc = tempBlockMeta.getBlockLocation();
} finally {
mMetadataReadLock.unlock();
}
// Heavy IO is guarded by block lock but not metadata lock. This may throw IOException.
FileUtils.move(srcPath, dstPath);
mMetadataWriteLock.lock();
try {
mMetaManager.commitTempBlockMeta(tempBlockMeta);
} catch (BlockAlreadyExistsException e) {
throw Throwables.propagate(e); // we shall never reach here
} catch (BlockDoesNotExistException e) {
throw Throwables.propagate(e); // we shall never reach here
} catch (WorkerOutOfSpaceException e) {
throw Throwables.propagate(e); // we shall never reach here
} finally {
mMetadataWriteLock.unlock();
}
return loc;
} finally {
mLockManager.unlockBlock(lockId);
}
}
@Override
public BlockMeta getBlockMeta(long sessionId, long blockId, long lockId)
throws BlockDoesNotExistException, InvalidWorkerStateException {
mLockManager.validateLock(sessionId, blockId, lockId);
mMetadataReadLock.lock();
try {
return mMetaManager.getBlockMeta(blockId);
} finally {
mMetadataReadLock.unlock();
}
}
@Override
public BlockStoreMeta getBlockStoreMetaFull() {
mMetadataReadLock.lock();
BlockStoreMeta storeMeta = null;
try {
storeMeta = mMetaManager.getBlockStoreMetaFull();
} finally {
mMetadataReadLock.unlock();
}
return storeMeta;
}
@Override
public BlockReader getBlockReader(long sessionId, long blockId, long lockId)
throws BlockDoesNotExistException, InvalidWorkerStateException, IOException {
mLockManager.validateLock(sessionId, blockId, lockId);
mMetadataReadLock.lock();
try {
BlockMeta blockMeta = mMetaManager.getBlockMeta(blockId);
return new LocalFileBlockReader(blockMeta.getPath());
} finally {
mMetadataReadLock.unlock();
}
}
@Override
public long lockBlock(long sessionId, long blockId) throws BlockDoesNotExistException {
long lockId = mLockManager.lockBlock(sessionId, blockId, BlockLockType.READ);
mMetadataReadLock.lock();
boolean hasBlock = mMetaManager.hasBlockMeta(blockId);
mMetadataReadLock.unlock();
if (hasBlock) {
return lockId;
}
mLockManager.unlockBlock(lockId);
throw new BlockDoesNotExistException(
ExceptionMessage.LOCK_RECORD_NOT_FOUND_FOR_BLOCK_AND_SESSION, blockId, sessionId);
}
/**
* Increases the temp block size only if this temp block's parent dir has enough available space.
*
* @param blockId block Id
* @param additionalBytes additional bytes to request for this block
* @return a pair of boolean and {@link BlockStoreLocation}. The boolean indicates if the
* operation succeeds and the {@link BlockStoreLocation} denotes where to free more space if
* it fails.
* @throws BlockDoesNotExistException if this block is not found
*/
private Pair<Boolean, BlockStoreLocation> requestSpaceInternal(long blockId, long additionalBytes)
throws BlockDoesNotExistException {
// NOTE: a temp block is supposed to be visible for its own writer, unnecessary to acquire
// block lock here since no sharing
mMetadataWriteLock.lock();
try {
TempBlockMeta tempBlockMeta = mMetaManager.getTempBlockMeta(blockId);
if (tempBlockMeta.getParentDir().getAvailableBytes() < additionalBytes) {
return new Pair<Boolean, BlockStoreLocation>(false, tempBlockMeta.getBlockLocation());
}
// Increase the size of this temp block
try {
mMetaManager.resizeTempBlockMeta(
tempBlockMeta, tempBlockMeta.getBlockSize() + additionalBytes);
} catch (InvalidWorkerStateException e) {
throw Throwables.propagate(e); // we shall never reach here
}
return new Pair<Boolean, BlockStoreLocation>(true, null);
} finally {
mMetadataWriteLock.unlock();
}
}
@Override
public void accessBlock(long sessionId, long blockId) throws BlockDoesNotExistException {
mMetadataReadLock.lock();
boolean hasBlock = mMetaManager.hasBlockMeta(blockId);
mMetadataReadLock.unlock();
if (!hasBlock) {
throw new BlockDoesNotExistException(ExceptionMessage.NO_BLOCK_ID_FOUND, blockId);
}
synchronized (mBlockStoreEventListeners) {
for (BlockStoreEventListener listener : mBlockStoreEventListeners) {
listener.onAccessBlock(sessionId, blockId);
}
}
}
@Override
public BlockWriter getBlockWriter(long sessionId, long blockId)
throws BlockDoesNotExistException, IOException {
// NOTE: a temp block is supposed to only be visible by its own writer, unnecessary to acquire
// block lock here since no sharing
// TODO(bin): Handle the case where multiple writers compete for the same block.
mMetadataReadLock.lock();
try {
TempBlockMeta tempBlockMeta = mMetaManager.getTempBlockMeta(blockId);
return new LocalFileBlockWriter(tempBlockMeta.getPath());
} finally {
mMetadataReadLock.unlock();
}
}
/**
* Creates a temp block meta only if allocator finds available space. This method will not trigger
* any eviction.
*
* @param sessionId session Id
* @param blockId block Id
* @param location location to create the block
* @param initialBlockSize initial block size in bytes
* @param newBlock true if this temp block is created for a new block
* @return a temp block created if successful, or null if allocation failed (instead of throwing
* {@link WorkerOutOfSpaceException} because allocation failure could be an expected case)
* @throws BlockAlreadyExistsException if there is already a block with the same block id
*/
private TempBlockMeta createBlockMetaInternal(
long sessionId,
long blockId,
BlockStoreLocation location,
long initialBlockSize,
boolean newBlock)
throws BlockAlreadyExistsException {
// NOTE: a temp block is supposed to be visible for its own writer, unnecessary to acquire
// block lock here since no sharing
mMetadataWriteLock.lock();
try {
if (newBlock) {
checkTempBlockIdAvailable(blockId);
}
StorageDirView dirView =
mAllocator.allocateBlockWithView(sessionId, initialBlockSize, location, getUpdatedView());
if (dirView == null) {
// Allocator fails to find a proper place for this new block.
return null;
}
// TODO(carson): Add tempBlock to corresponding storageDir and remove the use of
// StorageDirView.createTempBlockMeta.
TempBlockMeta tempBlock = dirView.createTempBlockMeta(sessionId, blockId, initialBlockSize);
try {
// Add allocated temp block to metadata manager. This should never fail if allocator
// correctly assigns a StorageDir.
mMetaManager.addTempBlockMeta(tempBlock);
} catch (WorkerOutOfSpaceException e) {
// If we reach here, allocator is not working properly
LOG.error(
"Unexpected failure: {} bytes allocated at {} by allocator, "
+ "but addTempBlockMeta failed",
initialBlockSize,
location);
throw Throwables.propagate(e);
} catch (BlockAlreadyExistsException e) {
// If we reach here, allocator is not working properly
LOG.error(
"Unexpected failure: {} bytes allocated at {} by allocator, "
+ "but addTempBlockMeta failed",
initialBlockSize,
location);
throw Throwables.propagate(e);
}
return tempBlock;
} finally {
mMetadataWriteLock.unlock();
}
}
/**
* Removes a block.
*
* @param sessionId session Id
* @param blockId block Id
* @param location the source location of the block
* @throws InvalidWorkerStateException if the block to remove is a temp block
* @throws BlockDoesNotExistException if this block can not be found
* @throws IOException if I/O errors occur when removing this block file
*/
private void removeBlockInternal(long sessionId, long blockId, BlockStoreLocation location)
throws InvalidWorkerStateException, BlockDoesNotExistException, IOException {
long lockId = mLockManager.lockBlock(sessionId, blockId, BlockLockType.WRITE);
try {
String filePath;
BlockMeta blockMeta;
mMetadataReadLock.lock();
try {
if (mMetaManager.hasTempBlockMeta(blockId)) {
throw new InvalidWorkerStateException(ExceptionMessage.REMOVE_UNCOMMITTED_BLOCK, blockId);
}
blockMeta = mMetaManager.getBlockMeta(blockId);
filePath = blockMeta.getPath();
} finally {
mMetadataReadLock.unlock();
}
if (!blockMeta.getBlockLocation().belongsTo(location)) {
throw new BlockDoesNotExistException(
ExceptionMessage.BLOCK_NOT_FOUND_AT_LOCATION, blockId, location);
}
// Heavy IO is guarded by block lock but not metadata lock. This may throw IOException.
Files.delete(Paths.get(filePath));
mMetadataWriteLock.lock();
try {
mMetaManager.removeBlockMeta(blockMeta);
} catch (BlockDoesNotExistException e) {
throw Throwables.propagate(e); // we shall never reach here
} finally {
mMetadataWriteLock.unlock();
}
} finally {
mLockManager.unlockBlock(lockId);
}
}
@Override
public void cleanupSession(long sessionId) {
// Release all locks the session is holding.
mLockManager.cleanupSession(sessionId);
// Collect a list of temp blocks the given session owns and abort all of them with best effort
List<TempBlockMeta> tempBlocksToRemove;
mMetadataReadLock.lock();
try {
tempBlocksToRemove = mMetaManager.getSessionTempBlocks(sessionId);
} finally {
mMetadataReadLock.unlock();
}
for (TempBlockMeta tempBlockMeta : tempBlocksToRemove) {
try {
abortBlockInternal(sessionId, tempBlockMeta.getBlockId());
} catch (Exception e) {
LOG.error(
"Failed to cleanup tempBlock {} due to {}", tempBlockMeta.getBlockId(), e.getMessage());
}
}
}
/** Creates a new instance of {@link TieredBlockStore}. */
public TieredBlockStore() {
mConfiguration = WorkerContext.getConf();
mMetaManager = BlockMetadataManager.createBlockMetadataManager();
mLockManager = new BlockLockManager();
BlockMetadataManagerView initManagerView =
new BlockMetadataManagerView(
mMetaManager, Collections.<Long>emptySet(), Collections.<Long>emptySet());
mAllocator = Allocator.Factory.create(mConfiguration, initManagerView);
if (mAllocator instanceof BlockStoreEventListener) {
registerBlockStoreEventListener((BlockStoreEventListener) mAllocator);
}
initManagerView =
new BlockMetadataManagerView(
mMetaManager, Collections.<Long>emptySet(), Collections.<Long>emptySet());
mEvictor = Evictor.Factory.create(mConfiguration, initManagerView, mAllocator);
if (mEvictor instanceof BlockStoreEventListener) {
registerBlockStoreEventListener((BlockStoreEventListener) mEvictor);
}
mStorageTierAssoc = new WorkerStorageTierAssoc(mConfiguration);
}
/**
* Moves a block to new location only if allocator finds available space in newLocation. This
* method will not trigger any eviction. Returns {@link MoveBlockResult}.
*
* @param sessionId session Id
* @param blockId block Id
* @param oldLocation the source location of the block
* @param newLocation new location to move this block
* @return the resulting information about the move operation
* @throws BlockDoesNotExistException if block is not found
* @throws BlockAlreadyExistsException if a block with same Id already exists in new location
* @throws InvalidWorkerStateException if the block to move is a temp block
* @throws IOException if I/O errors occur when moving block file
*/
private MoveBlockResult moveBlockInternal(
long sessionId, long blockId, BlockStoreLocation oldLocation, BlockStoreLocation newLocation)
throws BlockDoesNotExistException, BlockAlreadyExistsException, InvalidWorkerStateException,
IOException {
long lockId = mLockManager.lockBlock(sessionId, blockId, BlockLockType.WRITE);
try {
long blockSize;
String srcFilePath;
String dstFilePath;
BlockMeta srcBlockMeta;
BlockStoreLocation srcLocation;
BlockStoreLocation dstLocation;
mMetadataReadLock.lock();
try {
if (mMetaManager.hasTempBlockMeta(blockId)) {
throw new InvalidWorkerStateException(ExceptionMessage.MOVE_UNCOMMITTED_BLOCK, blockId);
}
srcBlockMeta = mMetaManager.getBlockMeta(blockId);
srcLocation = srcBlockMeta.getBlockLocation();
srcFilePath = srcBlockMeta.getPath();
blockSize = srcBlockMeta.getBlockSize();
} finally {
mMetadataReadLock.unlock();
}
if (!srcLocation.belongsTo(oldLocation)) {
throw new BlockDoesNotExistException(
ExceptionMessage.BLOCK_NOT_FOUND_AT_LOCATION, blockId, oldLocation);
}
TempBlockMeta dstTempBlock =
createBlockMetaInternal(sessionId, blockId, newLocation, blockSize, false);
if (dstTempBlock == null) {
return new MoveBlockResult(false, blockSize, null, null);
}
// When `newLocation` is some specific location, the `newLocation` and the `dstLocation` are
// just the same; while for `newLocation` with a wildcard significance, the `dstLocation`
// is a specific one with specific tier and dir which belongs to newLocation.
dstLocation = dstTempBlock.getBlockLocation();
// When the dstLocation belongs to srcLocation, simply abort the tempBlockMeta just created
// internally from the newLocation and return success with specific block location.
if (dstLocation.belongsTo(srcLocation)) {
mMetaManager.abortTempBlockMeta(dstTempBlock);
return new MoveBlockResult(true, blockSize, srcLocation, dstLocation);
}
dstFilePath = dstTempBlock.getCommitPath();
// Heavy IO is guarded by block lock but not metadata lock. This may throw IOException.
FileUtils.move(srcFilePath, dstFilePath);
mMetadataWriteLock.lock();
try {
// If this metadata update fails, we panic for now.
// TODO(bin): Implement rollback scheme to recover from IO failures.
mMetaManager.moveBlockMeta(srcBlockMeta, dstTempBlock);
} catch (BlockAlreadyExistsException e) {
throw Throwables.propagate(e); // we shall never reach here
} catch (BlockDoesNotExistException e) {
throw Throwables.propagate(e); // we shall never reach here
} catch (WorkerOutOfSpaceException e) {
// Only possible if session id gets cleaned between createBlockMetaInternal and
// moveBlockMeta.
throw Throwables.propagate(e);
} finally {
mMetadataWriteLock.unlock();
}
return new MoveBlockResult(true, blockSize, srcLocation, dstLocation);
} finally {
mLockManager.unlockBlock(lockId);
}
}