@Test
public void reserveTest() throws Exception {
// Reserve on top tier
long blockId = 100;
BlockStoreLocation tier0 = BlockStoreLocation.anyDirInTier(StorageLevelAlias.MEM.getValue());
for (int i = 0; i < 3; i++) {
TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0);
}
CommonUtils.sleepMs(
WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS));
BlockStoreMeta storeMeta = mBlockStore.getBlockStoreMeta();
Assert.assertEquals(3 * BLOCK_SIZE, storeMeta.getUsedBytes());
List<Long> usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
Assert.assertEquals(
2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1));
Assert.assertEquals(
BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1));
// Reserve on under tier
for (int i = 0; i < 7; i++) {
TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0);
}
CommonUtils.sleepMs(
WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS));
storeMeta = mBlockStore.getBlockStoreMeta();
Assert.assertEquals(9 * BLOCK_SIZE, storeMeta.getUsedBytes());
usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
Assert.assertEquals(
2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1));
Assert.assertEquals(
7 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1));
}
/**
* * Get sorted fileIds of the files cached in the worker.
*
* @return a sorted fileId list
*/
private List<Integer> getSortedFileIds() {
Set<Integer> fileIds = new HashSet<Integer>();
BlockStoreMeta storeMeta = mBlockDataManager.getStoreMeta();
for (List<Long> blockIds : storeMeta.getBlockList().values()) {
for (long blockId : blockIds) {
int fileId = BlockInfo.computeInodeId(blockId);
fileIds.add(fileId);
}
}
List<Integer> sortedFileIds = new ArrayList<Integer>(fileIds);
Collections.sort(sortedFileIds);
return sortedFileIds;
}
/**
* * Get sorted fileIds of the files cached in the worker.
*
* @return a sorted fileId list
*/
private List<Long> getSortedFileIds() {
Set<Long> fileIds = new HashSet<Long>();
BlockStoreMeta storeMeta = mBlockDataManager.getStoreMeta();
for (List<Long> blockIds : storeMeta.getBlockList().values()) {
for (long blockId : blockIds) {
long fileId =
BlockId.createBlockId(BlockId.getContainerId(blockId), BlockId.getMaxSequenceNumber());
fileIds.add(fileId);
}
}
List<Long> sortedFileIds = new ArrayList<Long>(fileIds);
Collections.sort(sortedFileIds);
return sortedFileIds;
}
/**
* Commits a block to Tachyon managed space. The block must be temporary. The block is persisted
* after {@link BlockStore#commitBlock(long, long)}. The block will not be accessible until {@link
* WorkerBlockMasterClient#commitBlock} succeeds.
*
* @param sessionId The id of the client
* @param blockId The id of the block to commit
* @throws BlockAlreadyExistsException if blockId already exists in committed blocks
* @throws BlockDoesNotExistException if the temporary block cannot be found
* @throws InvalidWorkerStateException if blockId does not belong to sessionId
* @throws IOException if the block cannot be moved from temporary path to committed path
* @throws WorkerOutOfSpaceException if there is no more space left to hold the block
*/
public void commitBlock(long sessionId, long blockId)
throws BlockAlreadyExistsException, BlockDoesNotExistException, InvalidWorkerStateException,
IOException, WorkerOutOfSpaceException {
mBlockStore.commitBlock(sessionId, blockId);
// TODO(calvin): Reconsider how to do this without heavy locking.
// Block successfully committed, update master with new block metadata
Long lockId = mBlockStore.lockBlock(sessionId, blockId);
try {
BlockMeta meta = mBlockStore.getBlockMeta(sessionId, blockId, lockId);
BlockStoreLocation loc = meta.getBlockLocation();
Long length = meta.getBlockSize();
BlockStoreMeta storeMeta = mBlockStore.getBlockStoreMeta();
Long bytesUsedOnTier = storeMeta.getUsedBytesOnTiers().get(loc.tierAlias());
mBlockMasterClient.commitBlock(
WorkerIdRegistry.getWorkerId(), bytesUsedOnTier, loc.tierAlias(), blockId, length);
} catch (IOException ioe) {
throw new IOException("Failed to commit block to master.", ioe);
} finally {
mBlockStore.unlockBlock(lockId);
}
}