/**
* Updates the worker and block metadata for blocks removed from a worker.
*
* @param workerInfo The worker metadata object
* @param removedBlockIds A list of block ids removed from the worker
*/
@GuardedBy("workerInfo")
private void processWorkerRemovedBlocks(
MasterWorkerInfo workerInfo, Collection<Long> removedBlockIds) {
for (long removedBlockId : removedBlockIds) {
MasterBlockInfo block = mBlocks.get(removedBlockId);
// TODO(calvin): Investigate if this branching logic can be simplified.
if (block == null) {
// LOG.warn("Worker {} informs the removed block {}, but block metadata does not exist"
// + " on Master!", workerInfo.getId(), removedBlockId);
// TODO(pfxuan): [ALLUXIO-1804] should find a better way to handle the removed blocks.
// Ideally, the delete/free I/O flow should never reach this point. Because Master may
// update the block metadata only after receiving the acknowledgement from Workers.
workerInfo.removeBlock(removedBlockId);
// Continue to remove the remaining blocks.
continue;
}
synchronized (block) {
LOG.info("Block {} is removed on worker {}.", removedBlockId, workerInfo.getId());
workerInfo.removeBlock(block.getBlockId());
block.removeWorker(workerInfo.getId());
if (block.getNumLocations() == 0) {
mLostBlocks.add(removedBlockId);
}
}
}
}
/**
* @param blockId the block id to get information for
* @return the {@link BlockInfo} for the given block id
* @throws BlockInfoException if the block info is not found
*/
public BlockInfo getBlockInfo(long blockId) throws BlockInfoException {
MasterBlockInfo block = mBlocks.get(blockId);
if (block == null) {
throw new BlockInfoException(ExceptionMessage.BLOCK_META_NOT_FOUND, blockId);
}
synchronized (block) {
return generateBlockInfo(block);
}
}
/**
* Marks a block as committed, but without a worker location. This means the block is only in ufs.
*
* @param blockId the id of the block to commit
* @param length the length of the block
*/
public void commitBlockInUFS(long blockId, long length) {
LOG.debug("Commit block in ufs. blockId: {}, length: {}", blockId, length);
if (mBlocks.get(blockId) != null) {
// Block metadata already exists, so do not need to create a new one.
return;
}
// The block has not been committed previously, so add the metadata to commit the block.
MasterBlockInfo block = new MasterBlockInfo(blockId, length);
long counter = AsyncJournalWriter.INVALID_FLUSH_COUNTER;
synchronized (block) {
if (mBlocks.putIfAbsent(blockId, block) == null) {
// Successfully added the new block metadata. Append a journal entry for the new metadata.
BlockInfoEntry blockInfo =
BlockInfoEntry.newBuilder().setBlockId(blockId).setLength(length).build();
counter = appendJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
}
}
waitForJournalFlush(counter);
}
@Override
public void streamToJournalCheckpoint(JournalOutputStream outputStream) throws IOException {
outputStream.writeEntry(getContainerIdJournalEntry());
for (MasterBlockInfo blockInfo : mBlocks.values()) {
BlockInfoEntry blockInfoEntry =
BlockInfoEntry.newBuilder()
.setBlockId(blockInfo.getBlockId())
.setLength(blockInfo.getLength())
.build();
outputStream.writeEntry(JournalEntry.newBuilder().setBlockInfo(blockInfoEntry).build());
}
}
/**
* Retrieves information for the given list of block ids.
*
* @param blockIds A list of block ids to retrieve the information for
* @return A list of {@link BlockInfo} objects corresponding to the input list of block ids. The
* list is in the same order as the input list
*/
public List<BlockInfo> getBlockInfoList(List<Long> blockIds) {
List<BlockInfo> ret = new ArrayList<>(blockIds.size());
for (long blockId : blockIds) {
MasterBlockInfo block = mBlocks.get(blockId);
if (block == null) {
continue;
}
synchronized (block) {
ret.add(generateBlockInfo(block));
}
}
return ret;
}
@Override
public void processJournalEntry(JournalEntry entry) throws IOException {
Message innerEntry = JournalProtoUtils.unwrap(entry);
// TODO(gene): A better way to process entries besides a huge switch?
if (innerEntry instanceof BlockContainerIdGeneratorEntry) {
mJournaledNextContainerId =
((BlockContainerIdGeneratorEntry) innerEntry).getNextContainerId();
mBlockContainerIdGenerator.setNextContainerId((mJournaledNextContainerId));
} else if (innerEntry instanceof BlockInfoEntry) {
BlockInfoEntry blockInfoEntry = (BlockInfoEntry) innerEntry;
if (mBlocks.containsKey(blockInfoEntry.getBlockId())) {
// Update the existing block info.
MasterBlockInfo blockInfo = mBlocks.get(blockInfoEntry.getBlockId());
blockInfo.updateLength(blockInfoEntry.getLength());
} else {
mBlocks.put(
blockInfoEntry.getBlockId(),
new MasterBlockInfo(blockInfoEntry.getBlockId(), blockInfoEntry.getLength()));
}
} else {
throw new IOException(ExceptionMessage.UNEXPECTED_JOURNAL_ENTRY.getMessage(entry));
}
}
/**
* Removes blocks from workers.
*
* @param blockIds a list of block ids to remove from Alluxio space
* @param delete whether to delete blocks' metadata in Master
*/
public void removeBlocks(List<Long> blockIds, boolean delete) {
for (long blockId : blockIds) {
MasterBlockInfo block = mBlocks.get(blockId);
if (block == null) {
continue;
}
HashSet<Long> workerIds = new HashSet<>();
synchronized (block) {
// Technically, 'block' should be confirmed to still be in the data structure. A
// concurrent removeBlock call can remove it. However, we are intentionally ignoring this
// race, since deleting the same block again is a noop.
workerIds.addAll(block.getWorkers());
// Two cases here:
// 1) For delete: delete the block metadata.
// 2) For free: keep the block metadata. mLostBlocks will be changed in
// processWorkerRemovedBlocks
if (delete) {
// Make sure blockId is removed from mLostBlocks when the block metadata is deleted.
// Otherwise blockId in mLostBlock can be dangling index if the metadata is gone.
mLostBlocks.remove(blockId);
mBlocks.remove(blockId);
}
}
// Outside of locking the block. This does not have to be synchronized with the block
// metadata, since it is essentially an asynchronous signal to the worker to remove the block.
for (long workerId : workerIds) {
MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
if (worker != null) {
synchronized (worker) {
worker.updateToRemovedBlock(true, blockId);
}
}
}
}
}
/**
* Updates the worker and block metadata for blocks added to a worker.
*
* @param workerInfo The worker metadata object
* @param addedBlockIds A mapping from storage tier alias to a list of block ids added
*/
@GuardedBy("workerInfo")
private void processWorkerAddedBlocks(
MasterWorkerInfo workerInfo, Map<String, List<Long>> addedBlockIds) {
for (Map.Entry<String, List<Long>> entry : addedBlockIds.entrySet()) {
for (long blockId : entry.getValue()) {
MasterBlockInfo block = mBlocks.get(blockId);
if (block != null) {
synchronized (block) {
workerInfo.addBlock(blockId);
block.addWorker(workerInfo.getId(), entry.getKey());
mLostBlocks.remove(blockId);
}
} else {
LOG.warn("Failed to register workerId: {} to blockId: {}", workerInfo.getId(), blockId);
}
}
}
}
// TODO(binfan): check the logic is correct or not when commitBlock is a retry
public void commitBlock(
long workerId, long usedBytesOnTier, String tierAlias, long blockId, long length)
throws NoWorkerException {
LOG.debug(
"Commit block from workerId: {}, usedBytesOnTier: {}, blockId: {}, length: {}",
workerId,
usedBytesOnTier,
blockId,
length);
long counter = AsyncJournalWriter.INVALID_FLUSH_COUNTER;
MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
// TODO(peis): Check lost workers as well.
if (worker == null) {
throw new NoWorkerException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
}
// Lock the worker metadata first.
synchronized (worker) {
// Loop until block metadata is successfully locked.
for (; ; ) {
boolean newBlock = false;
MasterBlockInfo block = mBlocks.get(blockId);
if (block == null) {
// The block metadata doesn't exist yet.
block = new MasterBlockInfo(blockId, length);
newBlock = true;
}
// Lock the block metadata.
synchronized (block) {
boolean writeJournal = false;
if (newBlock) {
if (mBlocks.putIfAbsent(blockId, block) != null) {
// Another thread already inserted the metadata for this block, so start loop over.
continue;
}
// Successfully added the new block metadata. Append a journal entry for the new
// metadata.
writeJournal = true;
} else if (block.getLength() != length && block.getLength() == Constants.UNKNOWN_SIZE) {
// The block size was previously unknown. Update the block size with the committed
// size, and append a journal entry.
block.updateLength(length);
writeJournal = true;
}
if (writeJournal) {
BlockInfoEntry blockInfo =
BlockInfoEntry.newBuilder().setBlockId(blockId).setLength(length).build();
counter = appendJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
}
// At this point, both the worker and the block metadata are locked.
// Update the block metadata with the new worker location.
block.addWorker(workerId, tierAlias);
// This worker has this block, so it is no longer lost.
mLostBlocks.remove(blockId);
// Update the worker information for this new block.
// TODO(binfan): when retry commitBlock on master is expected, make sure metrics are not
// double counted.
worker.addBlock(blockId);
worker.updateUsedBytes(tierAlias, usedBytesOnTier);
worker.updateLastUpdatedTimeMs();
}
break;
}
}
waitForJournalFlush(counter);
}
@Override
public void processJournalCheckpoint(JournalInputStream inputStream) throws IOException {
// clear state before processing checkpoint.
mBlocks.clear();
super.processJournalCheckpoint(inputStream);
}
