/**
* Updates the worker and block metadata for blocks removed from a worker.
*
* @param workerInfo The worker metadata object
* @param removedBlockIds A list of block ids removed from the worker
*/
@GuardedBy("workerInfo")
private void processWorkerRemovedBlocks(
MasterWorkerInfo workerInfo, Collection<Long> removedBlockIds) {
for (long removedBlockId : removedBlockIds) {
MasterBlockInfo block = mBlocks.get(removedBlockId);
// TODO(calvin): Investigate if this branching logic can be simplified.
if (block == null) {
// LOG.warn("Worker {} informs the removed block {}, but block metadata does not exist"
// + " on Master!", workerInfo.getId(), removedBlockId);
// TODO(pfxuan): [ALLUXIO-1804] should find a better way to handle the removed blocks.
// Ideally, the delete/free I/O flow should never reach this point. Because Master may
// update the block metadata only after receiving the acknowledgement from Workers.
workerInfo.removeBlock(removedBlockId);
// Continue to remove the remaining blocks.
continue;
}
synchronized (block) {
LOG.info("Block {} is removed on worker {}.", removedBlockId, workerInfo.getId());
workerInfo.removeBlock(block.getBlockId());
block.removeWorker(workerInfo.getId());
if (block.getNumLocations() == 0) {
mLostBlocks.add(removedBlockId);
}
}
}
}
/**
* Returns a worker id for the given worker.
*
* @param workerNetAddress the worker {@link WorkerNetAddress}
* @return the worker id for this worker
*/
public long getWorkerId(WorkerNetAddress workerNetAddress) {
// TODO(gpang): This NetAddress cloned in case thrift re-uses the object. Does thrift re-use it?
MasterWorkerInfo existingWorker = mWorkers.getFirstByField(ADDRESS_INDEX, workerNetAddress);
if (existingWorker != null) {
// This worker address is already mapped to a worker id.
long oldWorkerId = existingWorker.getId();
LOG.warn("The worker {} already exists as id {}.", workerNetAddress, oldWorkerId);
return oldWorkerId;
}
MasterWorkerInfo lostWorker = mLostWorkers.getFirstByField(ADDRESS_INDEX, workerNetAddress);
if (lostWorker != null) {
// this is one of the lost workers
synchronized (lostWorker) {
final long lostWorkerId = lostWorker.getId();
LOG.warn("A lost worker {} has requested its old id {}.", workerNetAddress, lostWorkerId);
// Update the timestamp of the worker before it is considered an active worker.
lostWorker.updateLastUpdatedTimeMs();
mWorkers.add(lostWorker);
mLostWorkers.remove(lostWorker);
return lostWorkerId;
}
}
// Generate a new worker id.
long workerId = mNextWorkerId.getAndIncrement();
mWorkers.add(new MasterWorkerInfo(workerId, workerNetAddress));
LOG.info("getWorkerId(): WorkerNetAddress: {} id: {}", workerNetAddress, workerId);
return workerId;
}
/**
* Updates the worker and block metadata for blocks added to a worker.
*
* @param workerInfo The worker metadata object
* @param addedBlockIds A mapping from storage tier alias to a list of block ids added
*/
@GuardedBy("workerInfo")
private void processWorkerAddedBlocks(
MasterWorkerInfo workerInfo, Map<String, List<Long>> addedBlockIds) {
for (Map.Entry<String, List<Long>> entry : addedBlockIds.entrySet()) {
for (long blockId : entry.getValue()) {
MasterBlockInfo block = mBlocks.get(blockId);
if (block != null) {
synchronized (block) {
workerInfo.addBlock(blockId);
block.addWorker(workerInfo.getId(), entry.getKey());
mLostBlocks.remove(blockId);
}
} else {
LOG.warn("Failed to register workerId: {} to blockId: {}", workerInfo.getId(), blockId);
}
}
}
}
@Override
public Object getFieldValue(MasterWorkerInfo o) {
return o.getId();
}