예제 #1
0
 /**
  * Updates the worker and block metadata for blocks removed from a worker.
  *
  * @param workerInfo The worker metadata object
  * @param removedBlockIds A list of block ids removed from the worker
  */
 @GuardedBy("workerInfo")
 private void processWorkerRemovedBlocks(
     MasterWorkerInfo workerInfo, Collection<Long> removedBlockIds) {
   for (long removedBlockId : removedBlockIds) {
     MasterBlockInfo block = mBlocks.get(removedBlockId);
     // TODO(calvin): Investigate if this branching logic can be simplified.
     if (block == null) {
       // LOG.warn("Worker {} informs the removed block {}, but block metadata does not exist"
       //    + " on Master!", workerInfo.getId(), removedBlockId);
       // TODO(pfxuan): [ALLUXIO-1804] should find a better way to handle the removed blocks.
       // Ideally, the delete/free I/O flow should never reach this point. Because Master may
       // update the block metadata only after receiving the acknowledgement from Workers.
       workerInfo.removeBlock(removedBlockId);
       // Continue to remove the remaining blocks.
       continue;
     }
     synchronized (block) {
       LOG.info("Block {} is removed on worker {}.", removedBlockId, workerInfo.getId());
       workerInfo.removeBlock(block.getBlockId());
       block.removeWorker(workerInfo.getId());
       if (block.getNumLocations() == 0) {
         mLostBlocks.add(removedBlockId);
       }
     }
   }
 }
예제 #2
0
  /**
   * Returns a worker id for the given worker.
   *
   * @param workerNetAddress the worker {@link WorkerNetAddress}
   * @return the worker id for this worker
   */
  public long getWorkerId(WorkerNetAddress workerNetAddress) {
    // TODO(gpang): This NetAddress cloned in case thrift re-uses the object. Does thrift re-use it?
    MasterWorkerInfo existingWorker = mWorkers.getFirstByField(ADDRESS_INDEX, workerNetAddress);
    if (existingWorker != null) {
      // This worker address is already mapped to a worker id.
      long oldWorkerId = existingWorker.getId();
      LOG.warn("The worker {} already exists as id {}.", workerNetAddress, oldWorkerId);
      return oldWorkerId;
    }

    MasterWorkerInfo lostWorker = mLostWorkers.getFirstByField(ADDRESS_INDEX, workerNetAddress);
    if (lostWorker != null) {
      // this is one of the lost workers
      synchronized (lostWorker) {
        final long lostWorkerId = lostWorker.getId();
        LOG.warn("A lost worker {} has requested its old id {}.", workerNetAddress, lostWorkerId);

        // Update the timestamp of the worker before it is considered an active worker.
        lostWorker.updateLastUpdatedTimeMs();
        mWorkers.add(lostWorker);
        mLostWorkers.remove(lostWorker);
        return lostWorkerId;
      }
    }

    // Generate a new worker id.
    long workerId = mNextWorkerId.getAndIncrement();
    mWorkers.add(new MasterWorkerInfo(workerId, workerNetAddress));

    LOG.info("getWorkerId(): WorkerNetAddress: {} id: {}", workerNetAddress, workerId);
    return workerId;
  }
예제 #3
0
 /**
  * Updates the worker and block metadata for blocks added to a worker.
  *
  * @param workerInfo The worker metadata object
  * @param addedBlockIds A mapping from storage tier alias to a list of block ids added
  */
 @GuardedBy("workerInfo")
 private void processWorkerAddedBlocks(
     MasterWorkerInfo workerInfo, Map<String, List<Long>> addedBlockIds) {
   for (Map.Entry<String, List<Long>> entry : addedBlockIds.entrySet()) {
     for (long blockId : entry.getValue()) {
       MasterBlockInfo block = mBlocks.get(blockId);
       if (block != null) {
         synchronized (block) {
           workerInfo.addBlock(blockId);
           block.addWorker(workerInfo.getId(), entry.getKey());
           mLostBlocks.remove(blockId);
         }
       } else {
         LOG.warn("Failed to register workerId: {} to blockId: {}", workerInfo.getId(), blockId);
       }
     }
   }
 }
예제 #4
0
 @Override
 public Object getFieldValue(MasterWorkerInfo o) {
   return o.getId();
 }