Exemple #1
0
  /**
   * Returns a worker id for the given worker.
   *
   * @param workerNetAddress the worker {@link WorkerNetAddress}
   * @return the worker id for this worker
   */
  public long getWorkerId(WorkerNetAddress workerNetAddress) {
    // TODO(gene): This NetAddress cloned in case thrift re-uses the object. Does thrift re-use it?
    synchronized (mWorkers) {
      if (mWorkers.contains(mAddressIndex, workerNetAddress)) {
        // This worker address is already mapped to a worker id.
        long oldWorkerId = mWorkers.getFirstByField(mAddressIndex, workerNetAddress).getId();
        LOG.warn("The worker {} already exists as id {}.", workerNetAddress, oldWorkerId);
        return oldWorkerId;
      }

      if (mLostWorkers.contains(mAddressIndex, workerNetAddress)) {
        // this is one of the lost workers
        final MasterWorkerInfo lostWorkerInfo =
            mLostWorkers.getFirstByField(mAddressIndex, workerNetAddress);
        final long lostWorkerId = lostWorkerInfo.getId();
        LOG.warn("A lost worker {} has requested its old id {}.", workerNetAddress, lostWorkerId);

        // Update the timestamp of the worker before it is considered an active worker.
        lostWorkerInfo.updateLastUpdatedTimeMs();
        mWorkers.add(lostWorkerInfo);
        mLostWorkers.remove(lostWorkerInfo);
        return lostWorkerId;
      }

      // Generate a new worker id.
      long workerId = mNextWorkerId.getAndIncrement();
      mWorkers.add(new MasterWorkerInfo(workerId, workerNetAddress));

      LOG.info("getWorkerId(): WorkerNetAddress: {} id: {}", workerNetAddress, workerId);
      return workerId;
    }
  }
Exemple #2
0
  /**
   * Updates metadata when a worker periodically heartbeats with the master.
   *
   * @param workerId the worker id
   * @param usedBytesOnTiers a mapping from tier alias to the used bytes
   * @param removedBlockIds a list of block ids removed from this worker
   * @param addedBlocksOnTiers a mapping from tier alias to the added blocks
   * @return an optional command for the worker to execute
   */
  public Command workerHeartbeat(
      long workerId,
      Map<String, Long> usedBytesOnTiers,
      List<Long> removedBlockIds,
      Map<String, List<Long>> addedBlocksOnTiers) {
    synchronized (mBlocks) {
      synchronized (mWorkers) {
        if (!mWorkers.contains(mIdIndex, workerId)) {
          LOG.warn("Could not find worker id: {} for heartbeat.", workerId);
          return new Command(CommandType.Register, new ArrayList<Long>());
        }

        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(mIdIndex, workerId);
        processWorkerRemovedBlocks(workerInfo, removedBlockIds);
        processWorkerAddedBlocks(workerInfo, addedBlocksOnTiers);

        workerInfo.updateUsedBytes(usedBytesOnTiers);
        workerInfo.updateLastUpdatedTimeMs();

        List<Long> toRemoveBlocks = workerInfo.getToRemoveBlocks();
        if (toRemoveBlocks.isEmpty()) {
          return new Command(CommandType.Nothing, new ArrayList<Long>());
        }
        return new Command(CommandType.Free, toRemoveBlocks);
      }
    }
  }
Exemple #3
0
  /**
   * Updates metadata when a worker registers with the master.
   *
   * @param workerId the worker id of the worker registering
   * @param storageTiers a list of storage tier aliases in order of their position in the worker's
   *     hierarchy
   * @param totalBytesOnTiers a mapping from storage tier alias to total bytes
   * @param usedBytesOnTiers a mapping from storage tier alias to the used byes
   * @param currentBlocksOnTiers a mapping from storage tier alias to a list of blocks
   * @throws NoWorkerException if workerId cannot be found
   */
  public void workerRegister(
      long workerId,
      List<String> storageTiers,
      Map<String, Long> totalBytesOnTiers,
      Map<String, Long> usedBytesOnTiers,
      Map<String, List<Long>> currentBlocksOnTiers)
      throws NoWorkerException {
    synchronized (mBlocks) {
      synchronized (mWorkers) {
        if (!mWorkers.contains(mIdIndex, workerId)) {
          throw new NoWorkerException("Could not find worker id: " + workerId + " to register.");
        }
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(mIdIndex, workerId);
        workerInfo.updateLastUpdatedTimeMs();

        // Gather all blocks on this worker.
        HashSet<Long> blocks = new HashSet<Long>();
        for (List<Long> blockIds : currentBlocksOnTiers.values()) {
          blocks.addAll(blockIds);
        }

        // Detect any lost blocks on this worker.
        Set<Long> removedBlocks =
            workerInfo.register(
                mGlobalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers, blocks);

        processWorkerRemovedBlocks(workerInfo, removedBlocks);
        processWorkerAddedBlocks(workerInfo, currentBlocksOnTiers);
        LOG.info("registerWorker(): {}", workerInfo);
      }
    }
  }