Exemplo n.º 1
0
  /**
   * Marks a block as committed on a specific worker.
   *
   * @param workerId the worker id committing the block
   * @param usedBytesOnTier the updated used bytes on the tier of the worker
   * @param tierAlias the alias of the storage tier where the worker is committing the block to
   * @param blockId the committing block id
   * @param length the length of the block
   */
  public void commitBlock(
      long workerId, long usedBytesOnTier, String tierAlias, long blockId, long length) {
    LOG.debug(
        "Commit block from worker: {}",
        FormatUtils.parametersToString(workerId, usedBytesOnTier, blockId, length));
    synchronized (mBlocks) {
      synchronized (mWorkers) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(mIdIndex, workerId);
        workerInfo.addBlock(blockId);
        workerInfo.updateUsedBytes(tierAlias, usedBytesOnTier);
        workerInfo.updateLastUpdatedTimeMs();

        MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
        if (masterBlockInfo == null) {
          masterBlockInfo = new MasterBlockInfo(blockId, length);
          mBlocks.put(blockId, masterBlockInfo);
          BlockInfoEntry blockInfo =
              BlockInfoEntry.newBuilder()
                  .setBlockId(masterBlockInfo.getBlockId())
                  .setLength(masterBlockInfo.getLength())
                  .build();
          writeJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
          flushJournal();
        }
        masterBlockInfo.addWorker(workerId, tierAlias);
        mLostBlocks.remove(blockId);
      }
    }
  }
Exemplo n.º 2
0
 /**
  * Creates a {@link BlockInfo} form a given {@link MasterBlockInfo}, by populating worker
  * locations.
  *
  * <p>NOTE: {@link #mWorkers} should already be locked before calling this method.
  *
  * @param masterBlockInfo the {@link MasterBlockInfo}
  * @return a {@link BlockInfo} from a {@link MasterBlockInfo}. Populates worker locations
  */
 private BlockInfo generateBlockInfo(MasterBlockInfo masterBlockInfo) {
   // "Join" to get all the addresses of the workers.
   List<BlockLocation> locations = new ArrayList<BlockLocation>();
   List<MasterBlockLocation> blockLocations = masterBlockInfo.getBlockLocations();
   // Sort the block locations by their alias ordinal in the master storage tier mapping
   Collections.sort(
       blockLocations,
       new Comparator<MasterBlockLocation>() {
         @Override
         public int compare(MasterBlockLocation o1, MasterBlockLocation o2) {
           return mGlobalStorageTierAssoc.getOrdinal(o1.getTierAlias())
               - mGlobalStorageTierAssoc.getOrdinal(o2.getTierAlias());
         }
       });
   for (MasterBlockLocation masterBlockLocation : blockLocations) {
     MasterWorkerInfo workerInfo =
         mWorkers.getFirstByField(mIdIndex, masterBlockLocation.getWorkerId());
     if (workerInfo != null) {
       locations.add(
           new BlockLocation()
               .setWorkerId(masterBlockLocation.getWorkerId())
               .setWorkerAddress(workerInfo.getWorkerAddress())
               .setTierAlias(masterBlockLocation.getTierAlias()));
     }
   }
   return new BlockInfo()
       .setBlockId(masterBlockInfo.getBlockId())
       .setLength(masterBlockInfo.getLength())
       .setLocations(locations);
 }
Exemplo n.º 3
0
 @Override
 public void streamToJournalCheckpoint(JournalOutputStream outputStream) throws IOException {
   outputStream.writeEntry(mBlockContainerIdGenerator.toJournalEntry());
   for (MasterBlockInfo blockInfo : mBlocks.values()) {
     BlockInfoEntry blockInfoEntry =
         BlockInfoEntry.newBuilder()
             .setBlockId(blockInfo.getBlockId())
             .setLength(blockInfo.getLength())
             .build();
     outputStream.writeEntry(JournalEntry.newBuilder().setBlockInfo(blockInfoEntry).build());
   }
 }
Exemplo n.º 4
0
 /**
  * Updates the worker and block metadata for blocks added to a worker.
  *
  * <p>NOTE: {@link #mBlocks} should already be locked before calling this method.
  *
  * @param workerInfo The worker metadata object
  * @param addedBlockIds A mapping from storage tier alias to a list of block ids added
  */
 private void processWorkerAddedBlocks(
     MasterWorkerInfo workerInfo, Map<String, List<Long>> addedBlockIds) {
   for (Map.Entry<String, List<Long>> entry : addedBlockIds.entrySet()) {
     for (long blockId : entry.getValue()) {
       MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
       if (masterBlockInfo != null) {
         workerInfo.addBlock(blockId);
         masterBlockInfo.addWorker(workerInfo.getId(), entry.getKey());
         mLostBlocks.remove(blockId);
       } else {
         LOG.warn("Failed to register workerId: {} to blockId: {}", workerInfo.getId(), blockId);
       }
     }
   }
 }
Exemplo n.º 5
0
  /**
   * Marks a block as committed, but without a worker location. This means the block is only in ufs.
   *
   * @param blockId the id of the block to commit
   * @param length the length of the block
   */
  public void commitBlockInUFS(long blockId, long length) {
    LOG.debug("Commit block to ufs: {}", FormatUtils.parametersToString(blockId, length));
    synchronized (mBlocks) {
      MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
      if (masterBlockInfo == null) {
        // The block has not been committed previously, so add the metadata to commit the block.
        masterBlockInfo = new MasterBlockInfo(blockId, length);
        mBlocks.put(blockId, masterBlockInfo);

        BlockInfoEntry blockInfo =
            BlockInfoEntry.newBuilder()
                .setBlockId(masterBlockInfo.getBlockId())
                .setLength(masterBlockInfo.getLength())
                .build();
        writeJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
        flushJournal();
      }
    }
  }
Exemplo n.º 6
0
 /**
  * Updates the worker and block metadata for blocks removed from a worker.
  *
  * <p>NOTE: {@link #mBlocks} should already be locked before calling this method.
  *
  * @param workerInfo The worker metadata object
  * @param removedBlockIds A list of block ids removed from the worker
  */
 private void processWorkerRemovedBlocks(
     MasterWorkerInfo workerInfo, Collection<Long> removedBlockIds) {
   for (long removedBlockId : removedBlockIds) {
     MasterBlockInfo masterBlockInfo = mBlocks.get(removedBlockId);
     if (masterBlockInfo == null) {
       LOG.warn(
           "Worker {} removed block {} but block does not exist.",
           workerInfo.getId(),
           removedBlockId);
       // Continue to remove the remaining blocks.
       continue;
     }
     LOG.info("Block {} is removed on worker {}.", removedBlockId, workerInfo.getId());
     workerInfo.removeBlock(masterBlockInfo.getBlockId());
     masterBlockInfo.removeWorker(workerInfo.getId());
     if (masterBlockInfo.getNumLocations() == 0) {
       mLostBlocks.add(removedBlockId);
     }
   }
 }
Exemplo n.º 7
0
 /**
  * Removes blocks from workers.
  *
  * @param blockIds a list of block ids to remove from Tachyon space
  */
 public void removeBlocks(List<Long> blockIds) {
   synchronized (mBlocks) {
     synchronized (mWorkers) {
       for (long blockId : blockIds) {
         MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
         if (masterBlockInfo == null) {
           continue;
         }
         for (long workerId : new ArrayList<Long>(masterBlockInfo.getWorkers())) {
           masterBlockInfo.removeWorker(workerId);
           MasterWorkerInfo worker = mWorkers.getFirstByField(mIdIndex, workerId);
           if (worker != null) {
             worker.updateToRemovedBlock(true, blockId);
           }
         }
         mLostBlocks.remove(blockId);
       }
     }
   }
 }