예제 #1
0
  @Test
  public void reserveTest() throws Exception {
    // Reserve on top tier
    long blockId = 100;
    BlockStoreLocation tier0 = BlockStoreLocation.anyDirInTier(StorageLevelAlias.MEM.getValue());
    for (int i = 0; i < 3; i++) {
      TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0);
    }
    CommonUtils.sleepMs(
        WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS));
    BlockStoreMeta storeMeta = mBlockStore.getBlockStoreMeta();
    Assert.assertEquals(3 * BLOCK_SIZE, storeMeta.getUsedBytes());
    List<Long> usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
    Assert.assertEquals(
        2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1));
    Assert.assertEquals(
        BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1));

    // Reserve on under tier
    for (int i = 0; i < 7; i++) {
      TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0);
    }
    CommonUtils.sleepMs(
        WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS));
    storeMeta = mBlockStore.getBlockStoreMeta();
    Assert.assertEquals(9 * BLOCK_SIZE, storeMeta.getUsedBytes());
    usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
    Assert.assertEquals(
        2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1));
    Assert.assertEquals(
        7 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1));
  }
 /**
  * * Get sorted fileIds of the files cached in the worker.
  *
  * @return a sorted fileId list
  */
 private List<Integer> getSortedFileIds() {
   Set<Integer> fileIds = new HashSet<Integer>();
   BlockStoreMeta storeMeta = mBlockDataManager.getStoreMeta();
   for (List<Long> blockIds : storeMeta.getBlockList().values()) {
     for (long blockId : blockIds) {
       int fileId = BlockInfo.computeInodeId(blockId);
       fileIds.add(fileId);
     }
   }
   List<Integer> sortedFileIds = new ArrayList<Integer>(fileIds);
   Collections.sort(sortedFileIds);
   return sortedFileIds;
 }
 /**
  * * Get sorted fileIds of the files cached in the worker.
  *
  * @return a sorted fileId list
  */
 private List<Long> getSortedFileIds() {
   Set<Long> fileIds = new HashSet<Long>();
   BlockStoreMeta storeMeta = mBlockDataManager.getStoreMeta();
   for (List<Long> blockIds : storeMeta.getBlockList().values()) {
     for (long blockId : blockIds) {
       long fileId =
           BlockId.createBlockId(BlockId.getContainerId(blockId), BlockId.getMaxSequenceNumber());
       fileIds.add(fileId);
     }
   }
   List<Long> sortedFileIds = new ArrayList<Long>(fileIds);
   Collections.sort(sortedFileIds);
   return sortedFileIds;
 }
예제 #4
0
  /**
   * Commits a block to Tachyon managed space. The block must be temporary. The block is persisted
   * after {@link BlockStore#commitBlock(long, long)}. The block will not be accessible until {@link
   * WorkerBlockMasterClient#commitBlock} succeeds.
   *
   * @param sessionId The id of the client
   * @param blockId The id of the block to commit
   * @throws BlockAlreadyExistsException if blockId already exists in committed blocks
   * @throws BlockDoesNotExistException if the temporary block cannot be found
   * @throws InvalidWorkerStateException if blockId does not belong to sessionId
   * @throws IOException if the block cannot be moved from temporary path to committed path
   * @throws WorkerOutOfSpaceException if there is no more space left to hold the block
   */
  public void commitBlock(long sessionId, long blockId)
      throws BlockAlreadyExistsException, BlockDoesNotExistException, InvalidWorkerStateException,
          IOException, WorkerOutOfSpaceException {
    mBlockStore.commitBlock(sessionId, blockId);

    // TODO(calvin): Reconsider how to do this without heavy locking.
    // Block successfully committed, update master with new block metadata
    Long lockId = mBlockStore.lockBlock(sessionId, blockId);
    try {
      BlockMeta meta = mBlockStore.getBlockMeta(sessionId, blockId, lockId);
      BlockStoreLocation loc = meta.getBlockLocation();
      Long length = meta.getBlockSize();
      BlockStoreMeta storeMeta = mBlockStore.getBlockStoreMeta();
      Long bytesUsedOnTier = storeMeta.getUsedBytesOnTiers().get(loc.tierAlias());
      mBlockMasterClient.commitBlock(
          WorkerIdRegistry.getWorkerId(), bytesUsedOnTier, loc.tierAlias(), blockId, length);
    } catch (IOException ioe) {
      throw new IOException("Failed to commit block to master.", ioe);
    } finally {
      mBlockStore.unlockBlock(lockId);
    }
  }