Exemplo n.º 1
0
  /**
   * Marks a block as committed on a specific worker.
   *
   * @param workerId the worker id committing the block
   * @param usedBytesOnTier the updated used bytes on the tier of the worker
   * @param tierAlias the alias of the storage tier where the worker is committing the block to
   * @param blockId the committing block id
   * @param length the length of the block
   */
  public void commitBlock(
      long workerId, long usedBytesOnTier, String tierAlias, long blockId, long length) {
    LOG.debug(
        "Commit block from worker: {}",
        FormatUtils.parametersToString(workerId, usedBytesOnTier, blockId, length));
    synchronized (mBlocks) {
      synchronized (mWorkers) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(mIdIndex, workerId);
        workerInfo.addBlock(blockId);
        workerInfo.updateUsedBytes(tierAlias, usedBytesOnTier);
        workerInfo.updateLastUpdatedTimeMs();

        MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
        if (masterBlockInfo == null) {
          masterBlockInfo = new MasterBlockInfo(blockId, length);
          mBlocks.put(blockId, masterBlockInfo);
          BlockInfoEntry blockInfo =
              BlockInfoEntry.newBuilder()
                  .setBlockId(masterBlockInfo.getBlockId())
                  .setLength(masterBlockInfo.getLength())
                  .build();
          writeJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
          flushJournal();
        }
        masterBlockInfo.addWorker(workerId, tierAlias);
        mLostBlocks.remove(blockId);
      }
    }
  }
Exemplo n.º 2
0
 private JournalEntry newCompletePartitionEntry(long fileId, PartitionInfo info) {
   CompletePartitionEntry completePartition =
       CompletePartitionEntry.newBuilder()
           .setStoreId(fileId)
           .setBlockId(info.blockId)
           .setKeyStartBytes(ByteString.copyFrom(info.keyStart))
           .setKeyLimitBytes(ByteString.copyFrom(info.keyLimit))
           .build();
   return JournalEntry.newBuilder().setCompletePartition(completePartition).build();
 }
Exemplo n.º 3
0
 @Override
 public void streamToJournalCheckpoint(JournalOutputStream outputStream) throws IOException {
   outputStream.writeEntry(mBlockContainerIdGenerator.toJournalEntry());
   for (MasterBlockInfo blockInfo : mBlocks.values()) {
     BlockInfoEntry blockInfoEntry =
         BlockInfoEntry.newBuilder()
             .setBlockId(blockInfo.getBlockId())
             .setLength(blockInfo.getLength())
             .build();
     outputStream.writeEntry(JournalEntry.newBuilder().setBlockInfo(blockInfoEntry).build());
   }
 }
Exemplo n.º 4
0
 @Override
 public synchronized JournalEntry toJournalEntry() {
   InodeDirectoryEntry inodeDirectory =
       InodeDirectoryEntry.newBuilder()
           .setCreationTimeMs(getCreationTimeMs())
           .setId(getId())
           .setName(getName())
           .setParentId(getParentId())
           .setPersisted(isPersisted())
           .setPinned(isPinned())
           .setLastModificationTimeMs(getLastModificationTimeMs())
           .setUserName(getUserName())
           .setGroupName(getGroupName())
           .setPermission(getPermission())
           .build();
   return JournalEntry.newBuilder().setInodeDirectory(inodeDirectory).build();
 }
Exemplo n.º 5
0
  /**
   * Marks a block as committed, but without a worker location. This means the block is only in ufs.
   *
   * @param blockId the id of the block to commit
   * @param length the length of the block
   */
  public void commitBlockInUFS(long blockId, long length) {
    LOG.debug("Commit block to ufs: {}", FormatUtils.parametersToString(blockId, length));
    synchronized (mBlocks) {
      MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
      if (masterBlockInfo == null) {
        // The block has not been committed previously, so add the metadata to commit the block.
        masterBlockInfo = new MasterBlockInfo(blockId, length);
        mBlocks.put(blockId, masterBlockInfo);

        BlockInfoEntry blockInfo =
            BlockInfoEntry.newBuilder()
                .setBlockId(masterBlockInfo.getBlockId())
                .setLength(masterBlockInfo.getLength())
                .build();
        writeJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
        flushJournal();
      }
    }
  }
Exemplo n.º 6
0
 private JournalEntry newCompleteStoreEntry(long fileId) {
   CompleteStoreEntry completeStore = CompleteStoreEntry.newBuilder().setStoreId(fileId).build();
   return JournalEntry.newBuilder().setCompleteStore(completeStore).build();
 }
Exemplo n.º 7
0
 private JournalEntry newCreateStoreEntry(long fileId) {
   CreateStoreEntry createStore = CreateStoreEntry.newBuilder().setStoreId(fileId).build();
   return JournalEntry.newBuilder().setCreateStore(createStore).build();
 }