/**
   * Make sure at least one non-transient volume has a saved copy of the replica. An infinite loop
   * is used to ensure the async lazy persist tasks are completely done before verification. Caller
   * of ensureLazyPersistBlocksAreSaved expects either a successful pass or timeout failure.
   */
  protected final void ensureLazyPersistBlocksAreSaved(LocatedBlocks locatedBlocks)
      throws IOException, InterruptedException {
    final String bpid = cluster.getNamesystem().getBlockPoolId();
    List<? extends FsVolumeSpi> volumes = cluster.getDataNodes().get(0).getFSDataset().getVolumes();
    final Set<Long> persistedBlockIds = new HashSet<Long>();

    while (persistedBlockIds.size() < locatedBlocks.getLocatedBlocks().size()) {
      // Take 1 second sleep before each verification iteration
      Thread.sleep(1000);

      for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
        for (FsVolumeSpi v : volumes) {
          if (v.isTransientStorage()) {
            continue;
          }

          FsVolumeImpl volume = (FsVolumeImpl) v;
          File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();

          long blockId = lb.getBlock().getBlockId();
          File targetDir = DatanodeUtil.idToBlockDir(lazyPersistDir, blockId);
          File blockFile = new File(targetDir, lb.getBlock().getBlockName());
          if (blockFile.exists()) {
            // Found a persisted copy for this block and added to the Set
            persistedBlockIds.add(blockId);
          }
        }
      }
    }

    // We should have found a persisted copy for each located block.
    assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
  }
 /** Move a persisted replica from lazypersist directory to a subdirectory under finalized. */
 File activateSavedReplica(Block b, File metaFile, File blockFile) throws IOException {
   final File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId());
   final File targetBlockFile = new File(blockDir, blockFile.getName());
   final File targetMetaFile = new File(blockDir, metaFile.getName());
   FileUtils.moveFile(blockFile, targetBlockFile);
   FsDatasetImpl.LOG.info("Moved " + blockFile + " to " + targetBlockFile);
   FileUtils.moveFile(metaFile, targetMetaFile);
   FsDatasetImpl.LOG.info("Moved " + metaFile + " to " + targetMetaFile);
   return targetBlockFile;
 }
 File addBlock(Block b, File f) throws IOException {
   File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId());
   if (!blockDir.exists()) {
     if (!blockDir.mkdirs()) {
       throw new IOException("Failed to mkdirs " + blockDir);
     }
   }
   File blockFile = FsDatasetImpl.moveBlockFiles(b, f, blockDir);
   File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp());
   dfsUsage.incDfsUsed(b.getNumBytes() + metaFile.length());
   return blockFile;
 }
  protected final boolean verifyBlockDeletedFromDir(File dir, LocatedBlocks locatedBlocks) {

    for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
      File targetDir = DatanodeUtil.idToBlockDir(dir, lb.getBlock().getBlockId());

      File blockFile = new File(targetDir, lb.getBlock().getBlockName());
      if (blockFile.exists()) {
        LOG.warn("blockFile: " + blockFile.getAbsolutePath() + " exists after deletion.");
        return false;
      }
      File metaFile =
          new File(
              targetDir,
              DatanodeUtil.getMetaName(
                  lb.getBlock().getBlockName(), lb.getBlock().getGenerationStamp()));
      if (metaFile.exists()) {
        LOG.warn("metaFile: " + metaFile.getAbsolutePath() + " exists after deletion.");
        return false;
      }
    }
    return true;
  }
  /**
   * Move replicas in the lazy persist directory to their corresponding locations in the finalized
   * directory.
   *
   * @return number of replicas recovered.
   */
  private int moveLazyPersistReplicasToFinalized(File source) throws IOException {
    File files[] = FileUtil.listFiles(source);
    int numRecovered = 0;
    for (File file : files) {
      if (file.isDirectory()) {
        numRecovered += moveLazyPersistReplicasToFinalized(file);
      }

      if (Block.isMetaFilename(file.getName())) {
        File metaFile = file;
        File blockFile = Block.metaToBlockFile(metaFile);
        long blockId = Block.filename2id(blockFile.getName());
        File targetDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId);

        if (blockFile.exists()) {

          if (!targetDir.exists() && !targetDir.mkdirs()) {
            LOG.warn("Failed to mkdirs " + targetDir);
            continue;
          }

          final File targetMetaFile = new File(targetDir, metaFile.getName());
          try {
            NativeIO.renameTo(metaFile, targetMetaFile);
          } catch (IOException e) {
            LOG.warn("Failed to move meta file from " + metaFile + " to " + targetMetaFile, e);
            continue;
          }

          final File targetBlockFile = new File(targetDir, blockFile.getName());
          try {
            NativeIO.renameTo(blockFile, targetBlockFile);
          } catch (IOException e) {
            LOG.warn("Failed to move block file from " + blockFile + " to " + targetBlockFile, e);
            continue;
          }

          if (targetBlockFile.exists() && targetMetaFile.exists()) {
            ++numRecovered;
          } else {
            // Failure should be rare.
            LOG.warn("Failed to move " + blockFile + " to " + targetDir);
          }
        }
      }
    }

    FileUtil.fullyDelete(source);
    return numRecovered;
  }
 /** RBW files. They get moved to the finalized block directory when the block is finalized. */
 File createRbwFile(Block b) throws IOException {
   File f = new File(rbwDir, b.getBlockName());
   return DatanodeUtil.createTmpFile(b, f);
 }