コード例 #1
0
ファイル: TestFileUtil.java プロジェクト: JoeChien23/hadoop
  @Test(timeout = 30000)
  public void testListFiles() throws IOException {
    setupDirs();
    // Test existing files case
    File[] files = FileUtil.listFiles(partitioned);
    Assert.assertEquals(2, files.length);

    // Test existing directory with no files case
    File newDir = new File(tmp.getPath(), "test");
    newDir.mkdir();
    Assert.assertTrue("Failed to create test dir", newDir.exists());
    files = FileUtil.listFiles(newDir);
    Assert.assertEquals(0, files.length);
    newDir.delete();
    Assert.assertFalse("Failed to delete test dir", newDir.exists());

    // Test non-existing directory case, this throws
    // IOException
    try {
      files = FileUtil.listFiles(newDir);
      Assert.fail("IOException expected on listFiles() for non-existent dir " + newDir.toString());
    } catch (IOException ioe) {
      // Expected an IOException
    }
  }
コード例 #2
0
 /**
  * Delete the given path to a file or directory.
  *
  * @param p the path to delete
  * @param recursive to delete sub-directories
  * @return true if the file or directory and all its contents were deleted
  * @throws IOException if p is non-empty and recursive is false
  */
 @Override
 public boolean delete(Path p, boolean recursive) throws IOException {
   File f = pathToFile(p);
   if (f.isFile()) {
     return f.delete();
   } else if (!recursive && f.isDirectory() && (FileUtil.listFiles(f).length != 0)) {
     throw new IOException("Directory " + f.toString() + " is not empty");
   }
   return FileUtil.fullyDelete(f);
 }
コード例 #3
0
  /**
   * Move replicas in the lazy persist directory to their corresponding locations in the finalized
   * directory.
   *
   * @return number of replicas recovered.
   */
  private int moveLazyPersistReplicasToFinalized(File source) throws IOException {
    File files[] = FileUtil.listFiles(source);
    int numRecovered = 0;
    for (File file : files) {
      if (file.isDirectory()) {
        numRecovered += moveLazyPersistReplicasToFinalized(file);
      }

      if (Block.isMetaFilename(file.getName())) {
        File metaFile = file;
        File blockFile = Block.metaToBlockFile(metaFile);
        long blockId = Block.filename2id(blockFile.getName());
        File targetDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId);

        if (blockFile.exists()) {

          if (!targetDir.exists() && !targetDir.mkdirs()) {
            LOG.warn("Failed to mkdirs " + targetDir);
            continue;
          }

          final File targetMetaFile = new File(targetDir, metaFile.getName());
          try {
            NativeIO.renameTo(metaFile, targetMetaFile);
          } catch (IOException e) {
            LOG.warn("Failed to move meta file from " + metaFile + " to " + targetMetaFile, e);
            continue;
          }

          final File targetBlockFile = new File(targetDir, blockFile.getName());
          try {
            NativeIO.renameTo(blockFile, targetBlockFile);
          } catch (IOException e) {
            LOG.warn("Failed to move block file from " + blockFile + " to " + targetBlockFile, e);
            continue;
          }

          if (targetBlockFile.exists() && targetMetaFile.exists()) {
            ++numRecovered;
          } else {
            // Failure should be rare.
            LOG.warn("Failed to move " + blockFile + " to " + targetDir);
          }
        }
      }
    }

    FileUtil.fullyDelete(source);
    return numRecovered;
  }
コード例 #4
0
  @Override
  public void inspectDirectory(StorageDirectory sd) throws IOException {
    // Was the directory just formatted?
    if (!sd.getVersionFile().exists()) {
      LOG.info("No version file in " + sd.getRoot());
      needToSave |= true;
      return;
    }

    maxSeenTxId = Math.max(maxSeenTxId, NNStorage.readTransactionIdFile(sd));

    File currentDir = sd.getCurrentDir();
    File filesInStorage[];
    try {
      filesInStorage = FileUtil.listFiles(currentDir);
    } catch (IOException ioe) {
      LOG.warn("Unable to inspect storage directory " + currentDir, ioe);
      return;
    }

    for (File f : filesInStorage) {
      LOG.info("Inspecting images: Checking file " + f);
      String name = f.getName();

      // Check for fsimage_*
      Matcher imageMatch = IMAGE_REGEX.matcher(name);
      addImageIfMatching(imageMatch, foundImages, sd, f);

      // Check for fsimage_*
      Matcher imageCkptMatch = IMAGE_CKPT_REGEX.matcher(name);
      addImageIfMatching(imageCkptMatch, foundCkptImages, sd, f);
    }

    // Check for a seen_txid file, which marks a minimum transaction ID that
    // must be included in our load plan.
    try {
      maxSeenTxId = Math.max(maxSeenTxId, NNStorage.readTransactionIdFile(sd));
    } catch (IOException ioe) {
      LOG.warn("Unable to determine the max transaction ID seen by " + sd, ioe);
    }

    // set finalized flag
    isUpgradeFinalized = isUpgradeFinalized && !sd.getPreviousDir().exists();
  }
コード例 #5
0
  /**
   * Add replicas under the given directory to the volume map
   *
   * @param volumeMap the replicas map
   * @param dir an input directory
   * @param lazyWriteReplicaMap Map of replicas on transient storage.
   * @param isFinalized true if the directory has finalized replicas; false if the directory has rbw
   *     replicas
   */
  void addToReplicasMap(
      ReplicaMap volumeMap,
      File dir,
      final RamDiskReplicaTracker lazyWriteReplicaMap,
      boolean isFinalized)
      throws IOException {
    File files[] = FileUtil.listFiles(dir);
    for (File file : files) {
      if (file.isDirectory()) {
        addToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized);
      }

      if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) {
        file = recoverTempUnlinkedBlock(file);
        if (file == null) { // the original block still exists, so we cover it
          // in another iteration and can continue here
          continue;
        }
      }
      if (!Block.isBlockFilename(file)) continue;

      long genStamp = FsDatasetUtil.getGenerationStampFromFile(files, file);
      long blockId = Block.filename2id(file.getName());
      ReplicaInfo newReplica = null;
      if (isFinalized) {
        newReplica =
            new FinalizedReplica(blockId, file.length(), genStamp, volume, file.getParentFile());
      } else {

        boolean loadRwr = true;
        File restartMeta =
            new File(file.getParent() + File.pathSeparator + "." + file.getName() + ".restart");
        Scanner sc = null;
        try {
          sc = new Scanner(restartMeta, "UTF-8");
          // The restart meta file exists
          if (sc.hasNextLong() && (sc.nextLong() > Time.now())) {
            // It didn't expire. Load the replica as a RBW.
            // We don't know the expected block length, so just use 0
            // and don't reserve any more space for writes.
            newReplica =
                new ReplicaBeingWritten(
                    blockId,
                    validateIntegrityAndSetLength(file, genStamp),
                    genStamp,
                    volume,
                    file.getParentFile(),
                    null,
                    0);
            loadRwr = false;
          }
          sc.close();
          if (!restartMeta.delete()) {
            FsDatasetImpl.LOG.warn("Failed to delete restart meta file: " + restartMeta.getPath());
          }
        } catch (FileNotFoundException fnfe) {
          // nothing to do hereFile dir =
        } finally {
          if (sc != null) {
            sc.close();
          }
        }
        // Restart meta doesn't exist or expired.
        if (loadRwr) {
          newReplica =
              new ReplicaWaitingToBeRecovered(
                  blockId,
                  validateIntegrityAndSetLength(file, genStamp),
                  genStamp,
                  volume,
                  file.getParentFile());
        }
      }

      ReplicaInfo oldReplica = volumeMap.get(bpid, newReplica.getBlockId());
      if (oldReplica == null) {
        volumeMap.add(bpid, newReplica);
      } else {
        // We have multiple replicas of the same block so decide which one
        // to keep.
        newReplica = resolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
      }

      // If we are retaining a replica on transient storage make sure
      // it is in the lazyWriteReplicaMap so it can be persisted
      // eventually.
      if (newReplica.getVolume().isTransientStorage()) {
        lazyWriteReplicaMap.addReplica(bpid, blockId, (FsVolumeImpl) newReplica.getVolume());
      } else {
        lazyWriteReplicaMap.discardReplica(bpid, blockId, false);
      }
    }
  }