コード例 #1
0
ファイル: TestFileUtil.java プロジェクト: JoeChien23/hadoop
  /**
   * Test that getDU is able to handle cycles caused due to symbolic links and that directory sizes
   * are not added to the final calculated size
   *
   * @throws IOException
   */
  @Test(timeout = 30000)
  public void testGetDU() throws Exception {
    setupDirs();

    long du = FileUtil.getDU(TEST_DIR);
    // Only two files (in partitioned).  Each has 3 characters + system-specific
    // line separator.
    final long expected = 2 * (3 + System.getProperty("line.separator").length());
    Assert.assertEquals(expected, du);

    // target file does not exist:
    final File doesNotExist = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog");
    long duDoesNotExist = FileUtil.getDU(doesNotExist);
    assertEquals(0, duDoesNotExist);

    // target file is not a directory:
    File notADirectory = new File(partitioned, "part-r-00000");
    long duNotADirectoryActual = FileUtil.getDU(notADirectory);
    long duNotADirectoryExpected = 3 + System.getProperty("line.separator").length();
    assertEquals(duNotADirectoryExpected, duNotADirectoryActual);

    try {
      // one of target files is not accessible, but the containing directory
      // is accessible:
      try {
        FileUtil.chmod(notADirectory.getAbsolutePath(), "0000");
      } catch (InterruptedException ie) {
        // should never happen since that method never throws InterruptedException.
        assertNull(ie);
      }
      assertFalse(FileUtil.canRead(notADirectory));
      final long du3 = FileUtil.getDU(partitioned);
      assertEquals(expected, du3);

      // some target files and containing directory are not accessible:
      try {
        FileUtil.chmod(partitioned.getAbsolutePath(), "0000");
      } catch (InterruptedException ie) {
        // should never happen since that method never throws InterruptedException.
        assertNull(ie);
      }
      assertFalse(FileUtil.canRead(partitioned));
      final long du4 = FileUtil.getDU(partitioned);
      assertEquals(0, du4);
    } finally {
      // Restore the permissions so that we can delete the folder
      // in @After method:
      FileUtil.chmod(partitioned.getAbsolutePath(), "0777", true /*recursive*/);
    }
  }
コード例 #2
0
  // the method which actually copies the caches locally and unjars/unzips them
  // and does chmod for the files
  private static Path localizeCache(
      Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, boolean isArchive)
      throws IOException {
    FileSystem fs = getFileSystem(cache, conf);
    FileSystem localFs = FileSystem.getLocal(conf);
    Path parchive = null;

    if (isArchive) {
      parchive =
          new Path(
              cacheStatus.localizedLoadPath, new Path(cacheStatus.localizedLoadPath.getName()));
    } else {
      parchive = cacheStatus.localizedLoadPath;
    }
    if (!localFs.mkdirs(parchive.getParent())) {
      throw new IOException(
          "Mkdirs failed to create directory " + cacheStatus.localizedLoadPath.toString());
    }
    String cacheId = cache.getPath();

    fs.copyToLocalFile(new Path(cacheId), parchive);
    if (isArchive) {
      String tmpArchive = parchive.toString().toLowerCase();
      File srcFile = new File(parchive.toString());
      File destDir = new File(parchive.getParent().toString());
      if (tmpArchive.endsWith(".jar")) {
        RunJar.unJar(srcFile, destDir);
      } else if (tmpArchive.endsWith(".zip")) {
        FileUtil.unZip(srcFile, destDir);
      } else if (isTarFile(tmpArchive)) {
        FileUtil.unTar(srcFile, destDir);
      }
      // else will not do anyhting
      // and copy the file into the dir as it is
    }
    long cacheSize = FileUtil.getDU(new File(parchive.getParent().toString()));
    cacheStatus.size = cacheSize;
    addCacheInfoUpdate(cacheStatus);

    // do chmod here
    try {
      // Setting recursive permission to grant everyone read and execute
      Path localDir = new Path(cacheStatus.localizedBaseDir, cacheStatus.uniqueParentDir);
      LOG.info("Doing chmod on localdir :" + localDir);
      FileUtil.chmod(localDir.toString(), "ugo+rx", true);
    } catch (InterruptedException e) {
      LOG.warn("Exception in chmod" + e.toString());
    }

    // update cacheStatus to reflect the newly cached file
    cacheStatus.mtime = getTimestamp(conf, cache);
    return cacheStatus.localizedLoadPath;
  }