Ejemplo n.º 1
0
  private void checkLocalizedPath(boolean visibility)
      throws IOException, LoginException, InterruptedException {
    TrackerDistributedCacheManager manager =
        new TrackerDistributedCacheManager(conf, taskController);
    String userName = getJobOwnerName();
    File workDir = new File(TEST_ROOT_DIR, "workdir");
    Path cacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile");
    if (visibility) {
      createPublicTempFile(cacheFile);
    } else {
      createPrivateTempFile(cacheFile);
    }

    Configuration conf1 = new Configuration(conf);
    conf1.set("user.name", userName);
    DistributedCache.addCacheFile(cacheFile.toUri(), conf1);
    TrackerDistributedCacheManager.determineTimestamps(conf1);
    TrackerDistributedCacheManager.determineCacheVisibilities(conf1);
    dumpState(conf1);

    // Task localizing for job
    TaskDistributedCacheManager handle =
        manager.newTaskDistributedCacheManager(new JobID("jt", 1), conf1);
    handle.setupCache(
        conf1,
        TaskTracker.getPublicDistributedCacheDir(),
        TaskTracker.getPrivateDistributedCacheDir(userName));
    JobLocalizer.downloadPrivateCache(conf1);
    TaskDistributedCacheManager.CacheFile c = handle.getCacheFiles().get(0);
    String distCacheDir;
    if (visibility) {
      distCacheDir = TaskTracker.getPublicDistributedCacheDir();
    } else {
      distCacheDir = TaskTracker.getPrivateDistributedCacheDir(userName);
    }
    Path localizedPath =
        manager.getLocalCache(
            cacheFile.toUri(),
            conf1,
            distCacheDir,
            fs.getFileStatus(cacheFile),
            false,
            c.timestamp,
            visibility,
            c);
    assertTrue(
        "Cache file didn't get localized in the expected directory. "
            + "Expected localization to happen within "
            + ROOT_MAPRED_LOCAL_DIR
            + "/"
            + distCacheDir
            + ", but was localized at "
            + localizedPath,
        localizedPath.toString().contains(distCacheDir));
    if (visibility) {
      checkPublicFilePermissions(new Path[] {localizedPath});
    } else {
      checkFilePermissions(new Path[] {localizedPath});
    }
  }
Ejemplo n.º 2
0
  /**
   * This is the typical flow for using the DistributedCache classes.
   *
   * @throws IOException
   * @throws LoginException
   */
  public void testManagerFlow() throws IOException, LoginException {
    if (!canRun()) {
      return;
    }

    // ****** Imitate JobClient code
    // Configures a task/job with both a regular file and a "classpath" file.
    Configuration subConf = new Configuration(conf);
    String userName = getJobOwnerName();
    subConf.set("user.name", userName);
    JobID jobid = new JobID("jt", 1);
    DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf);
    DistributedCache.addFileToClassPath(secondCacheFile, subConf, FileSystem.get(subConf));
    TrackerDistributedCacheManager.determineTimestamps(subConf);
    TrackerDistributedCacheManager.determineCacheVisibilities(subConf);
    // ****** End of imitating JobClient code

    Path jobFile = new Path(TEST_ROOT_DIR, "job.xml");
    FileOutputStream os = new FileOutputStream(new File(jobFile.toString()));
    subConf.writeXml(os);
    os.close();

    // ****** Imitate TaskRunner code.
    TrackerDistributedCacheManager manager =
        new TrackerDistributedCacheManager(conf, taskController);
    TaskDistributedCacheManager handle = manager.newTaskDistributedCacheManager(jobid, subConf);
    assertNull(null, DistributedCache.getLocalCacheFiles(subConf));
    File workDir = new File(new Path(TEST_ROOT_DIR, "workdir").toString());
    handle.setupCache(
        subConf,
        TaskTracker.getPublicDistributedCacheDir(),
        TaskTracker.getPrivateDistributedCacheDir(userName));
    JobLocalizer.downloadPrivateCache(subConf);
    // DOESN'T ACTUALLY HAPPEN IN THE TaskRunner (THIS IS A TODO)
    //    handle.setupPrivateCache(localDirAllocator, TaskTracker
    //        .getPrivateDistributedCacheDir(userName));
    //    // ****** End of imitating TaskRunner code

    Path[] localCacheFiles = DistributedCache.getLocalCacheFiles(subConf);
    assertNotNull(null, localCacheFiles);
    assertEquals(2, localCacheFiles.length);
    Path cachedFirstFile = localCacheFiles[0];
    Path cachedSecondFile = localCacheFiles[1];
    assertFileLengthEquals(firstCacheFile, cachedFirstFile);
    assertFalse("Paths should be different.", firstCacheFile.equals(cachedFirstFile));

    assertEquals(1, handle.getClassPaths().size());
    assertEquals(cachedSecondFile.toString(), handle.getClassPaths().get(0));

    checkFilePermissions(localCacheFiles);

    // Cleanup
    handle.release();
    manager.purgeCache();
    assertFalse(pathToFile(cachedFirstFile).exists());
  }
Ejemplo n.º 3
0
 public void testFileSystemOtherThanDefault() throws Exception {
   if (!canRun()) {
     return;
   }
   TrackerDistributedCacheManager manager =
       new TrackerDistributedCacheManager(conf, taskController);
   conf.set("fs.fakefile.impl", conf.get("fs.file.impl"));
   String userName = getJobOwnerName();
   conf.set("user.name", userName);
   Path fileToCache = new Path("fakefile:///" + firstCacheFile.toUri().getPath());
   CacheFile file =
       new CacheFile(fileToCache.toUri(), CacheFile.FileType.REGULAR, false, 0, false);
   Path result =
       manager.getLocalCache(
           fileToCache.toUri(),
           conf,
           TaskTracker.getPrivateDistributedCacheDir(userName),
           fs.getFileStatus(firstCacheFile),
           false,
           System.currentTimeMillis(),
           false,
           file);
   assertNotNull("DistributedCache cached file on non-default filesystem.", result);
 }
Ejemplo n.º 4
0
  /**
   * Initialize the local directories for a particular user on this TT. This involves creation and
   * setting permissions of the following directories
   *
   * <ul>
   *   <li>$mapred.local.dir/taskTracker/$user
   *   <li>$mapred.local.dir/taskTracker/$user/jobcache
   *   <li>$mapred.local.dir/taskTracker/$user/distcache
   * </ul>
   *
   * @param user
   * @throws IOException
   */
  public void initializeUserDirs(String user) throws IOException {

    if (user == null) {
      // This shouldn't happen in general
      throw new IOException("User is null. Cannot initialized user-directories.");
    }

    AtomicBoolean localizedUser;
    synchronized (localizedUsers) {
      if (!localizedUsers.containsKey(user)) {
        localizedUsers.put(user, new AtomicBoolean(false));
      }
      localizedUser = localizedUsers.get(user);
    }

    synchronized (localizedUser) {
      if (localizedUser.get()) {
        // User-directories are already localized for this user.
        LOG.info(
            "User-directories for the user "
                + user
                + " are already initialized on this TT. Not doing anything.");
        return;
      }

      LOG.info("Initializing user " + user + " on this TT.");

      boolean userDirStatus = false;
      boolean jobCacheDirStatus = false;
      boolean distributedCacheDirStatus = false;

      for (String localDir : localDirs) {

        Path userDir = new Path(localDir, TaskTracker.getUserDir(user));

        // Set up the user-directory.
        if (fs.exists(userDir) || fs.mkdirs(userDir)) {

          // Set permissions on the user-directory
          FsPermission userOnly = new FsPermission((short) 0700);
          FileUtil.setPermission(new File(userDir.toUri().getPath()), userOnly);
          userDirStatus = true;

          // Set up the jobcache directory
          File jobCacheDir = new File(localDir, TaskTracker.getJobCacheSubdir(user));
          if (jobCacheDir.exists() || jobCacheDir.mkdirs()) {
            // Set permissions on the jobcache-directory
            FileUtil.setPermission(jobCacheDir, userOnly);
            jobCacheDirStatus = true;
          } else {
            LOG.warn("Unable to create job cache directory : " + jobCacheDir);
          }

          // Set up the cache directory used for distributed cache files
          File distributedCacheDir =
              new File(localDir, TaskTracker.getPrivateDistributedCacheDir(user));
          if (distributedCacheDir.exists() || distributedCacheDir.mkdirs()) {
            // Set permissions on the distcache-directory
            FileUtil.setPermission(distributedCacheDir, userOnly);
            distributedCacheDirStatus = true;
          } else {
            LOG.warn("Unable to create distributed-cache directory : " + distributedCacheDir);
          }
        } else {
          LOG.warn("Unable to create the user directory : " + userDir);
        }
      }

      if (!userDirStatus) {
        throw new IOException(
            "Not able to initialize user directories "
                + "in any of the configured local directories for user "
                + user);
      }
      if (!jobCacheDirStatus) {
        throw new IOException(
            "Not able to initialize job-cache directories "
                + "in any of the configured local directories for user "
                + user);
      }
      if (!distributedCacheDirStatus) {
        throw new IOException(
            "Not able to initialize distributed-cache directories "
                + "in any of the configured local directories for user "
                + user);
      }

      // Localization of the user is done
      localizedUser.set(true);
    }
  }
Ejemplo n.º 5
0
  /**
   * Localize a file. After localization is complete, create a file, "myFile", under the directory
   * where the file is localized and ensure that it has permissions different from what is set by
   * default. Then, localize another file. Verify that "myFile" has the right permissions.
   *
   * @throws Exception
   */
  public void testCustomPermissions() throws Exception {
    if (!canRun()) {
      return;
    }
    String userName = getJobOwnerName();
    conf.set("user.name", userName);
    TrackerDistributedCacheManager manager =
        new TrackerDistributedCacheManager(conf, taskController);
    FileSystem localfs = FileSystem.getLocal(conf);
    long now = System.currentTimeMillis();

    Path[] localCache = new Path[2];
    FileStatus stat = fs.getFileStatus(firstCacheFile);
    CacheFile file =
        new CacheFile(
            firstCacheFilePublic.toUri(),
            CacheFile.FileType.REGULAR,
            true,
            stat.getModificationTime(),
            false);
    localCache[0] =
        manager.getLocalCache(
            firstCacheFilePublic.toUri(),
            conf,
            TaskTracker.getPrivateDistributedCacheDir(userName),
            fs.getFileStatus(firstCacheFilePublic),
            false,
            fs.getFileStatus(firstCacheFilePublic).getModificationTime(),
            true,
            file);
    FsPermission myPermission = new FsPermission((short) 0600);
    Path myFile = new Path(localCache[0].getParent(), "myfile.txt");
    if (FileSystem.create(localfs, myFile, myPermission) == null) {
      throw new IOException("Could not create " + myFile);
    }
    try {
      stat = fs.getFileStatus(secondCacheFilePublic);
      file =
          new CacheFile(
              secondCacheFilePublic.toUri(),
              CacheFile.FileType.REGULAR,
              true,
              stat.getModificationTime(),
              false);
      localCache[1] =
          manager.getLocalCache(
              secondCacheFilePublic.toUri(),
              conf,
              TaskTracker.getPrivateDistributedCacheDir(userName),
              fs.getFileStatus(secondCacheFilePublic),
              false,
              fs.getFileStatus(secondCacheFilePublic).getModificationTime(),
              true,
              file);
      stat = localfs.getFileStatus(myFile);
      assertTrue(stat.getPermission().equals(myPermission));
      // validate permissions of localized files.
      checkFilePermissions(localCache);
    } finally {
      localfs.delete(myFile, false);
    }
  }
Ejemplo n.º 6
0
  public void testFreshness() throws Exception {
    if (!canRun()) {
      return;
    }
    Configuration myConf = new Configuration(conf);
    myConf.set("fs.default.name", "refresh:///");
    myConf.setClass("fs.refresh.impl", FakeFileSystem.class, FileSystem.class);
    String userName = getJobOwnerName();

    TrackerDistributedCacheManager manager =
        new TrackerDistributedCacheManager(myConf, taskController);
    // ****** Imitate JobClient code
    // Configures a task/job with both a regular file and a "classpath" file.
    Configuration subConf = new Configuration(myConf);
    subConf.set("user.name", userName);
    DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf);
    TrackerDistributedCacheManager.determineTimestamps(subConf);
    TrackerDistributedCacheManager.determineCacheVisibilities(subConf);
    // ****** End of imitating JobClient code

    // ****** Imitate TaskRunner code.
    TaskDistributedCacheManager handle =
        manager.newTaskDistributedCacheManager(new JobID("jt", 1), subConf);
    assertNull(null, DistributedCache.getLocalCacheFiles(subConf));
    File workDir = new File(new Path(TEST_ROOT_DIR, "workdir").toString());
    handle.setupCache(
        subConf,
        TaskTracker.getPublicDistributedCacheDir(),
        TaskTracker.getPrivateDistributedCacheDir(userName));
    // TODO this doesn't really happen in the TaskRunner
    //    handle.setupPrivateCache(localDirAllocator, TaskTracker
    //        .getPrivateDistributedCacheDir(userName));
    // ****** End of imitating TaskRunner code

    Path[] localCacheFiles = DistributedCache.getLocalCacheFiles(subConf);
    assertNotNull(null, localCacheFiles);
    assertEquals(1, localCacheFiles.length);
    Path cachedFirstFile = localCacheFiles[0];
    assertFileLengthEquals(firstCacheFile, cachedFirstFile);
    assertFalse("Paths should be different.", firstCacheFile.equals(cachedFirstFile));
    // release
    handle.release();

    // change the file timestamp
    FileSystem fs = FileSystem.get(myConf);
    ((FakeFileSystem) fs).advanceClock(1);

    // running a task of the same job
    Throwable th = null;
    try {
      handle.setupCache(
          subConf,
          TaskTracker.getPublicDistributedCacheDir(),
          TaskTracker.getPrivateDistributedCacheDir(userName));
      //      handle.setupPrivateCache(localDirAllocator, TaskTracker
      //          .getPrivateDistributedCacheDir(userName));
    } catch (IOException ie) {
      th = ie;
    }
    assertNotNull("Throwable is null", th);
    assertTrue(
        "Exception message does not match",
        th.getMessage().contains("has changed on HDFS since job started"));
    // release
    handle.release();

    // submit another job
    Configuration subConf2 = new Configuration(myConf);
    subConf2.set("user.name", userName);
    DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf2);
    TrackerDistributedCacheManager.determineTimestamps(subConf2);
    TrackerDistributedCacheManager.determineCacheVisibilities(subConf2);

    handle = manager.newTaskDistributedCacheManager(new JobID("jt", 2), subConf2);
    handle.setupCache(
        subConf2,
        TaskTracker.getPublicDistributedCacheDir(),
        TaskTracker.getPrivateDistributedCacheDir(userName));
    Path[] localCacheFiles2 = DistributedCache.getLocalCacheFiles(subConf2);
    assertNotNull(null, localCacheFiles2);
    assertEquals(1, localCacheFiles2.length);
    Path cachedFirstFile2 = localCacheFiles2[0];
    assertFileLengthEquals(firstCacheFile, cachedFirstFile2);
    assertFalse("Paths should be different.", firstCacheFile.equals(cachedFirstFile2));

    // assert that two localizations point to different paths
    assertFalse(
        "two jobs with different timestamps did not localize" + " in different paths",
        cachedFirstFile.equals(cachedFirstFile2));
    // release
    handle.release();
  }
Ejemplo n.º 7
0
  /** test delete cache */
  public void testDeleteCache() throws Exception {
    if (!canRun()) {
      return;
    }
    // This test needs mapred.local.dir to be single directory
    // instead of four, because it assumes that both
    // firstcachefile and secondcachefile will be localized on same directory
    // so that second localization triggers deleteCache.
    // If mapred.local.dir is four directories, second localization might not
    // trigger deleteCache, if it is localized in different directory.
    Configuration conf2 = new Configuration(conf);
    conf2.set("mapred.local.dir", ROOT_MAPRED_LOCAL_DIR.toString());
    conf2.setLong("local.cache.size", LOCAL_CACHE_LIMIT);

    refreshConf(conf2);
    TrackerDistributedCacheManager manager =
        new TrackerDistributedCacheManager(conf2, taskController);
    FileSystem localfs = FileSystem.getLocal(conf2);
    long now = System.currentTimeMillis();
    String userName = getJobOwnerName();
    conf2.set("user.name", userName);

    // We first test the size limit
    FileStatus stat = fs.getFileStatus(firstCacheFilePublic);
    CacheFile cfile1 =
        new CacheFile(
            firstCacheFilePublic.toUri(),
            CacheFile.FileType.REGULAR,
            true,
            stat.getModificationTime(),
            true);
    Path firstLocalCache =
        manager.getLocalCache(
            firstCacheFilePublic.toUri(),
            conf2,
            TaskTracker.getPrivateDistributedCacheDir(userName),
            fs.getFileStatus(firstCacheFilePublic),
            false,
            fs.getFileStatus(firstCacheFilePublic).getModificationTime(),
            true,
            cfile1);
    manager.releaseCache(cfile1.getStatus());
    // in above code,localized a file of size 4K and then release the cache
    // which will cause the cache be deleted when the limit goes out.
    // The below code localize another cache which's designed to
    // sweep away the first cache.
    stat = fs.getFileStatus(secondCacheFilePublic);
    CacheFile cfile2 =
        new CacheFile(
            secondCacheFilePublic.toUri(),
            CacheFile.FileType.REGULAR,
            true,
            stat.getModificationTime(),
            true);
    assertTrue(
        "DistributedCache currently doesn't have cached file", localfs.exists(firstLocalCache));
    Path secondLocalCache =
        manager.getLocalCache(
            secondCacheFilePublic.toUri(),
            conf2,
            TaskTracker.getPrivateDistributedCacheDir(userName),
            fs.getFileStatus(secondCacheFilePublic),
            false,
            fs.getFileStatus(secondCacheFilePublic).getModificationTime(),
            true,
            cfile2);
    assertFalse(
        "DistributedCache failed deleting old" + " cache when the cache store is full.",
        localfs.exists(firstLocalCache));

    // find the root directory of distributed caches
    Path firstCursor = firstLocalCache;
    Path secondCursor = secondLocalCache;

    while (!firstCursor.equals(secondCursor)) {
      // Debug code, to see what these things look like
      System.err.println("cursors: " + firstCursor);
      System.err.println(" and " + secondCursor);

      firstCursor = firstCursor.getParent();
      secondCursor = secondCursor.getParent();
    }

    System.err.println("The final cursor is " + firstCursor);

    System.err.println(
        "That directory ends up with "
            + localfs.listStatus(firstCursor).length
            + " subdirectories");
    Path cachesBase = firstCursor;

    assertFalse(
        "DistributedCache did not delete the gensym'ed distcache "
            + "directory names when it deleted the files they contained "
            + "because they collectively exceeded the size limit.",
        localfs.listStatus(cachesBase).length > 1);
    conf2.setLong("local.cache.size", LOCAL_CACHE_LIMIT * 10);
    conf2.setLong("mapreduce.tasktracker.local.cache.numberdirectories", LOCAL_CACHE_SUBDIR_LIMIT);
    manager = new TrackerDistributedCacheManager(conf2, taskController);
    // Now we test the number of sub directories limit
    // Create the temporary cache files to be used in the tests.
    Path thirdCacheFile = new Path(TEST_ROOT_DIR, "thirdcachefile");
    Path fourthCacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile");
    // Adding two more small files, so it triggers the number of sub directory
    // limit but does not trigger the file size limit.
    createPrivateTempFile(thirdCacheFile);
    createPrivateTempFile(fourthCacheFile);
    DistributedCache.setCacheFiles(new URI[] {thirdCacheFile.toUri()}, conf2);
    TrackerDistributedCacheManager.determineCacheVisibilities(conf2);
    TrackerDistributedCacheManager.determineTimestamps(conf2);
    stat = fs.getFileStatus(thirdCacheFile);
    CacheFile cfile3 =
        new CacheFile(
            thirdCacheFile.toUri(),
            CacheFile.FileType.REGULAR,
            false,
            stat.getModificationTime(),
            true);
    Path thirdLocalCache =
        manager.getLocalCache(
            thirdCacheFile.toUri(),
            conf2,
            TaskTracker.getPrivateDistributedCacheDir(userName),
            fs.getFileStatus(thirdCacheFile),
            false,
            fs.getFileStatus(thirdCacheFile).getModificationTime(),
            false,
            cfile3);
    DistributedCache.setLocalFiles(conf2, thirdLocalCache.toString());
    JobLocalizer.downloadPrivateCache(conf2);
    // Release the third cache so that it can be deleted while sweeping
    manager.releaseCache(cfile3.getStatus());
    // Getting the fourth cache will make the number of sub directories becomes
    // 3 which is greater than 2. So the released cache will be deleted.
    stat = fs.getFileStatus(fourthCacheFile);
    CacheFile cfile4 =
        new CacheFile(
            fourthCacheFile.toUri(),
            CacheFile.FileType.REGULAR,
            false,
            stat.getModificationTime(),
            true);
    assertTrue(
        "DistributedCache currently doesn't have cached file", localfs.exists(thirdLocalCache));

    DistributedCache.setCacheFiles(new URI[] {fourthCacheFile.toUri()}, conf2);
    DistributedCache.setLocalFiles(conf2, thirdCacheFile.toUri().toString());
    TrackerDistributedCacheManager.determineCacheVisibilities(conf2);
    TrackerDistributedCacheManager.determineTimestamps(conf2);
    Path fourthLocalCache =
        manager.getLocalCache(
            fourthCacheFile.toUri(),
            conf2,
            TaskTracker.getPrivateDistributedCacheDir(userName),
            fs.getFileStatus(fourthCacheFile),
            false,
            fs.getFileStatus(fourthCacheFile).getModificationTime(),
            false,
            cfile4);
    assertFalse(
        "DistributedCache failed deleting old"
            + " cache when the cache exceeds the number of sub directories limit.",
        localfs.exists(thirdLocalCache));

    assertFalse(
        "DistributedCache did not delete the gensym'ed distcache "
            + "directory names when it deleted the files they contained "
            + "because there were too many.",
        localfs.listStatus(cachesBase).length > LOCAL_CACHE_SUBDIR_LIMIT);

    // Clean up the files created in this test
    new File(thirdCacheFile.toString()).delete();
    new File(fourthCacheFile.toString()).delete();
  }
Ejemplo n.º 8
0
  public void testReferenceCount()
      throws IOException, LoginException, URISyntaxException, InterruptedException {
    if (!canRun()) {
      return;
    }
    TrackerDistributedCacheManager manager = new FakeTrackerDistributedCacheManager(conf);

    String userName = getJobOwnerName();
    File workDir = new File(new Path(TEST_ROOT_DIR, "workdir").toString());

    // Configures a job with a regular file
    Job job1 = new Job(conf);
    Configuration conf1 = job1.getConfiguration();
    conf1.set("user.name", userName);
    DistributedCache.addCacheFile(secondCacheFile.toUri(), conf1);

    TrackerDistributedCacheManager.determineTimestamps(conf1);
    TrackerDistributedCacheManager.determineCacheVisibilities(conf1);

    // Task localizing for first job
    TaskDistributedCacheManager handle =
        manager.newTaskDistributedCacheManager(new JobID("jt", 1), conf1);
    handle.setupCache(
        conf1,
        TaskTracker.getPublicDistributedCacheDir(),
        TaskTracker.getPrivateDistributedCacheDir(userName));
    JobLocalizer.downloadPrivateCache(conf1);
    handle.release();
    for (TaskDistributedCacheManager.CacheFile c : handle.getCacheFiles()) {
      assertEquals(0, manager.getReferenceCount(c.getStatus()));
    }

    Path thirdCacheFile = new Path(TEST_ROOT_DIR, "thirdcachefile");
    createPrivateTempFile(thirdCacheFile);

    // Configures another job with three regular files.
    Job job2 = new Job(conf);
    Configuration conf2 = job2.getConfiguration();
    conf2.set("user.name", userName);
    // add a file that would get failed to localize
    DistributedCache.addCacheFile(firstCacheFilePublic.toUri(), conf2);
    // add a file that is already localized by different job
    DistributedCache.addCacheFile(secondCacheFile.toUri(), conf2);
    // add a file that is never localized
    DistributedCache.addCacheFile(thirdCacheFile.toUri(), conf2);

    TrackerDistributedCacheManager.determineTimestamps(conf2);
    TrackerDistributedCacheManager.determineCacheVisibilities(conf2);

    // Task localizing for second job
    // localization for the "firstCacheFile" will fail.
    handle = manager.newTaskDistributedCacheManager(new JobID("jt", 2), conf2);
    Throwable th = null;
    try {
      handle.setupCache(
          conf2,
          TaskTracker.getPublicDistributedCacheDir(),
          TaskTracker.getPrivateDistributedCacheDir(userName));
      JobLocalizer.downloadPrivateCache(conf2);
    } catch (IOException e) {
      th = e;
      LOG.info("Exception during setup", e);
    }
    assertNotNull(th);
    assertTrue(th.getMessage().contains("fake fail"));
    handle.release();
    th = null;
    for (TaskDistributedCacheManager.CacheFile c : handle.getCacheFiles()) {
      try {
        int refcount = manager.getReferenceCount(c.getStatus());
        LOG.info("checking refcount " + c.uri + " of " + refcount);
        assertEquals(0, refcount);
      } catch (NullPointerException ie) {
        th = ie;
        LOG.info("Exception getting reference count for " + c.uri, ie);
      }
    }
    assertNotNull(th);
    fs.delete(thirdCacheFile, false);
  }