private void checkLocalizedPath(boolean visibility) throws IOException, LoginException, InterruptedException { TrackerDistributedCacheManager manager = new TrackerDistributedCacheManager(conf, taskController); String userName = getJobOwnerName(); File workDir = new File(TEST_ROOT_DIR, "workdir"); Path cacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile"); if (visibility) { createPublicTempFile(cacheFile); } else { createPrivateTempFile(cacheFile); } Configuration conf1 = new Configuration(conf); conf1.set("user.name", userName); DistributedCache.addCacheFile(cacheFile.toUri(), conf1); TrackerDistributedCacheManager.determineTimestamps(conf1); TrackerDistributedCacheManager.determineCacheVisibilities(conf1); dumpState(conf1); // Task localizing for job TaskDistributedCacheManager handle = manager.newTaskDistributedCacheManager(new JobID("jt", 1), conf1); handle.setupCache( conf1, TaskTracker.getPublicDistributedCacheDir(), TaskTracker.getPrivateDistributedCacheDir(userName)); JobLocalizer.downloadPrivateCache(conf1); TaskDistributedCacheManager.CacheFile c = handle.getCacheFiles().get(0); String distCacheDir; if (visibility) { distCacheDir = TaskTracker.getPublicDistributedCacheDir(); } else { distCacheDir = TaskTracker.getPrivateDistributedCacheDir(userName); } Path localizedPath = manager.getLocalCache( cacheFile.toUri(), conf1, distCacheDir, fs.getFileStatus(cacheFile), false, c.timestamp, visibility, c); assertTrue( "Cache file didn't get localized in the expected directory. " + "Expected localization to happen within " + ROOT_MAPRED_LOCAL_DIR + "/" + distCacheDir + ", but was localized at " + localizedPath, localizedPath.toString().contains(distCacheDir)); if (visibility) { checkPublicFilePermissions(new Path[] {localizedPath}); } else { checkFilePermissions(new Path[] {localizedPath}); } }
public void testFileSystemOtherThanDefault() throws Exception { if (!canRun()) { return; } TrackerDistributedCacheManager manager = new TrackerDistributedCacheManager(conf, taskController); conf.set("fs.fakefile.impl", conf.get("fs.file.impl")); String userName = getJobOwnerName(); conf.set("user.name", userName); Path fileToCache = new Path("fakefile:///" + firstCacheFile.toUri().getPath()); CacheFile file = new CacheFile(fileToCache.toUri(), CacheFile.FileType.REGULAR, false, 0, false); Path result = manager.getLocalCache( fileToCache.toUri(), conf, TaskTracker.getPrivateDistributedCacheDir(userName), fs.getFileStatus(firstCacheFile), false, System.currentTimeMillis(), false, file); assertNotNull("DistributedCache cached file on non-default filesystem.", result); }
/** * Localize a file. After localization is complete, create a file, "myFile", under the directory * where the file is localized and ensure that it has permissions different from what is set by * default. Then, localize another file. Verify that "myFile" has the right permissions. * * @throws Exception */ public void testCustomPermissions() throws Exception { if (!canRun()) { return; } String userName = getJobOwnerName(); conf.set("user.name", userName); TrackerDistributedCacheManager manager = new TrackerDistributedCacheManager(conf, taskController); FileSystem localfs = FileSystem.getLocal(conf); long now = System.currentTimeMillis(); Path[] localCache = new Path[2]; FileStatus stat = fs.getFileStatus(firstCacheFile); CacheFile file = new CacheFile( firstCacheFilePublic.toUri(), CacheFile.FileType.REGULAR, true, stat.getModificationTime(), false); localCache[0] = manager.getLocalCache( firstCacheFilePublic.toUri(), conf, TaskTracker.getPrivateDistributedCacheDir(userName), fs.getFileStatus(firstCacheFilePublic), false, fs.getFileStatus(firstCacheFilePublic).getModificationTime(), true, file); FsPermission myPermission = new FsPermission((short) 0600); Path myFile = new Path(localCache[0].getParent(), "myfile.txt"); if (FileSystem.create(localfs, myFile, myPermission) == null) { throw new IOException("Could not create " + myFile); } try { stat = fs.getFileStatus(secondCacheFilePublic); file = new CacheFile( secondCacheFilePublic.toUri(), CacheFile.FileType.REGULAR, true, stat.getModificationTime(), false); localCache[1] = manager.getLocalCache( secondCacheFilePublic.toUri(), conf, TaskTracker.getPrivateDistributedCacheDir(userName), fs.getFileStatus(secondCacheFilePublic), false, fs.getFileStatus(secondCacheFilePublic).getModificationTime(), true, file); stat = localfs.getFileStatus(myFile); assertTrue(stat.getPermission().equals(myPermission)); // validate permissions of localized files. checkFilePermissions(localCache); } finally { localfs.delete(myFile, false); } }
/** test delete cache */ public void testDeleteCache() throws Exception { if (!canRun()) { return; } // This test needs mapred.local.dir to be single directory // instead of four, because it assumes that both // firstcachefile and secondcachefile will be localized on same directory // so that second localization triggers deleteCache. // If mapred.local.dir is four directories, second localization might not // trigger deleteCache, if it is localized in different directory. Configuration conf2 = new Configuration(conf); conf2.set("mapred.local.dir", ROOT_MAPRED_LOCAL_DIR.toString()); conf2.setLong("local.cache.size", LOCAL_CACHE_LIMIT); refreshConf(conf2); TrackerDistributedCacheManager manager = new TrackerDistributedCacheManager(conf2, taskController); FileSystem localfs = FileSystem.getLocal(conf2); long now = System.currentTimeMillis(); String userName = getJobOwnerName(); conf2.set("user.name", userName); // We first test the size limit FileStatus stat = fs.getFileStatus(firstCacheFilePublic); CacheFile cfile1 = new CacheFile( firstCacheFilePublic.toUri(), CacheFile.FileType.REGULAR, true, stat.getModificationTime(), true); Path firstLocalCache = manager.getLocalCache( firstCacheFilePublic.toUri(), conf2, TaskTracker.getPrivateDistributedCacheDir(userName), fs.getFileStatus(firstCacheFilePublic), false, fs.getFileStatus(firstCacheFilePublic).getModificationTime(), true, cfile1); manager.releaseCache(cfile1.getStatus()); // in above code,localized a file of size 4K and then release the cache // which will cause the cache be deleted when the limit goes out. // The below code localize another cache which's designed to // sweep away the first cache. stat = fs.getFileStatus(secondCacheFilePublic); CacheFile cfile2 = new CacheFile( secondCacheFilePublic.toUri(), CacheFile.FileType.REGULAR, true, stat.getModificationTime(), true); assertTrue( "DistributedCache currently doesn't have cached file", localfs.exists(firstLocalCache)); Path secondLocalCache = manager.getLocalCache( secondCacheFilePublic.toUri(), conf2, TaskTracker.getPrivateDistributedCacheDir(userName), fs.getFileStatus(secondCacheFilePublic), false, fs.getFileStatus(secondCacheFilePublic).getModificationTime(), true, cfile2); assertFalse( "DistributedCache failed deleting old" + " cache when the cache store is full.", localfs.exists(firstLocalCache)); // find the root directory of distributed caches Path firstCursor = firstLocalCache; Path secondCursor = secondLocalCache; while (!firstCursor.equals(secondCursor)) { // Debug code, to see what these things look like System.err.println("cursors: " + firstCursor); System.err.println(" and " + secondCursor); firstCursor = firstCursor.getParent(); secondCursor = secondCursor.getParent(); } System.err.println("The final cursor is " + firstCursor); System.err.println( "That directory ends up with " + localfs.listStatus(firstCursor).length + " subdirectories"); Path cachesBase = firstCursor; assertFalse( "DistributedCache did not delete the gensym'ed distcache " + "directory names when it deleted the files they contained " + "because they collectively exceeded the size limit.", localfs.listStatus(cachesBase).length > 1); conf2.setLong("local.cache.size", LOCAL_CACHE_LIMIT * 10); conf2.setLong("mapreduce.tasktracker.local.cache.numberdirectories", LOCAL_CACHE_SUBDIR_LIMIT); manager = new TrackerDistributedCacheManager(conf2, taskController); // Now we test the number of sub directories limit // Create the temporary cache files to be used in the tests. Path thirdCacheFile = new Path(TEST_ROOT_DIR, "thirdcachefile"); Path fourthCacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile"); // Adding two more small files, so it triggers the number of sub directory // limit but does not trigger the file size limit. createPrivateTempFile(thirdCacheFile); createPrivateTempFile(fourthCacheFile); DistributedCache.setCacheFiles(new URI[] {thirdCacheFile.toUri()}, conf2); TrackerDistributedCacheManager.determineCacheVisibilities(conf2); TrackerDistributedCacheManager.determineTimestamps(conf2); stat = fs.getFileStatus(thirdCacheFile); CacheFile cfile3 = new CacheFile( thirdCacheFile.toUri(), CacheFile.FileType.REGULAR, false, stat.getModificationTime(), true); Path thirdLocalCache = manager.getLocalCache( thirdCacheFile.toUri(), conf2, TaskTracker.getPrivateDistributedCacheDir(userName), fs.getFileStatus(thirdCacheFile), false, fs.getFileStatus(thirdCacheFile).getModificationTime(), false, cfile3); DistributedCache.setLocalFiles(conf2, thirdLocalCache.toString()); JobLocalizer.downloadPrivateCache(conf2); // Release the third cache so that it can be deleted while sweeping manager.releaseCache(cfile3.getStatus()); // Getting the fourth cache will make the number of sub directories becomes // 3 which is greater than 2. So the released cache will be deleted. stat = fs.getFileStatus(fourthCacheFile); CacheFile cfile4 = new CacheFile( fourthCacheFile.toUri(), CacheFile.FileType.REGULAR, false, stat.getModificationTime(), true); assertTrue( "DistributedCache currently doesn't have cached file", localfs.exists(thirdLocalCache)); DistributedCache.setCacheFiles(new URI[] {fourthCacheFile.toUri()}, conf2); DistributedCache.setLocalFiles(conf2, thirdCacheFile.toUri().toString()); TrackerDistributedCacheManager.determineCacheVisibilities(conf2); TrackerDistributedCacheManager.determineTimestamps(conf2); Path fourthLocalCache = manager.getLocalCache( fourthCacheFile.toUri(), conf2, TaskTracker.getPrivateDistributedCacheDir(userName), fs.getFileStatus(fourthCacheFile), false, fs.getFileStatus(fourthCacheFile).getModificationTime(), false, cfile4); assertFalse( "DistributedCache failed deleting old" + " cache when the cache exceeds the number of sub directories limit.", localfs.exists(thirdLocalCache)); assertFalse( "DistributedCache did not delete the gensym'ed distcache " + "directory names when it deleted the files they contained " + "because there were too many.", localfs.listStatus(cachesBase).length > LOCAL_CACHE_SUBDIR_LIMIT); // Clean up the files created in this test new File(thirdCacheFile.toString()).delete(); new File(fourthCacheFile.toString()).delete(); }