// the method which actually copies the caches locally and unjars/unzips them
// and does chmod for the files
private static Path localizeCache(
Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, boolean isArchive)
throws IOException {
FileSystem fs = getFileSystem(cache, conf);
FileSystem localFs = FileSystem.getLocal(conf);
Path parchive = null;
if (isArchive) {
parchive =
new Path(
cacheStatus.localizedLoadPath, new Path(cacheStatus.localizedLoadPath.getName()));
} else {
parchive = cacheStatus.localizedLoadPath;
}
if (!localFs.mkdirs(parchive.getParent())) {
throw new IOException(
"Mkdirs failed to create directory " + cacheStatus.localizedLoadPath.toString());
}
String cacheId = cache.getPath();
fs.copyToLocalFile(new Path(cacheId), parchive);
if (isArchive) {
String tmpArchive = parchive.toString().toLowerCase();
File srcFile = new File(parchive.toString());
File destDir = new File(parchive.getParent().toString());
if (tmpArchive.endsWith(".jar")) {
RunJar.unJar(srcFile, destDir);
} else if (tmpArchive.endsWith(".zip")) {
FileUtil.unZip(srcFile, destDir);
} else if (isTarFile(tmpArchive)) {
FileUtil.unTar(srcFile, destDir);
}
// else will not do anyhting
// and copy the file into the dir as it is
}
long cacheSize = FileUtil.getDU(new File(parchive.getParent().toString()));
cacheStatus.size = cacheSize;
addCacheInfoUpdate(cacheStatus);
// do chmod here
try {
// Setting recursive permission to grant everyone read and execute
Path localDir = new Path(cacheStatus.localizedBaseDir, cacheStatus.uniqueParentDir);
LOG.info("Doing chmod on localdir :" + localDir);
FileUtil.chmod(localDir.toString(), "ugo+rx", true);
} catch (InterruptedException e) {
LOG.warn("Exception in chmod" + e.toString());
}
// update cacheStatus to reflect the newly cached file
cacheStatus.mtime = getTimestamp(conf, cache);
return cacheStatus.localizedLoadPath;
}
/**
* Clear the entire contents of the cache and delete the backing files. This should only be used
* when the server is reinitializing, because the users are going to lose their files.
*/
public static void purgeCache(Configuration conf, MRAsyncDiskService service) throws IOException {
synchronized (cachedArchives) {
LocalFileSystem localFs = FileSystem.getLocal(conf);
for (Map.Entry<String, CacheStatus> f : cachedArchives.entrySet()) {
try {
deleteLocalPath(service, localFs, f.getValue().localizedLoadPath);
} catch (IOException ie) {
LOG.debug("Error cleaning up cache", ie);
}
}
cachedArchives.clear();
}
}
public void testcheckOutputSpecsForbidRecordCompression() throws IOException {
Job job = Job.getInstance(new Configuration(), "testcheckOutputSpecsForbidRecordCompression");
FileSystem fs = FileSystem.getLocal(job.getConfiguration());
Path outputdir = new Path(System.getProperty("test.build.data", "/tmp") + "/output");
fs.delete(outputdir, true);
// Without outputpath, FileOutputFormat.checkoutputspecs will throw
// InvalidJobConfException
FileOutputFormat.setOutputPath(job, outputdir);
// SequenceFileAsBinaryOutputFormat doesn't support record compression
// It should throw an exception when checked by checkOutputSpecs
SequenceFileAsBinaryOutputFormat.setCompressOutput(job, true);
SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);
try {
new SequenceFileAsBinaryOutputFormat().checkOutputSpecs(job);
} catch (Exception e) {
fail(
"Block compression should be allowed for "
+ "SequenceFileAsBinaryOutputFormat:Caught "
+ e.getClass().getName());
}
SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.RECORD);
try {
new SequenceFileAsBinaryOutputFormat().checkOutputSpecs(job);
fail("Record compression should not be allowed for " + "SequenceFileAsBinaryOutputFormat");
} catch (InvalidJobConfException ie) {
// expected
} catch (Exception e) {
fail(
"Expected "
+ InvalidJobConfException.class.getName()
+ "but caught "
+ e.getClass().getName());
}
}
/*
* Returns the relative path of the dir this cache will be localized in
* relative path that this cache will be localized in. For
* hdfs://hostname:port/absolute_path -- the relative path is
* hostname/absolute path -- if it is just /absolute_path -- then the
* relative path is hostname of DFS this mapred cluster is running
* on/absolute_path
*/
public static String makeRelative(URI cache, Configuration conf) throws IOException {
String host = cache.getHost();
if (host == null) {
host = cache.getScheme();
}
if (host == null) {
URI defaultUri = FileSystem.get(conf).getUri();
host = defaultUri.getHost();
if (host == null) {
host = defaultUri.getScheme();
}
}
String path = host + cache.getPath();
path = path.replace(":/", "/"); // remove windows device colon
return path;
}
private static Path checkCacheStatusValidity(
Configuration conf,
URI cache,
long confFileStamp,
CacheStatus cacheStatus,
FileStatus fileStatus,
boolean isArchive)
throws IOException {
FileSystem fs = FileSystem.get(cache, conf);
// Has to be
if (!ifExistsAndFresh(conf, fs, cache, confFileStamp, cacheStatus, fileStatus)) {
throw new IOException(
"Stale cache file: " + cacheStatus.localizedLoadPath + " for cache-file: " + cache);
}
LOG.info(
String.format(
"Using existing cache of %s->%s", cache.toString(), cacheStatus.localizedLoadPath));
return cacheStatus.localizedLoadPath;
}
private static void deleteCache(Configuration conf, MRAsyncDiskService asyncDiskService)
throws IOException {
List<CacheStatus> deleteSet = new LinkedList<CacheStatus>();
// try deleting cache Status with refcount of zero
synchronized (cachedArchives) {
for (Iterator<String> it = cachedArchives.keySet().iterator(); it.hasNext(); ) {
String cacheId = (String) it.next();
CacheStatus lcacheStatus = cachedArchives.get(cacheId);
if (lcacheStatus.refcount == 0) {
// delete this cache entry from the global list
// and mark the localized file for deletion
deleteSet.add(lcacheStatus);
it.remove();
}
}
}
// do the deletion asynchronously, after releasing the global lock
Thread cacheFileCleaner =
new Thread(new CacheFileCleanTask(asyncDiskService, FileSystem.getLocal(conf), deleteSet));
cacheFileCleaner.start();
}
public void testFormat() throws Exception {
JobConf job = new JobConf(conf);
FileSystem fs = FileSystem.getLocal(conf);
Path dir = new Path(System.getProperty("test.build.data", ".") + "/mapred");
Path file = new Path(dir, "test.seq");
Reporter reporter = Reporter.NULL;
int seed = new Random().nextInt();
// LOG.info("seed = "+seed);
Random random = new Random(seed);
fs.delete(dir, true);
FileInputFormat.setInputPaths(job, dir);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
// LOG.info("creating; entries = " + length);
// create a file with length entries
SequenceFile.Writer writer =
SequenceFile.createWriter(fs, conf, file, IntWritable.class, BytesWritable.class);
try {
for (int i = 0; i < length; i++) {
IntWritable key = new IntWritable(i);
byte[] data = new byte[random.nextInt(10)];
random.nextBytes(data);
BytesWritable value = new BytesWritable(data);
writer.append(key, value);
}
} finally {
writer.close();
}
// try splitting the file in a variety of sizes
InputFormat<IntWritable, BytesWritable> format =
new SequenceFileInputFormat<IntWritable, BytesWritable>();
IntWritable key = new IntWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < 3; i++) {
int numSplits = random.nextInt(MAX_LENGTH / (SequenceFile.SYNC_INTERVAL / 20)) + 1;
// LOG.info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(job, numSplits);
// LOG.info("splitting: got = " + splits.length);
// check each split
BitSet bits = new BitSet(length);
for (int j = 0; j < splits.length; j++) {
RecordReader<IntWritable, BytesWritable> reader =
format.getRecordReader(splits[j], job, reporter);
try {
int count = 0;
while (reader.next(key, value)) {
// if (bits.get(key.get())) {
// LOG.info("splits["+j+"]="+splits[j]+" : " +
// key.get());
// LOG.info("@"+reader.getPos());
// }
assertFalse("Key in multiple partitions.", bits.get(key.get()));
bits.set(key.get());
count++;
}
// LOG.info("splits["+j+"]="+splits[j]+" count=" +
// count);
} finally {
reader.close();
}
}
assertEquals("Some keys in no partition.", length, bits.cardinality());
}
}
}
private static FileSystem getFileSystem(URI cache, Configuration conf) throws IOException {
String fileSysName = getFileSysName(cache);
if (fileSysName != null) return FileSystem.getNamed(fileSysName, conf);
else return FileSystem.get(conf);
}
/**
* Returns the status of a given cache file on hdfs.
*
* @param conf configuration
* @param cache cache file
* @return FileStatus object of the file
* @throws IOException
*/
public static FileStatus getFileStatus(Configuration conf, URI cache) throws IOException {
FileSystem fileSystem = FileSystem.get(cache, conf);
Path filePath = new Path(cache.getPath());
return fileSystem.getFileStatus(filePath);
}
/**
* Returns mtime of a given cache file on hdfs.
*
* @param conf configuration
* @param cache cache file
* @return mtime of a given cache file on hdfs
* @throws IOException
*/
public static long getTimestamp(Configuration conf, URI cache) throws IOException {
FileSystem fileSystem = FileSystem.get(cache, conf);
Path filePath = new Path(cache.getPath());
return fileSystem.getFileStatus(filePath).getModificationTime();
}