private File getHfdsFileToTmpFile(String hdfsPath, HdfsConfiguration configuration) {
try {
String fname = hdfsPath.substring(hdfsPath.lastIndexOf('/'));
File outputDest = File.createTempFile(fname, ".hdfs");
if (outputDest.exists()) {
outputDest.delete();
}
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
FileSystem fileSystem = hdfsInfo.getFileSystem();
FileUtil.copy(fileSystem, new Path(hdfsPath), outputDest, false, fileSystem.getConf());
try {
FileUtil.copyMerge(
fileSystem, // src
new Path(hdfsPath),
FileSystem.getLocal(new Configuration()), // dest
new Path(outputDest.toURI()),
false,
fileSystem.getConf(),
null);
} catch (IOException e) {
return outputDest;
}
return new File(outputDest, fname);
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
/**
* Tests if fullyDelete deletes (a) dangling symlink to file properly (b) dangling symlink to
* directory properly
*
* @throws IOException
*/
@Test(timeout = 30000)
public void testFullyDeleteDanglingSymlinks() throws IOException {
setupDirs();
// delete the directory tmp to make tmpDir a dangling link to dir tmp and
// to make y as a dangling link to file tmp/x
boolean ret = FileUtil.fullyDelete(tmp);
Assert.assertTrue(ret);
Assert.assertFalse(tmp.exists());
// dangling symlink to file
File link = new File(del, LINK);
Assert.assertEquals(5, del.list().length);
// Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y)
// should delete 'y' properly.
ret = FileUtil.fullyDelete(link);
Assert.assertTrue(ret);
Assert.assertEquals(4, del.list().length);
// dangling symlink to directory
File linkDir = new File(del, "tmpDir");
// Even though tmpDir is dangling symlink to tmp, fullyDelete(tmpDir) should
// delete tmpDir properly.
ret = FileUtil.fullyDelete(linkDir);
Assert.assertTrue(ret);
Assert.assertEquals(3, del.list().length);
}
@Test(timeout = 30000)
public void testListAPI() throws IOException {
setupDirs();
// Test existing files case
String[] files = FileUtil.list(partitioned);
Assert.assertEquals("Unexpected number of pre-existing files", 2, files.length);
// Test existing directory with no files case
File newDir = new File(tmp.getPath(), "test");
newDir.mkdir();
Assert.assertTrue("Failed to create test dir", newDir.exists());
files = FileUtil.list(newDir);
Assert.assertEquals("New directory unexpectedly contains files", 0, files.length);
newDir.delete();
Assert.assertFalse("Failed to delete test dir", newDir.exists());
// Test non-existing directory case, this throws
// IOException
try {
files = FileUtil.list(newDir);
Assert.fail("IOException expected on list() for non-existent dir " + newDir.toString());
} catch (IOException ioe) {
// Expected an IOException
}
}
// Mostly for setting up the symlinks. Note that when we setup the distributed
// cache, we didn't create the symlinks. This is done on a per task basis
// by the currently executing task.
public static void setupWorkDir(JobConf conf) throws IOException {
File workDir = new File(".").getAbsoluteFile();
FileUtil.fullyDelete(workDir);
if (DistributedCache.getSymlink(conf)) {
URI[] archives = DistributedCache.getCacheArchives(conf);
URI[] files = DistributedCache.getCacheFiles(conf);
Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
if (archives != null) {
for (int i = 0; i < archives.length; i++) {
String link = archives[i].getFragment();
if (link != null) {
link = workDir.toString() + Path.SEPARATOR + link;
File flink = new File(link);
if (!flink.exists()) {
FileUtil.symLink(localArchives[i].toString(), link);
}
}
}
}
if (files != null) {
for (int i = 0; i < files.length; i++) {
String link = files[i].getFragment();
if (link != null) {
link = workDir.toString() + Path.SEPARATOR + link;
File flink = new File(link);
if (!flink.exists()) {
FileUtil.symLink(localFiles[i].toString(), link);
}
}
}
}
}
File jobCacheDir = null;
if (conf.getJar() != null) {
jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
}
// create symlinks for all the files in job cache dir in current
// workingdir for streaming
try {
DistributedCache.createAllSymlink(conf, jobCacheDir, workDir);
} catch (IOException ie) {
// Do not exit even if symlinks have not been created.
LOG.warn(StringUtils.stringifyException(ie));
}
// add java.io.tmpdir given by mapred.child.tmp
String tmp = conf.get("mapred.child.tmp", "./tmp");
Path tmpDir = new Path(tmp);
// if temp directory path is not absolute
// prepend it with workDir.
if (!tmpDir.isAbsolute()) {
tmpDir = new Path(workDir.toString(), tmp);
FileSystem localFs = FileSystem.getLocal(conf);
if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
throw new IOException("Mkdirs failed to create " + tmpDir.toString());
}
}
}
/**
* Wrap a command in a shell to capture debug script's stdout and stderr to debugout.
*
* @param cmd The command and the arguments that should be run
* @param debugoutFilename The filename that stdout and stderr should be saved to.
* @return the modified command that should be run
* @throws IOException
*/
public static List<String> captureDebugOut(List<String> cmd, File debugoutFilename)
throws IOException {
String debugout = FileUtil.makeShellPath(debugoutFilename);
List<String> result = new ArrayList<String>(3);
result.add(bashCommand);
result.add("-c");
StringBuffer mergedCmd = new StringBuffer();
mergedCmd.append("exec ");
boolean isExecutable = true;
for (String s : cmd) {
if (isExecutable) {
// the executable name needs to be expressed as a shell path for
// the
// shell to find it.
mergedCmd.append(FileUtil.makeShellPath(new File(s)));
isExecutable = false;
} else {
mergedCmd.append(s);
}
mergedCmd.append(" ");
}
mergedCmd.append(" < /dev/null ");
mergedCmd.append(" >");
mergedCmd.append(debugout);
mergedCmd.append(" 2>&1 ");
result.add(mergedCmd.toString());
return result;
}
/**
* Given a Tar File as input it will untar the file in a the untar directory passed as the second
* parameter
*
* <p>This utility will untar ".tar" files and ".tar.gz","tgz" files.
*
* @param inFile The tar file as input.
* @param untarDir The untar directory where to untar the tar file.
* @throws IOException
*/
public static void unTar(File inFile, File untarDir) throws IOException {
if (!untarDir.mkdirs()) {
if (!untarDir.isDirectory()) {
throw new IOException("Mkdirs failed to create " + untarDir);
}
}
StringBuffer untarCommand = new StringBuffer();
boolean gzipped = inFile.toString().endsWith("gz");
if (gzipped) {
untarCommand.append(" gzip -dc '");
untarCommand.append(FileUtil.makeShellPath(inFile));
untarCommand.append("' | (");
}
untarCommand.append("cd '");
untarCommand.append(FileUtil.makeShellPath(untarDir));
untarCommand.append("' ; ");
untarCommand.append("tar -xf ");
if (gzipped) {
untarCommand.append(" -)");
} else {
untarCommand.append(FileUtil.makeShellPath(inFile));
}
String[] shellCmd = {"bash", "-c", untarCommand.toString()};
ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
shexec.execute();
int exitcode = shexec.getExitCode();
if (exitcode != 0) {
throw new IOException(
"Error untarring file " + inFile + ". Tar process exited with exit code " + exitcode);
}
}
private void doUntarAndVerify(File tarFile, File untarDir) throws IOException {
if (untarDir.exists() && !FileUtil.fullyDelete(untarDir)) {
throw new IOException("Could not delete directory '" + untarDir + "'");
}
FileUtil.unTar(tarFile, untarDir);
String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name";
File testFile = new File(parentDir + Path.SEPARATOR + "version");
Assert.assertTrue(testFile.exists());
Assert.assertTrue(testFile.length() == 0);
String imageDir = parentDir + Path.SEPARATOR + "image";
testFile = new File(imageDir + Path.SEPARATOR + "fsimage");
Assert.assertTrue(testFile.exists());
Assert.assertTrue(testFile.length() == 157);
String currentDir = parentDir + Path.SEPARATOR + "current";
testFile = new File(currentDir + Path.SEPARATOR + "fsimage");
Assert.assertTrue(testFile.exists());
Assert.assertTrue(testFile.length() == 4331);
testFile = new File(currentDir + Path.SEPARATOR + "edits");
Assert.assertTrue(testFile.exists());
Assert.assertTrue(testFile.length() == 1033);
testFile = new File(currentDir + Path.SEPARATOR + "fstime");
Assert.assertTrue(testFile.exists());
Assert.assertTrue(testFile.length() == 8);
}
/**
* Creates multiple directories for testing.
*
* <p>Contents of them are dir:tmp: file: x dir:del: file: x dir: dir1 : file:x dir: dir2 : file:x
* link: y to tmp/x link: tmpDir to tmp dir:partitioned: file: part-r-00000, contents: "foo" file:
* part-r-00001, contents: "bar"
*/
private void setupDirs() throws IOException {
Assert.assertFalse(del.exists());
Assert.assertFalse(tmp.exists());
Assert.assertFalse(partitioned.exists());
del.mkdirs();
tmp.mkdirs();
partitioned.mkdirs();
new File(del, FILE).createNewFile();
File tmpFile = new File(tmp, FILE);
tmpFile.createNewFile();
// create directories
dir1.mkdirs();
dir2.mkdirs();
new File(dir1, FILE).createNewFile();
new File(dir2, FILE).createNewFile();
// create a symlink to file
File link = new File(del, LINK);
FileUtil.symLink(tmpFile.toString(), link.toString());
// create a symlink to dir
File linkDir = new File(del, "tmpDir");
FileUtil.symLink(tmp.toString(), linkDir.toString());
Assert.assertEquals(5, del.listFiles().length);
// create files in partitioned directories
createFile(partitioned, "part-r-00000", "foo");
createFile(partitioned, "part-r-00001", "bar");
// create a cycle using symlinks. Cycles should be handled
FileUtil.symLink(del.toString(), dir1.toString() + "/cycle");
}
/** Test that the NN re-learns of volume failures after restart. */
@Test
public void testVolFailureStatsPreservedOnNNRestart() throws Exception {
assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
// Bring up two more datanodes that can tolerate 1 failure
cluster.startDataNodes(conf, 2, true, null, null);
cluster.waitActive();
final DatanodeManager dm = cluster.getNamesystem().getBlockManager().getDatanodeManager();
long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm);
long dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0);
// Fail the first volume on both datanodes (we have to keep the
// third healthy so one node in the pipeline will not fail).
File dn1Vol1 = new File(dataDir, "data" + (2 * 0 + 1));
File dn2Vol1 = new File(dataDir, "data" + (2 * 1 + 1));
assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, false));
assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false));
Path file1 = new Path("/test1");
DFSTestUtil.createFile(fs, file1, 1024, (short) 2, 1L);
DFSTestUtil.waitReplication(fs, file1, (short) 2);
// The NN reports two volumes failures
DFSTestUtil.waitForDatanodeStatus(
dm, 3, 0, 2, origCapacity - (1 * dnCapacity), WAIT_FOR_HEARTBEATS);
// After restarting the NN it still see the two failures
cluster.restartNameNode(0);
cluster.waitActive();
DFSTestUtil.waitForDatanodeStatus(
dm, 3, 0, 2, origCapacity - (1 * dnCapacity), WAIT_FOR_HEARTBEATS);
}
@After
public void tearDown() throws Exception {
for (int i = 0; i < 3; i++) {
FileUtil.setExecutable(new File(dataDir, "data" + (2 * i + 1)), true);
FileUtil.setExecutable(new File(dataDir, "data" + (2 * i + 2)), true);
}
cluster.shutdown();
}
private void cleanupImpl() throws IOException {
FileUtil.fullyDelete(del, true);
Assert.assertTrue(!del.exists());
FileUtil.fullyDelete(tmp, true);
Assert.assertTrue(!tmp.exists());
FileUtil.fullyDelete(partitioned, true);
Assert.assertTrue(!partitioned.exists());
}
/**
* Delete the given path to a file or directory.
*
* @param p the path to delete
* @param recursive to delete sub-directories
* @return true if the file or directory and all its contents were deleted
* @throws IOException if p is non-empty and recursive is false
*/
@Override
public boolean delete(Path p, boolean recursive) throws IOException {
File f = pathToFile(p);
if (f.isFile()) {
return f.delete();
} else if (!recursive && f.isDirectory() && (FileUtil.listFiles(f).length != 0)) {
throw new IOException("Directory " + f.toString() + " is not empty");
}
return FileUtil.fullyDelete(f);
}
static String buildCommandLine(
List<String> setup,
List<String> cmd,
File stdoutFilename,
File stderrFilename,
long tailLength,
boolean useSetSid)
throws IOException {
String stdout = FileUtil.makeShellPath(stdoutFilename);
String stderr = FileUtil.makeShellPath(stderrFilename);
StringBuilder mergedCmd = new StringBuilder();
if (!Shell.WINDOWS) {
mergedCmd.append("export JVM_PID=`echo $$`\n");
}
if (setup != null) {
for (String s : setup) {
mergedCmd.append(s);
mergedCmd.append("\n");
}
}
if (tailLength > 0) {
mergedCmd.append("(");
} else if (ProcessTree.isSetsidAvailable && useSetSid && !Shell.WINDOWS) {
mergedCmd.append("exec setsid ");
} else {
mergedCmd.append("exec ");
}
mergedCmd.append(addCommand(cmd, true));
mergedCmd.append(" < /dev/null ");
if (tailLength > 0) {
mergedCmd.append(" | ");
mergedCmd.append(tailCommand);
mergedCmd.append(" -c ");
mergedCmd.append(tailLength);
mergedCmd.append(" >> ");
mergedCmd.append(stdout);
mergedCmd.append(" ; exit $PIPESTATUS ) 2>&1 | ");
mergedCmd.append(tailCommand);
mergedCmd.append(" -c ");
mergedCmd.append(tailLength);
mergedCmd.append(" >> ");
mergedCmd.append(stderr);
mergedCmd.append(" ; exit $PIPESTATUS");
} else {
mergedCmd.append(" 1>> ");
mergedCmd.append(stdout);
mergedCmd.append(" 2>> ");
mergedCmd.append(stderr);
}
return mergedCmd.toString();
}
// the method which actually copies the caches locally and unjars/unzips them
// and does chmod for the files
private static Path localizeCache(
Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, boolean isArchive)
throws IOException {
FileSystem fs = getFileSystem(cache, conf);
FileSystem localFs = FileSystem.getLocal(conf);
Path parchive = null;
if (isArchive) {
parchive =
new Path(
cacheStatus.localizedLoadPath, new Path(cacheStatus.localizedLoadPath.getName()));
} else {
parchive = cacheStatus.localizedLoadPath;
}
if (!localFs.mkdirs(parchive.getParent())) {
throw new IOException(
"Mkdirs failed to create directory " + cacheStatus.localizedLoadPath.toString());
}
String cacheId = cache.getPath();
fs.copyToLocalFile(new Path(cacheId), parchive);
if (isArchive) {
String tmpArchive = parchive.toString().toLowerCase();
File srcFile = new File(parchive.toString());
File destDir = new File(parchive.getParent().toString());
if (tmpArchive.endsWith(".jar")) {
RunJar.unJar(srcFile, destDir);
} else if (tmpArchive.endsWith(".zip")) {
FileUtil.unZip(srcFile, destDir);
} else if (isTarFile(tmpArchive)) {
FileUtil.unTar(srcFile, destDir);
}
// else will not do anyhting
// and copy the file into the dir as it is
}
long cacheSize = FileUtil.getDU(new File(parchive.getParent().toString()));
cacheStatus.size = cacheSize;
addCacheInfoUpdate(cacheStatus);
// do chmod here
try {
// Setting recursive permission to grant everyone read and execute
Path localDir = new Path(cacheStatus.localizedBaseDir, cacheStatus.uniqueParentDir);
LOG.info("Doing chmod on localdir :" + localDir);
FileUtil.chmod(localDir.toString(), "ugo+rx", true);
} catch (InterruptedException e) {
LOG.warn("Exception in chmod" + e.toString());
}
// update cacheStatus to reflect the newly cached file
cacheStatus.mtime = getTimestamp(conf, cache);
return cacheStatus.localizedLoadPath;
}
/**
* Move replicas in the lazy persist directory to their corresponding locations in the finalized
* directory.
*
* @return number of replicas recovered.
*/
private int moveLazyPersistReplicasToFinalized(File source) throws IOException {
File files[] = FileUtil.listFiles(source);
int numRecovered = 0;
for (File file : files) {
if (file.isDirectory()) {
numRecovered += moveLazyPersistReplicasToFinalized(file);
}
if (Block.isMetaFilename(file.getName())) {
File metaFile = file;
File blockFile = Block.metaToBlockFile(metaFile);
long blockId = Block.filename2id(blockFile.getName());
File targetDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId);
if (blockFile.exists()) {
if (!targetDir.exists() && !targetDir.mkdirs()) {
LOG.warn("Failed to mkdirs " + targetDir);
continue;
}
final File targetMetaFile = new File(targetDir, metaFile.getName());
try {
NativeIO.renameTo(metaFile, targetMetaFile);
} catch (IOException e) {
LOG.warn("Failed to move meta file from " + metaFile + " to " + targetMetaFile, e);
continue;
}
final File targetBlockFile = new File(targetDir, blockFile.getName());
try {
NativeIO.renameTo(blockFile, targetBlockFile);
} catch (IOException e) {
LOG.warn("Failed to move block file from " + blockFile + " to " + targetBlockFile, e);
continue;
}
if (targetBlockFile.exists() && targetMetaFile.exists()) {
++numRecovered;
} else {
// Failure should be rare.
LOG.warn("Failed to move " + blockFile + " to " + targetDir);
}
}
}
}
FileUtil.fullyDelete(source);
return numRecovered;
}
@Test(timeout = 30000)
/*
* Test method copy(FileSystem srcFS, Path src, File dst, boolean deleteSource, Configuration conf)
*/
public void testCopy5() throws IOException {
setupDirs();
URI uri = tmp.toURI();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.newInstance(uri, conf);
final String content = "some-content";
File srcFile = createFile(tmp, "src", content);
Path srcPath = new Path(srcFile.toURI());
// copy regular file:
final File dest = new File(del, "dest");
boolean result = FileUtil.copy(fs, srcPath, dest, false, conf);
assertTrue(result);
assertTrue(dest.exists());
assertEquals(
content.getBytes().length + System.getProperty("line.separator").getBytes().length,
dest.length());
assertTrue(srcFile.exists()); // should not be deleted
// copy regular file, delete src:
dest.delete();
assertTrue(!dest.exists());
result = FileUtil.copy(fs, srcPath, dest, true, conf);
assertTrue(result);
assertTrue(dest.exists());
assertEquals(
content.getBytes().length + System.getProperty("line.separator").getBytes().length,
dest.length());
assertTrue(!srcFile.exists()); // should be deleted
// copy a dir:
dest.delete();
assertTrue(!dest.exists());
srcPath = new Path(partitioned.toURI());
result = FileUtil.copy(fs, srcPath, dest, true, conf);
assertTrue(result);
assertTrue(dest.exists() && dest.isDirectory());
File[] files = dest.listFiles();
assertTrue(files != null);
assertEquals(2, files.length);
for (File f : files) {
assertEquals(3 + System.getProperty("line.separator").getBytes().length, f.length());
}
assertTrue(!partitioned.exists()); // should be deleted
}
/**
* Test that getDU is able to handle cycles caused due to symbolic links and that directory sizes
* are not added to the final calculated size
*
* @throws IOException
*/
@Test(timeout = 30000)
public void testGetDU() throws Exception {
setupDirs();
long du = FileUtil.getDU(TEST_DIR);
// Only two files (in partitioned). Each has 3 characters + system-specific
// line separator.
final long expected = 2 * (3 + System.getProperty("line.separator").length());
Assert.assertEquals(expected, du);
// target file does not exist:
final File doesNotExist = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog");
long duDoesNotExist = FileUtil.getDU(doesNotExist);
assertEquals(0, duDoesNotExist);
// target file is not a directory:
File notADirectory = new File(partitioned, "part-r-00000");
long duNotADirectoryActual = FileUtil.getDU(notADirectory);
long duNotADirectoryExpected = 3 + System.getProperty("line.separator").length();
assertEquals(duNotADirectoryExpected, duNotADirectoryActual);
try {
// one of target files is not accessible, but the containing directory
// is accessible:
try {
FileUtil.chmod(notADirectory.getAbsolutePath(), "0000");
} catch (InterruptedException ie) {
// should never happen since that method never throws InterruptedException.
assertNull(ie);
}
assertFalse(FileUtil.canRead(notADirectory));
final long du3 = FileUtil.getDU(partitioned);
assertEquals(expected, du3);
// some target files and containing directory are not accessible:
try {
FileUtil.chmod(partitioned.getAbsolutePath(), "0000");
} catch (InterruptedException ie) {
// should never happen since that method never throws InterruptedException.
assertNull(ie);
}
assertFalse(FileUtil.canRead(partitioned));
final long du4 = FileUtil.getDU(partitioned);
assertEquals(0, du4);
} finally {
// Restore the permissions so that we can delete the folder
// in @After method:
FileUtil.chmod(partitioned.getAbsolutePath(), "0777", true /*recursive*/);
}
}
@Test(timeout = 30000)
public void testCreateLocalTempFile() throws IOException {
setupDirs();
final File baseFile = new File(tmp, "base");
File tmp1 = FileUtil.createLocalTempFile(baseFile, "foo", false);
File tmp2 = FileUtil.createLocalTempFile(baseFile, "foo", true);
assertFalse(tmp1.getAbsolutePath().equals(baseFile.getAbsolutePath()));
assertFalse(tmp2.getAbsolutePath().equals(baseFile.getAbsolutePath()));
assertTrue(tmp1.exists() && tmp2.exists());
assertTrue(tmp1.canWrite() && tmp2.canWrite());
assertTrue(tmp1.canRead() && tmp2.canRead());
tmp1.delete();
tmp2.delete();
assertTrue(!tmp1.exists() && !tmp2.exists());
}
@After
public void tearDown() throws Exception {
if (data_fail != null) {
FileUtil.setWritable(data_fail, true);
data_fail = null;
}
if (failedDir != null) {
FileUtil.setWritable(failedDir, true);
failedDir = null;
}
if (cluster != null) {
cluster.shutdown();
cluster = null;
}
}
public void testAbort() throws IOException {
JobConf job = new JobConf();
setConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, taskID.getJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.setWorkOutputPath(job, committer.getTempTaskOutputPath(tContext));
// do setup
committer.setupJob(jContext);
committer.setupTask(tContext);
String file = "test.txt";
// A reporter that does nothing
Reporter reporter = Reporter.NULL;
// write output
FileSystem localFs = FileSystem.getLocal(job);
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter);
writeOutput(theRecordWriter, reporter);
// do abort
committer.abortTask(tContext);
File expectedFile =
new File(new Path(committer.getTempTaskOutputPath(tContext), file).toString());
assertFalse("task temp dir still exists", expectedFile.exists());
committer.abortJob(jContext, JobStatus.State.FAILED);
expectedFile = new File(new Path(outDir, FileOutputCommitter.TEMP_DIR_NAME).toString());
assertFalse("job temp dir still exists", expectedFile.exists());
assertEquals("Output directory not empty", 0, new File(outDir.toString()).listFiles().length);
FileUtil.fullyDelete(new File(outDir.toString()));
}
private static String execCommand(File f, String... cmd) throws IOException {
String[] args = new String[cmd.length + 1];
System.arraycopy(cmd, 0, args, 0, cmd.length);
args[cmd.length] = FileUtil.makeShellPath(f, true);
String output = Shell.execCommand(args);
return output;
}
@Override
public boolean rename(Path src, Path dst) throws IOException {
// Attempt rename using Java API.
File srcFile = pathToFile(src);
File dstFile = pathToFile(dst);
if (srcFile.renameTo(dstFile)) {
return true;
}
// Enforce POSIX rename behavior that a source directory replaces an existing
// destination if the destination is an empty directory. On most platforms,
// this is already handled by the Java API call above. Some platforms
// (notably Windows) do not provide this behavior, so the Java API call above
// fails. Delete destination and attempt rename again.
if (this.exists(dst)) {
FileStatus sdst = this.getFileStatus(dst);
if (sdst.isDirectory() && dstFile.list().length == 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting empty destination and renaming " + src + " to " + dst);
}
if (this.delete(dst, false) && srcFile.renameTo(dstFile)) {
return true;
}
}
}
// The fallback behavior accomplishes the rename by a full copy.
if (LOG.isDebugEnabled()) {
LOG.debug("Falling through to a copy of " + src + " to " + dst);
}
return FileUtil.copy(this, src, this, dst, true, getConf());
}
/**
* Download the file from dfs to local file.
*
* @param fs
* @param destinationFile
* @param dfsFile
* @param conf
* @return
* @throws IOException
*/
public static File copyToLocalFileSystem(
FileSystem fs,
String destinationPath,
String destinationFile,
String dfsFile,
Configuration conf)
throws IOException {
File destinationDir = new File(destinationPath);
if (!destinationDir.exists() && !destinationDir.mkdirs()) {
throw new RuntimeException("Unable to create local directory");
}
try (RawLocalFileSystem localFileSystem = new RawLocalFileSystem()) {
// allow app user to access local dir
FsPermission permissions = new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE);
localFileSystem.setPermission(new Path(destinationDir.getAbsolutePath()), permissions);
Path dfsFilePath = new Path(dfsFile);
File localFile = new File(destinationDir, destinationFile);
FileUtil.copy(fs, dfsFilePath, localFile, false, conf);
// set permissions on actual file to be read-only for user
permissions = new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE);
localFileSystem.setPermission(new Path(localFile.getAbsolutePath()), permissions);
return localFile;
}
}
private String copyFromLocal(FileSystem fs, Path basePath, String[] files) throws IOException {
StringBuilder csv = new StringBuilder(files.length * (basePath.toString().length() + 16));
for (String localFile : files) {
Path src = new Path(localFile);
String filename = src.getName();
Path dst = new Path(basePath, filename);
URI localFileURI = null;
try {
localFileURI = new URI(localFile);
} catch (URISyntaxException e) {
throw new IOException(e);
}
if (localFileURI.getScheme() == null || localFileURI.getScheme().startsWith("file")) {
LOG.info("Copy {} from local filesystem to {}", localFile, dst);
fs.copyFromLocalFile(false, true, src, dst);
} else {
LOG.info("Copy {} from DFS to {}", localFile, dst);
FileUtil.copy(fs, src, fs, dst, false, true, conf);
}
if (csv.length() > 0) {
csv.append(LIB_JARS_SEP);
}
csv.append(dst.toString());
}
return csv.toString();
}
public void testComplexNameWithRegex() throws Exception {
OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt"));
Writer wr = new OutputStreamWriter(os);
wr.write("b a\n");
wr.close();
JobConf conf = createJobConf();
conf.setJobName("name \\Evalue]");
conf.setInputFormat(TextInputFormat.class);
conf.setOutputKeyClass(LongWritable.class);
conf.setOutputValueClass(Text.class);
conf.setMapperClass(IdentityMapper.class);
FileInputFormat.setInputPaths(conf, getInputDir());
FileOutputFormat.setOutputPath(conf, getOutputDir());
JobClient.runJob(conf);
Path[] outputFiles =
FileUtil.stat2Paths(getFileSystem().listStatus(getOutputDir(), new OutputLogFilter()));
assertEquals(1, outputFiles.length);
InputStream is = getFileSystem().open(outputFiles[0]);
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
assertEquals("0\tb a", reader.readLine());
assertNull(reader.readLine());
reader.close();
}
@Before
public void setup() throws Exception {
FileContext files = FileContext.getLocalFSFileContext();
Path workSpacePath = new Path(workSpace.getAbsolutePath());
files.mkdir(workSpacePath, null, true);
FileUtil.chmod(workSpace.getAbsolutePath(), "777");
File localDir = new File(workSpace.getAbsoluteFile(), "localDir");
files.mkdir(new Path(localDir.getAbsolutePath()), new FsPermission("777"), false);
File logDir = new File(workSpace.getAbsoluteFile(), "logDir");
files.mkdir(new Path(logDir.getAbsolutePath()), new FsPermission("777"), false);
String exec_path = System.getProperty("container-executor.path");
if (exec_path != null && !exec_path.isEmpty()) {
Configuration conf = new Configuration(false);
LOG.info("Setting " + YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH + "=" + exec_path);
conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, exec_path);
exec = new LinuxContainerExecutor();
exec.setConf(conf);
conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.getAbsolutePath());
dirsHandler = new LocalDirsHandlerService();
dirsHandler.init(conf);
}
appSubmitter = System.getProperty("application.submitter");
if (appSubmitter == null || appSubmitter.isEmpty()) {
appSubmitter = "nobody";
}
}
/**
* Create log directory for the given attempt. This involves creating the following and setting
* proper permissions for the new directories <br>
* {hadoop.log.dir}/userlogs/<jobid> <br>
* {hadoop.log.dir}/userlogs/<jobid>/<attempt-id-as-symlink> <br>
* {one of the mapred-local-dirs}/userlogs/<jobid> <br>
* {one of the mapred-local-dirs}/userlogs/<jobid>/<attempt-id>
*
* @param taskID attempt-id for which log dir is to be created
* @param isCleanup Is this attempt a cleanup attempt ?
* @param localDirs mapred local directories
* @throws IOException
*/
public static void createTaskAttemptLogDir(
TaskAttemptID taskID, boolean isCleanup, String[] localDirs) throws IOException {
String cleanupSuffix = isCleanup ? ".cleanup" : "";
String strAttemptLogDir = getTaskAttemptLogDir(taskID, cleanupSuffix, localDirs);
File attemptLogDir = new File(strAttemptLogDir);
if (!attemptLogDir.mkdirs()) {
throw new IOException("Creation of " + attemptLogDir + " failed.");
}
String strLinkAttemptLogDir =
getJobDir(taskID.getJobID()).getAbsolutePath()
+ File.separatorChar
+ taskID.toString()
+ cleanupSuffix;
if (FileUtil.symLink(strAttemptLogDir, strLinkAttemptLogDir) != 0) {
throw new IOException(
"Creation of symlink from "
+ strLinkAttemptLogDir
+ " to "
+ strAttemptLogDir
+ " failed.");
}
FileSystem localFs = FileSystem.getLocal(new Configuration());
localFs.setPermission(new Path(attemptLogDir.getPath()), new FsPermission((short) 0700));
}
private void testAbortInternal(int version) throws IOException, InterruptedException {
JobConf conf = new JobConf();
FileOutputFormat.setOutputPath(conf, outDir);
conf.set(JobContext.TASK_ATTEMPT_ID, attempt);
conf.setInt(
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION,
version);
JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// do setup
committer.setupJob(jContext);
committer.setupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(null, conf, partFile, null);
writeOutput(theRecordWriter, tContext);
// do abort
committer.abortTask(tContext);
File out = new File(outDir.toUri().getPath());
Path workPath = committer.getWorkPath(tContext, outDir);
File wp = new File(workPath.toUri().getPath());
File expectedFile = new File(wp, partFile);
assertFalse("task temp dir still exists", expectedFile.exists());
committer.abortJob(jContext, JobStatus.State.FAILED);
expectedFile = new File(out, FileOutputCommitter.TEMP_DIR_NAME);
assertFalse("job temp dir still exists", expectedFile.exists());
assertEquals("Output directory not empty", 0, out.listFiles().length);
FileUtil.fullyDelete(out);
}
/**
* Copy specified file into a temporary file. Then rename the temporary file to the original name.
* This will cause any hardlinks to the original file to be removed. The temporary files are
* created in the same directory. The temporary files will be recovered (especially on Windows) on
* datanode restart.
*/
private void unlinkFile(File file, Block b) throws IOException {
File tmpFile = DatanodeUtil.createTmpFile(b, DatanodeUtil.getUnlinkTmpFile(file));
try {
FileInputStream in = new FileInputStream(file);
try {
FileOutputStream out = new FileOutputStream(tmpFile);
try {
IOUtils.copyBytes(in, out, 16 * 1024);
} finally {
out.close();
}
} finally {
in.close();
}
if (file.length() != tmpFile.length()) {
throw new IOException(
"Copy of file "
+ file
+ " size "
+ file.length()
+ " into file "
+ tmpFile
+ " resulted in a size of "
+ tmpFile.length());
}
FileUtil.replaceFile(tmpFile, file);
} catch (IOException e) {
boolean done = tmpFile.delete();
if (!done) {
DataNode.LOG.info("detachFile failed to delete temporary file " + tmpFile);
}
throw e;
}
}
private void runTestOnTable(Table table)
throws IOException, InterruptedException, ClassNotFoundException {
Job job = null;
try {
LOG.info("Before map/reduce startup");
job = new Job(table.getConfiguration(), "process column contents");
job.setNumReduceTasks(1);
Scan scan = new Scan();
scan.addFamily(INPUT_FAMILY);
TableMapReduceUtil.initTableMapperJob(
table.getName(),
scan,
MultithreadedTableMapper.class,
ImmutableBytesWritable.class,
Put.class,
job);
MultithreadedTableMapper.setMapperClass(job, ProcessContentsMapper.class);
MultithreadedTableMapper.setNumberOfThreads(job, NUMBER_OF_THREADS);
TableMapReduceUtil.initTableReducerJob(
table.getName().getNameAsString(), IdentityTableReducer.class, job);
FileOutputFormat.setOutputPath(job, new Path("test"));
LOG.info("Started " + table.getName());
assertTrue(job.waitForCompletion(true));
LOG.info("After map/reduce completion");
// verify map-reduce results
verify(table.getName());
} finally {
table.close();
if (job != null) {
FileUtil.fullyDelete(new File(job.getConfiguration().get("hadoop.tmp.dir")));
}
}
}