int list(FileSystem fs) throws IOException {
int totalFiles = 0;
for (FileStatus s : fs.listStatus(dir)) {
FileSystem.LOG.info("" + s.getPath());
totalFiles++;
}
return totalFiles;
}
static void createTempFile(Path p, int size) throws IOException {
File f = new File(p.toString());
FileOutputStream os = new FileOutputStream(f);
byte[] toWrite = new byte[size];
new Random().nextBytes(toWrite);
os.write(toWrite);
os.close();
FileSystem.LOG.info("created: " + p + ", size=" + size);
}
static MyFile createFile(Path root, FileSystem fs, int levels) throws IOException {
MyFile f = levels < 0 ? new MyFile() : new MyFile(levels);
Path p = new Path(root, f.getName());
FSDataOutputStream out = fs.create(p);
byte[] toWrite = new byte[f.getSize()];
new Random(f.getSeed()).nextBytes(toWrite);
out.write(toWrite);
out.close();
FileSystem.LOG.info("created: " + p + ", size=" + f.getSize());
return f;
}
public void testCopySingleFile() throws Exception {
FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration());
Path root = new Path(TEST_ROOT_DIR + "/srcdat");
try {
MyFile[] files = {createFile(root, fs)};
// copy a dir with a single file
ToolRunner.run(
new DistCpV1(new Configuration()),
new String[] {
"file:///" + TEST_ROOT_DIR + "/srcdat", "file:///" + TEST_ROOT_DIR + "/destdat"
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(fs, TEST_ROOT_DIR + "/destdat", files));
// copy a single file
String fname = files[0].getName();
Path p = new Path(root, fname);
FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p));
ToolRunner.run(
new DistCpV1(new Configuration()),
new String[] {
"file:///" + TEST_ROOT_DIR + "/srcdat/" + fname,
"file:///" + TEST_ROOT_DIR + "/dest2/" + fname
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(fs, TEST_ROOT_DIR + "/dest2", files));
// single file update should skip copy if destination has the file already
String[] args = {
"-update",
"file:///" + TEST_ROOT_DIR + "/srcdat/" + fname,
"file:///" + TEST_ROOT_DIR + "/dest2/" + fname
};
Configuration conf = new Configuration();
JobConf job = new JobConf(conf, DistCpV1.class);
DistCpV1.Arguments distcpArgs = DistCpV1.Arguments.valueOf(args, conf);
assertFalse(
"Single file update failed to skip copying even though the "
+ "file exists at destination.",
DistCpV1.setup(conf, job, distcpArgs));
// copy single file to existing dir
deldir(fs, TEST_ROOT_DIR + "/dest2");
fs.mkdirs(new Path(TEST_ROOT_DIR + "/dest2"));
MyFile[] files2 = {createFile(root, fs, 0)};
String sname = files2[0].getName();
ToolRunner.run(
new DistCpV1(new Configuration()),
new String[] {
"-update",
"file:///" + TEST_ROOT_DIR + "/srcdat/" + sname,
"file:///" + TEST_ROOT_DIR + "/dest2/"
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(fs, TEST_ROOT_DIR + "/dest2", files2));
updateFiles(fs, TEST_ROOT_DIR + "/srcdat", files2, 1);
// copy single file to existing dir w/ dst name conflict
ToolRunner.run(
new DistCpV1(new Configuration()),
new String[] {
"-update",
"file:///" + TEST_ROOT_DIR + "/srcdat/" + sname,
"file:///" + TEST_ROOT_DIR + "/dest2/"
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(fs, TEST_ROOT_DIR + "/dest2", files2));
} finally {
deldir(fs, TEST_ROOT_DIR + "/destdat");
deldir(fs, TEST_ROOT_DIR + "/dest2");
deldir(fs, TEST_ROOT_DIR + "/srcdat");
}
}