Exemple #1
0
 int list(FileSystem fs) throws IOException {
   int totalFiles = 0;
   for (FileStatus s : fs.listStatus(dir)) {
     FileSystem.LOG.info("" + s.getPath());
     totalFiles++;
   }
   return totalFiles;
 }
Exemple #2
0
 static void createTempFile(Path p, int size) throws IOException {
   File f = new File(p.toString());
   FileOutputStream os = new FileOutputStream(f);
   byte[] toWrite = new byte[size];
   new Random().nextBytes(toWrite);
   os.write(toWrite);
   os.close();
   FileSystem.LOG.info("created: " + p + ", size=" + size);
 }
 static MyFile createFile(Path root, FileSystem fs, int levels) throws IOException {
   MyFile f = levels < 0 ? new MyFile() : new MyFile(levels);
   Path p = new Path(root, f.getName());
   FSDataOutputStream out = fs.create(p);
   byte[] toWrite = new byte[f.getSize()];
   new Random(f.getSeed()).nextBytes(toWrite);
   out.write(toWrite);
   out.close();
   FileSystem.LOG.info("created: " + p + ", size=" + f.getSize());
   return f;
 }
  public void testCopySingleFile() throws Exception {
    FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration());
    Path root = new Path(TEST_ROOT_DIR + "/srcdat");
    try {
      MyFile[] files = {createFile(root, fs)};
      // copy a dir with a single file
      ToolRunner.run(
          new DistCpV1(new Configuration()),
          new String[] {
            "file:///" + TEST_ROOT_DIR + "/srcdat", "file:///" + TEST_ROOT_DIR + "/destdat"
          });
      assertTrue(
          "Source and destination directories do not match.",
          checkFiles(fs, TEST_ROOT_DIR + "/destdat", files));

      // copy a single file
      String fname = files[0].getName();
      Path p = new Path(root, fname);
      FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p));
      ToolRunner.run(
          new DistCpV1(new Configuration()),
          new String[] {
            "file:///" + TEST_ROOT_DIR + "/srcdat/" + fname,
            "file:///" + TEST_ROOT_DIR + "/dest2/" + fname
          });
      assertTrue(
          "Source and destination directories do not match.",
          checkFiles(fs, TEST_ROOT_DIR + "/dest2", files));

      // single file update should skip copy if destination has the file already
      String[] args = {
        "-update",
        "file:///" + TEST_ROOT_DIR + "/srcdat/" + fname,
        "file:///" + TEST_ROOT_DIR + "/dest2/" + fname
      };
      Configuration conf = new Configuration();
      JobConf job = new JobConf(conf, DistCpV1.class);
      DistCpV1.Arguments distcpArgs = DistCpV1.Arguments.valueOf(args, conf);
      assertFalse(
          "Single file update failed to skip copying even though the "
              + "file exists at destination.",
          DistCpV1.setup(conf, job, distcpArgs));

      // copy single file to existing dir
      deldir(fs, TEST_ROOT_DIR + "/dest2");
      fs.mkdirs(new Path(TEST_ROOT_DIR + "/dest2"));
      MyFile[] files2 = {createFile(root, fs, 0)};
      String sname = files2[0].getName();
      ToolRunner.run(
          new DistCpV1(new Configuration()),
          new String[] {
            "-update",
            "file:///" + TEST_ROOT_DIR + "/srcdat/" + sname,
            "file:///" + TEST_ROOT_DIR + "/dest2/"
          });
      assertTrue(
          "Source and destination directories do not match.",
          checkFiles(fs, TEST_ROOT_DIR + "/dest2", files2));
      updateFiles(fs, TEST_ROOT_DIR + "/srcdat", files2, 1);
      // copy single file to existing dir w/ dst name conflict
      ToolRunner.run(
          new DistCpV1(new Configuration()),
          new String[] {
            "-update",
            "file:///" + TEST_ROOT_DIR + "/srcdat/" + sname,
            "file:///" + TEST_ROOT_DIR + "/dest2/"
          });
      assertTrue(
          "Source and destination directories do not match.",
          checkFiles(fs, TEST_ROOT_DIR + "/dest2", files2));
    } finally {
      deldir(fs, TEST_ROOT_DIR + "/destdat");
      deldir(fs, TEST_ROOT_DIR + "/dest2");
      deldir(fs, TEST_ROOT_DIR + "/srcdat");
    }
  }