Пример #1
0
  public void testPreserveOption() throws Exception {
    Configuration conf = new Configuration();
    MiniDFSCluster cluster = null;
    try {
      cluster = new MiniDFSCluster(conf, 2, true, null);
      String nnUri = FileSystem.getDefaultUri(conf).toString();
      FileSystem fs = FileSystem.get(URI.create(nnUri), conf);

      { // test preserving user
        MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
        FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
        for (int i = 0; i < srcstat.length; i++) {
          fs.setOwner(srcstat[i].getPath(), "u" + i, null);
        }
        ToolRunner.run(
            new DistCpV1(conf), new String[] {"-pu", nnUri + "/srcdat", nnUri + "/destdat"});
        assertTrue(
            "Source and destination directories do not match.", checkFiles(fs, "/destdat", files));

        FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
        for (int i = 0; i < dststat.length; i++) {
          assertEquals("i=" + i, "u" + i, dststat[i].getOwner());
        }
        deldir(fs, "/destdat");
        deldir(fs, "/srcdat");
      }

      { // test preserving group
        MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
        FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
        for (int i = 0; i < srcstat.length; i++) {
          fs.setOwner(srcstat[i].getPath(), null, "g" + i);
        }
        ToolRunner.run(
            new DistCpV1(conf), new String[] {"-pg", nnUri + "/srcdat", nnUri + "/destdat"});
        assertTrue(
            "Source and destination directories do not match.", checkFiles(fs, "/destdat", files));

        FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
        for (int i = 0; i < dststat.length; i++) {
          assertEquals("i=" + i, "g" + i, dststat[i].getGroup());
        }
        deldir(fs, "/destdat");
        deldir(fs, "/srcdat");
      }

      { // test preserving mode
        MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
        FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
        FsPermission[] permissions = new FsPermission[srcstat.length];
        for (int i = 0; i < srcstat.length; i++) {
          permissions[i] = new FsPermission((short) (i & 0666));
          fs.setPermission(srcstat[i].getPath(), permissions[i]);
        }

        ToolRunner.run(
            new DistCpV1(conf), new String[] {"-pp", nnUri + "/srcdat", nnUri + "/destdat"});
        assertTrue(
            "Source and destination directories do not match.", checkFiles(fs, "/destdat", files));

        FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
        for (int i = 0; i < dststat.length; i++) {
          assertEquals("i=" + i, permissions[i], dststat[i].getPermission());
        }
        deldir(fs, "/destdat");
        deldir(fs, "/srcdat");
      }

      { // test preserving times
        MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
        fs.mkdirs(new Path("/srcdat/tmpf1"));
        fs.mkdirs(new Path("/srcdat/tmpf2"));
        FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
        FsPermission[] permissions = new FsPermission[srcstat.length];
        for (int i = 0; i < srcstat.length; i++) {
          fs.setTimes(srcstat[i].getPath(), 40, 50);
        }

        ToolRunner.run(
            new DistCpV1(conf), new String[] {"-pt", nnUri + "/srcdat", nnUri + "/destdat"});

        FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
        for (int i = 0; i < dststat.length; i++) {
          assertEquals("Modif. Time i=" + i, 40, dststat[i].getModificationTime());
          assertEquals(
              "Access Time i=" + i + srcstat[i].getPath() + "-" + dststat[i].getPath(),
              50,
              dststat[i].getAccessTime());
        }

        assertTrue(
            "Source and destination directories do not match.", checkFiles(fs, "/destdat", files));

        deldir(fs, "/destdat");
        deldir(fs, "/srcdat");
      }
    } finally {
      if (cluster != null) {
        cluster.shutdown();
      }
    }
  }
  /** Tests mod & access time in DFS. */
  public void testTimes() throws IOException {
    Configuration conf = new HdfsConfiguration();
    final int MAX_IDLE_TIME = 2000; // 2s
    conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME);
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000);
    conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);

    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
    cluster.waitActive();
    final int nnport = cluster.getNameNodePort();
    InetSocketAddress addr = new InetSocketAddress("localhost", cluster.getNameNodePort());
    DFSClient client = new DFSClient(addr, conf);
    DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE);
    assertEquals("Number of Datanodes ", numDatanodes, info.length);
    FileSystem fileSys = cluster.getFileSystem();
    int replicas = 1;
    assertTrue(fileSys instanceof DistributedFileSystem);

    try {
      //
      // create file and record atime/mtime
      //
      System.out.println("Creating testdir1 and testdir1/test1.dat.");
      Path dir1 = new Path("testdir1");
      Path file1 = new Path(dir1, "test1.dat");
      FSDataOutputStream stm = writeFile(fileSys, file1, replicas);
      FileStatus stat = fileSys.getFileStatus(file1);
      long atimeBeforeClose = stat.getAccessTime();
      String adate = dateForm.format(new Date(atimeBeforeClose));
      System.out.println(
          "atime on " + file1 + " before close is " + adate + " (" + atimeBeforeClose + ")");
      assertTrue(atimeBeforeClose != 0);
      stm.close();

      stat = fileSys.getFileStatus(file1);
      long atime1 = stat.getAccessTime();
      long mtime1 = stat.getModificationTime();
      adate = dateForm.format(new Date(atime1));
      String mdate = dateForm.format(new Date(mtime1));
      System.out.println("atime on " + file1 + " is " + adate + " (" + atime1 + ")");
      System.out.println("mtime on " + file1 + " is " + mdate + " (" + mtime1 + ")");
      assertTrue(atime1 != 0);

      //
      // record dir times
      //
      stat = fileSys.getFileStatus(dir1);
      long mdir1 = stat.getAccessTime();
      assertTrue(mdir1 == 0);

      // set the access time to be one day in the past
      long atime2 = atime1 - (24L * 3600L * 1000L);
      fileSys.setTimes(file1, -1, atime2);

      // check new access time on file
      stat = fileSys.getFileStatus(file1);
      long atime3 = stat.getAccessTime();
      String adate3 = dateForm.format(new Date(atime3));
      System.out.println("new atime on " + file1 + " is " + adate3 + " (" + atime3 + ")");
      assertTrue(atime2 == atime3);
      assertTrue(mtime1 == stat.getModificationTime());

      // set the modification time to be 1 hour in the past
      long mtime2 = mtime1 - (3600L * 1000L);
      fileSys.setTimes(file1, mtime2, -1);

      // check new modification time on file
      stat = fileSys.getFileStatus(file1);
      long mtime3 = stat.getModificationTime();
      String mdate3 = dateForm.format(new Date(mtime3));
      System.out.println("new mtime on " + file1 + " is " + mdate3 + " (" + mtime3 + ")");
      assertTrue(atime2 == stat.getAccessTime());
      assertTrue(mtime2 == mtime3);

      long mtime4 = System.currentTimeMillis() - (3600L * 1000L);
      long atime4 = System.currentTimeMillis();
      fileSys.setTimes(dir1, mtime4, atime4);
      // check new modification time on file
      stat = fileSys.getFileStatus(dir1);
      assertTrue("Not matching the modification times", mtime4 == stat.getModificationTime());
      assertTrue("Not matching the access times", atime4 == stat.getAccessTime());

      Path nonExistingDir = new Path(dir1, "/nonExistingDir/");
      try {
        fileSys.setTimes(nonExistingDir, mtime4, atime4);
        fail("Expecting FileNotFoundException");
      } catch (FileNotFoundException e) {
        assertTrue(
            e.getMessage()
                .contains("File/Directory " + nonExistingDir.toString() + " does not exist."));
      }
      // shutdown cluster and restart
      cluster.shutdown();
      try {
        Thread.sleep(2 * MAX_IDLE_TIME);
      } catch (InterruptedException e) {
      }
      cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport).format(false).build();
      cluster.waitActive();
      fileSys = cluster.getFileSystem();

      // verify that access times and modification times persist after a
      // cluster restart.
      System.out.println("Verifying times after cluster restart");
      stat = fileSys.getFileStatus(file1);
      assertTrue(atime2 == stat.getAccessTime());
      assertTrue(mtime3 == stat.getModificationTime());

      cleanupFile(fileSys, file1);
      cleanupFile(fileSys, dir1);
    } catch (IOException e) {
      info = client.datanodeReport(DatanodeReportType.ALL);
      printDatanodeReport(info);
      throw e;
    } finally {
      fileSys.close();
      cluster.shutdown();
    }
  }