/**
  * Tests an empty directory can be deleted. Tests a non empty directory will not be deleted if
  * recursive is not specified. Tests a non empty directory will be deleted if recursive is
  * specified.
  */
 @Test
 public void deleteDir() throws IOException {
   String testDirEmpty = PathUtils.concatPath(mUnderfsAddress, "testDirEmpty");
   String testDirNonEmpty = PathUtils.concatPath(mUnderfsAddress, "testDirNonEmpty1");
   String testDirNonEmptyChildDir = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmpty2");
   String testDirNonEmptyChildFile = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmptyF");
   String testDirNonEmptyChildDirFile =
       PathUtils.concatPath(testDirNonEmptyChildDir, "testDirNonEmptyChildDirF");
   mUfs.mkdirs(testDirEmpty, false);
   mUfs.mkdirs(testDirNonEmpty, false);
   mUfs.mkdirs(testDirNonEmptyChildDir, false);
   createEmptyFile(testDirNonEmptyChildFile);
   createEmptyFile(testDirNonEmptyChildDirFile);
   mUfs.delete(testDirEmpty, false);
   Assert.assertFalse(mUfs.exists(testDirEmpty));
   try {
     mUfs.delete(testDirNonEmpty, false);
   } catch (IOException e) {
     // Some File systems may throw IOException
   }
   Assert.assertTrue(mUfs.exists(testDirNonEmpty));
   mUfs.delete(testDirNonEmpty, true);
   Assert.assertFalse(mUfs.exists(testDirNonEmpty));
   Assert.assertFalse(mUfs.exists(testDirNonEmptyChildDir));
   Assert.assertFalse(mUfs.exists(testDirNonEmptyChildFile));
   Assert.assertFalse(mUfs.exists(testDirNonEmptyChildDirFile));
 }
示例#2
0
 @Override
 public boolean rename(String src, String dst) throws IOException {
   if (!exists(src)) {
     LOG.error("Unable to rename {} to {} because source does not exist.", src, dst);
     return false;
   }
   if (exists(dst)) {
     LOG.error("Unable to rename {} to {} because destination already exists.", src, dst);
     return false;
   }
   // Source exists and destination does not exist
   if (isFolder(src)) {
     // Rename the source folder first
     if (!copy(convertToFolderName(src), convertToFolderName(dst))) {
       return false;
     }
     // Rename each child in the src folder to destination/child
     String[] children = list(src);
     for (String child : children) {
       if (!rename(PathUtils.concatPath(src, child), PathUtils.concatPath(dst, child))) {
         return false;
       }
     }
     // Delete src and everything under src
     return delete(src, true);
   }
   // Source is a file and Destination does not exist
   return copy(src, dst) && deleteInternal(src);
 }
 /** Tests if list correctly returns file names. */
 @Test
 public void list() throws IOException {
   String testDirNonEmpty = PathUtils.concatPath(mUnderfsAddress, "testDirNonEmpty1");
   String testDirNonEmptyChildDir = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmpty2");
   String testDirNonEmptyChildFile = PathUtils.concatPath(testDirNonEmpty, "testDirNonEmptyF");
   String testDirNonEmptyChildDirFile =
       PathUtils.concatPath(testDirNonEmptyChildDir, "testDirNonEmptyChildDirF");
   mUfs.mkdirs(testDirNonEmpty, false);
   mUfs.mkdirs(testDirNonEmptyChildDir, false);
   createEmptyFile(testDirNonEmptyChildFile);
   createEmptyFile(testDirNonEmptyChildDirFile);
   String[] expectedResTopDir = new String[] {"testDirNonEmpty2", "testDirNonEmptyF"};
   // Some file systems may prefix with a slash
   String[] expectedResTopDir2 = new String[] {"/testDirNonEmpty2", "/testDirNonEmptyF"};
   Arrays.sort(expectedResTopDir);
   Arrays.sort(expectedResTopDir2);
   String[] resTopDir = mUfs.list(testDirNonEmpty);
   Arrays.sort(resTopDir);
   Assert.assertTrue(
       Arrays.equals(expectedResTopDir, resTopDir)
           || Arrays.equals(expectedResTopDir2, resTopDir));
   Assert.assertTrue(
       mUfs.list(testDirNonEmptyChildDir)[0].equals("testDirNonEmptyChildDirF")
           || mUfs.list(testDirNonEmptyChildDir)[0].equals("/testDirNonEmptyChildDirF"));
 }
示例#4
0
 /**
  * Returns an array of strings naming the files and directories in the directory denoted by this
  * abstract pathname, and all of its subdirectories.
  *
  * <p>If this abstract pathname does not denote a directory, then this method returns {@code
  * null}. Otherwise an array of strings is returned, one for each file or directory in the
  * directory and its subdirectories. Names denoting the directory itself and the directory's
  * parent directory are not included in the result. Each string is a path relative to the given
  * directory.
  *
  * <p>There is no guarantee that the name strings in the resulting array will appear in any
  * specific order; they are not, in particular, guaranteed to appear in alphabetical order.
  *
  * @param path the abstract pathname to list
  * @return An array of strings naming the files and directories in the directory denoted by this
  *     abstract pathname and its subdirectories. The array will be empty if the directory is
  *     empty. Returns {@code null} if this abstract pathname does not denote a directory.
  * @throws IOException if a non-Alluxio error occurs
  */
 public String[] listRecursive(String path) throws IOException {
   // Clean the path by creating a URI and turning it back to a string
   AlluxioURI uri = new AlluxioURI(path);
   path = uri.toString();
   List<String> returnPaths = new ArrayList<>();
   Queue<String> pathsToProcess = new ArrayDeque<>();
   // We call list initially, so we can return null if the path doesn't denote a directory
   String[] subpaths = list(path);
   if (subpaths == null) {
     return null;
   } else {
     for (String subp : subpaths) {
       pathsToProcess.add(PathUtils.concatPath(path, subp));
     }
   }
   while (!pathsToProcess.isEmpty()) {
     String p = pathsToProcess.remove();
     returnPaths.add(p.substring(path.length() + 1));
     // Add all of its subpaths
     subpaths = list(p);
     if (subpaths != null) {
       for (String subp : subpaths) {
         pathsToProcess.add(PathUtils.concatPath(p, subp));
       }
     }
   }
   return returnPaths.toArray(new String[returnPaths.size()]);
 }
  // Prepare directory tree for pagination tests
  private LargeDirectoryConfig prepareLargeDirectoryTest() throws IOException {
    final String filePrefix = "a_";
    final String folderPrefix = "b_";

    String topLevelDirectory = PathUtils.concatPath(mUnderfsAddress, "topLevelDir");

    final int numFiles = 100;

    String[] children = new String[numFiles + numFiles];

    // Make top level directory
    mUfs.mkdirs(topLevelDirectory, false);

    // Make the children files
    for (int i = 0; i < numFiles; ++i) {
      children[i] = PathUtils.concatPath(topLevelDirectory, filePrefix + String.format("%04d", i));
      createEmptyFile(children[i]);
    }
    // Make the children folders
    for (int i = 0; i < numFiles; ++i) {
      children[numFiles + i] =
          PathUtils.concatPath(topLevelDirectory, folderPrefix + String.format("%04d", i));
      mUfs.mkdirs(children[numFiles + i], false);
    }

    return new LargeDirectoryConfig(topLevelDirectory, children);
  }
 /** Tests {@link UnderFileSystem#rename(String, String)} works file to new location. */
 @Test
 public void renameFile() throws IOException {
   String testFileSrc = PathUtils.concatPath(mUnderfsAddress, "testFileSrc");
   String testFileDst = PathUtils.concatPath(mUnderfsAddress, "testFileDst");
   createEmptyFile(testFileSrc);
   mUfs.rename(testFileSrc, testFileDst);
   Assert.assertFalse(mUfs.exists(testFileSrc));
   Assert.assertTrue(mUfs.exists(testFileDst));
 }
 /** Tests {@link UnderFileSystem#getFileSize(String)} correctly returns the file size. */
 @Test
 public void getFileSize() throws IOException {
   String testFileEmpty = PathUtils.concatPath(mUnderfsAddress, "testFileEmpty");
   String testFileNonEmpty = PathUtils.concatPath(mUnderfsAddress, "testFileNonEmpty");
   createEmptyFile(testFileEmpty);
   createTestBytesFile(testFileNonEmpty);
   Assert.assertEquals(mUfs.getFileSize(testFileEmpty), 0);
   Assert.assertEquals(mUfs.getFileSize(testFileNonEmpty), TEST_BYTES.length);
 }
 /**
  * Tests if {@link UnderFileSystem#isFile(String)} correctly returns true for files and false
  * otherwise.
  */
 @Test
 public void isFile() throws IOException {
   String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
   String testDir = PathUtils.concatPath(mUnderfsAddress, "testDir");
   Assert.assertFalse(mUfs.isFile(testFile));
   createEmptyFile(testFile);
   mUfs.mkdirs(testDir, false);
   Assert.assertTrue(mUfs.isFile(testFile));
   Assert.assertFalse(mUfs.isFile(testDir));
 }
 /** Tests {@link UnderFileSystem#rename(String, String)} works file to a folder if supported. */
 @Test
 public void renameFileToFolder() throws IOException {
   String testFileSrc = PathUtils.concatPath(mUnderfsAddress, "testFileSrc");
   String testFileDst = PathUtils.concatPath(mUnderfsAddress, "testDirDst");
   String testFileFinalDst = PathUtils.concatPath(testFileDst, "testFileSrc");
   createEmptyFile(testFileSrc);
   mUfs.mkdirs(testFileDst, false);
   if (mUfs.rename(testFileSrc, testFileDst)) {
     Assert.assertFalse(mUfs.exists(testFileSrc));
     Assert.assertTrue(mUfs.exists(testFileFinalDst));
   }
 }
 /** Tests a file can be deleted. */
 @Test
 public void deleteFile() throws IOException {
   String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
   createEmptyFile(testFile);
   mUfs.delete(testFile, false);
   Assert.assertFalse(mUfs.exists(testFile));
 }
示例#11
0
 /**
  * Returns an {@link AlluxioURI} representation for the {@link UnderFileSystem} given a base UFS
  * URI, and the Alluxio path from the base.
  *
  * <p>The default implementation simply concatenates the path to the base URI. This should be
  * overridden if a subclass needs alternate functionality.
  *
  * @param ufsBaseUri the base {@link AlluxioURI} in the ufs
  * @param alluxioPath the path in Alluxio from the given base
  * @return the UFS {@link AlluxioURI} representing the Alluxio path
  */
 public AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath) {
   return new AlluxioURI(
       ufsBaseUri.getScheme(),
       ufsBaseUri.getAuthority(),
       PathUtils.concatPath(ufsBaseUri.getPath(), alluxioPath),
       ufsBaseUri.getQueryMap());
 }
 /** Tests if file creation is atomic. */
 @Test
 public void createAtomic() throws IOException {
   String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
   OutputStream stream = mUfs.create(testFile);
   stream.write(TEST_BYTES);
   Assert.assertFalse(mUfs.exists(testFile));
   stream.close();
 }
 /** Tests that a file can be created and validates the data written to it. */
 @Test
 public void createOpen() throws IOException {
   String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
   createTestBytesFile(testFile);
   byte[] buf = new byte[TEST_BYTES.length];
   int bytesRead = mUfs.open(testFile).read(buf);
   Assert.assertTrue(bytesRead == TEST_BYTES.length);
   Assert.assertTrue(Arrays.equals(buf, TEST_BYTES));
 }
  /** Tests load metadata on list. */
  @Test
  public void loadMetadata() throws Exception {
    String dirName = "loadMetaDataRoot";

    String rootDir = PathUtils.concatPath(mUnderfsAddress, dirName);
    mUfs.mkdirs(rootDir, true);

    String rootFile1 = PathUtils.concatPath(rootDir, "file1");
    createEmptyFile(rootFile1);

    String rootFile2 = PathUtils.concatPath(rootDir, "file2");
    createEmptyFile(rootFile2);

    AlluxioURI rootAlluxioURI = new AlluxioURI("/" + dirName);
    FileSystem client = mLocalAlluxioClusterResource.get().getClient();
    client.listStatus(
        rootAlluxioURI, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));

    try {
      client.createDirectory(rootAlluxioURI, CreateDirectoryOptions.defaults());
      Assert.fail("create is expected to fail with FileAlreadyExistsException");
    } catch (FileAlreadyExistsException e) {
      Assert.assertEquals(
          ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(rootAlluxioURI), e.getMessage());
    }

    AlluxioURI file1URI = rootAlluxioURI.join("file1");
    try {
      client.createFile(file1URI, CreateFileOptions.defaults()).close();
      Assert.fail("create is expected to fail with FileAlreadyExistsException");
    } catch (FileAlreadyExistsException e) {
      Assert.assertEquals(
          ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file1URI), e.getMessage());
    }

    AlluxioURI file2URI = rootAlluxioURI.join("file2");
    try {
      client.createFile(file2URI, CreateFileOptions.defaults()).close();
      Assert.fail("create is expected to fail with FileAlreadyExistsException");
    } catch (FileAlreadyExistsException e) {
      Assert.assertEquals(
          ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file2URI), e.getMessage());
    }
  }
示例#15
0
  /**
   * @inheritDoc Rename will overwrite destination if it already exists
   * @param source the source file or folder name
   * @param destination the destination file or folder name
   * @return true if succeed, false otherwise
   * @throws IOException if a non-Alluxio error occurs
   */
  @Override
  public boolean rename(String source, String destination) throws IOException {
    String strippedSourcePath = stripContainerPrefixIfPresent(source);
    String strippedDestinationPath = stripContainerPrefixIfPresent(destination);

    if (isDirectory(destination)) {
      // If destination is a directory target is a file or folder within that directory
      strippedDestinationPath =
          PathUtils.concatPath(
              strippedDestinationPath,
              FilenameUtils.getName(stripFolderSuffixIfPresent(strippedSourcePath)));
    }

    if (isDirectory(source)) {
      // Source is a directory
      strippedSourcePath = addFolderSuffixIfNotPresent(strippedSourcePath);
      strippedDestinationPath = addFolderSuffixIfNotPresent(strippedDestinationPath);

      // Rename the source folder first
      if (!copy(strippedSourcePath, strippedDestinationPath)) {
        return false;
      }
      // TODO(adit): Use pagination to list large directories and merge duplicate call in delete
      // Rename each child in the source folder to destination/child
      String[] children = list(source);
      for (String child : children) {
        // TODO(adit): See how we can do this with better performance
        // Recursive call
        if (!rename(
            PathUtils.concatPath(source, child),
            PathUtils.concatPath(
                mContainerPrefix, PathUtils.concatPath(strippedDestinationPath, child)))) {
          return false;
        }
      }
      // Delete source and everything under source
      return delete(source, true);
    }

    // Source is a file and destination is also a file
    return copy(strippedSourcePath, strippedDestinationPath) && delete(source, false);
  }
 /**
  * Tests {@link UnderFileSystem#getModificationTimeMs(String)} returns a reasonably accurate time.
  */
 @Test
 public void getModTime() throws IOException {
   long slack = 1000; // Some file systems may report nearest second.
   long start = System.currentTimeMillis();
   String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
   createTestBytesFile(testFile);
   long end = System.currentTimeMillis();
   long modTime = mUfs.getModificationTimeMs(testFile);
   Assert.assertTrue(modTime >= start - slack);
   Assert.assertTrue(modTime <= end + slack);
 }
  /**
   * Tests {@link UnderFileSystem#mkdirs(String, boolean)} correctly creates a directory. Tests
   * {@link UnderFileSystem#mkdirs(String, boolean)} correctly makes parent directories if
   * createParent is specified.
   */
  @Test
  public void mkdirs() throws IOException {
    // make sure the underfs address dir exists already
    mUfs.mkdirs(mUnderfsAddress, true);
    // empty lsr should be empty
    Assert.assertEquals(0, mUfs.listRecursive(mUnderfsAddress).length);

    String testDirTop = PathUtils.concatPath(mUnderfsAddress, "testDirTop");
    String testDir1 = PathUtils.concatPath(mUnderfsAddress, "1");
    String testDir2 = PathUtils.concatPath(testDir1, "2");
    String testDir3 = PathUtils.concatPath(testDir2, "3");
    String testDirDeep = PathUtils.concatPath(testDir3, "testDirDeep");
    mUfs.mkdirs(testDirTop, false);
    Assert.assertTrue(mUfs.exists(testDirTop));
    mUfs.mkdirs(testDirDeep, true);
    Assert.assertTrue(mUfs.exists(testDir1));
    Assert.assertTrue(mUfs.exists(testDir2));
    Assert.assertTrue(mUfs.exists(testDir3));
    Assert.assertTrue(mUfs.exists(testDirDeep));
  }
示例#18
0
  private static String createStartAlluxioCommand(String command) {
    List<String> commands = new ArrayList<>();
    commands.add(String.format("echo 'Starting Alluxio with %s'", command));
    if (installJavaFromUrl()) {
      commands.add("export JAVA_HOME=" + Configuration.get(PropertyKey.INTEGRATION_MESOS_JDK_PATH));
      commands.add("export PATH=$PATH:$JAVA_HOME/bin");
    }

    commands.add("mkdir conf");
    commands.add("touch conf/alluxio-env.sh");

    // If a jar is supplied, start Alluxio from the jar. Otherwise assume that Alluxio is already
    // installed at PropertyKey.HOME.
    if (installAlluxioFromUrl()) {
      commands.add("rm *.tar.gz");
      commands.add("mv alluxio-* alluxio");
    }
    String home = installAlluxioFromUrl() ? "alluxio" : Configuration.get(PropertyKey.HOME);
    commands.add(
        String.format("cp %s conf", PathUtils.concatPath(home, "conf", "log4j.properties")));
    commands.add(PathUtils.concatPath(home, "integration", "bin", command));
    return Joiner.on(" && ").join(commands);
  }
  /** Tests if list recursive correctly returns all file names in all subdirectories. */
  @Test
  public void listRecursive() throws IOException {
    String root = mUnderfsAddress;
    // TODO(andrew): Should this directory be created in LocalAlluxioCluster creation code?
    mUfs.mkdirs(root, true);
    // Empty lsr should be empty
    Assert.assertEquals(0, mUfs.listRecursive(root).length);

    // Create a tree of subdirectories and files
    String sub1 = PathUtils.concatPath(root, "sub1");
    String sub2 = PathUtils.concatPath(root, "sub2");
    String sub11 = PathUtils.concatPath(sub1, "sub11");
    String file11 = PathUtils.concatPath(sub11, "file11");
    String file2 = PathUtils.concatPath(sub2, "file2");
    String file = PathUtils.concatPath(root, "file");
    // lsr of nonexistent path should be null
    Assert.assertNull(mUfs.listRecursive(sub1));

    mUfs.mkdirs(sub1, false);
    mUfs.mkdirs(sub2, false);
    mUfs.mkdirs(sub11, false);
    createEmptyFile(file11);
    createEmptyFile(file2);
    createEmptyFile(file);

    // lsr from root should return paths relative to the root
    String[] expectedResRoot = {
      "sub1", "sub2", "sub1/sub11", "sub1/sub11/file11", "sub2/file2", "file"
    };
    String[] actualResRoot = mUfs.listRecursive(root);
    Arrays.sort(expectedResRoot);
    Arrays.sort(actualResRoot);
    Assert.assertArrayEquals(expectedResRoot, actualResRoot);

    // lsr from sub1 should return paths relative to sub1
    String[] expectedResSub1 = {"sub11", "sub11/file11"};
    String[] actualResSub1 = mUfs.listRecursive(sub1);
    Arrays.sort(expectedResSub1);
    Arrays.sort(actualResSub1);
    Assert.assertArrayEquals(expectedResSub1, actualResSub1);

    // lsr of file should be null
    Assert.assertNull(mUfs.listRecursive(file));
  }
示例#20
0
 @Override
 public boolean delete(String path, boolean recursive) throws IOException {
   if (!recursive) {
     if (isFolder(path) && listInternal(path, false).length != 0) {
       LOG.error(
           "Unable to delete "
               + path
               + " because it is a non empty directory. Specify "
               + "recursive as true in order to delete non empty directories.");
       return false;
     }
     return deleteInternal(path);
   }
   // Get all relevant files
   String[] pathsToDelete = listInternal(path, true);
   for (String pathToDelete : pathsToDelete) {
     // If we fail to deleteInternal one file, stop
     if (!deleteInternal(PathUtils.concatPath(path, pathToDelete))) {
       LOG.error("Failed to delete path {}, aborting delete.", pathToDelete);
       return false;
     }
   }
   return deleteInternal(path);
 }
示例#21
0
 /**
  * @param baseDirectory the base journal directory
  * @return the journal directory for this master
  */
 public static String getJournalDirectory(String baseDirectory) {
   return PathUtils.concatPath(baseDirectory, Constants.BLOCK_MASTER_NAME);
 }
 /** Tests that an empty file can be created. */
 @Test
 public void createEmpty() throws IOException {
   String testFile = PathUtils.concatPath(mUnderfsAddress, "testFile");
   createEmptyFile(testFile);
   Assert.assertTrue(mUfs.exists(testFile));
 }