public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
   if (depth < 1) {
     return;
   } else if (depth == 1 || (path.hashCode() % 10 == 0)) {
     // Sometimes we want to try deleting a path when we're not all the way down, which is what
     // the second condition is for.
     doDelete(path);
   } else {
     if (concurrencyDepth > 0) {
       ExecutorService executor = Executors.newCachedThreadPool();
       try {
         ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
         for (int i = 0; i < FILES_PER_NODE; i++) {
           Callable<Void> call =
               (new ConcurrentDeleter(
                   depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i))));
           futures.add(executor.submit(call));
         }
         for (Future<Void> f : futures) {
           f.get();
         }
       } finally {
         executor.shutdown();
       }
     } else {
       for (int i = 0; i < FILES_PER_NODE; i++) {
         exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
       }
     }
     doDelete(path);
   }
 }
 public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
   if (depth < 1) {
     return;
   } else if (depth == 1 || (depth < mDepth && path.hashCode() % 10 < 3)) {
     // Sometimes we want to try renaming a path when we're not all the way down, which is what
     // the second condition is for. We have to create the path in the destination up till what
     // we're renaming. This might already exist, so createFile could throw a
     // FileAlreadyExistsException, which we silently handle.
     AlluxioURI srcPath = mRootPath.join(path);
     AlluxioURI dstPath = mRootPath2.join(path);
     long fileId = mFsMaster.getFileId(srcPath);
     try {
       CreateDirectoryOptions options = CreateDirectoryOptions.defaults().setRecursive(true);
       mFsMaster.createDirectory(dstPath.getParent(), options);
     } catch (FileAlreadyExistsException e) {
       // This is an acceptable exception to get, since we don't know if the parent has been
       // created yet by another thread.
     } catch (InvalidPathException e) {
       // This could happen if we are renaming something that's a child of the root.
     }
     mFsMaster.rename(srcPath, dstPath);
     Assert.assertEquals(fileId, mFsMaster.getFileId(dstPath));
   } else if (concurrencyDepth > 0) {
     ExecutorService executor = Executors.newCachedThreadPool();
     try {
       ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
       for (int i = 0; i < FILES_PER_NODE; i++) {
         Callable<Void> call =
             (new ConcurrentRenamer(
                 depth - 1,
                 concurrencyDepth - 1,
                 mRootPath,
                 mRootPath2,
                 path.join(Integer.toString(i))));
         futures.add(executor.submit(call));
       }
       for (Future<Void> f : futures) {
         f.get();
       }
     } finally {
       executor.shutdown();
     }
   } else {
     for (int i = 0; i < FILES_PER_NODE; i++) {
       exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
     }
   }
 }
  @Test
  public void listFilesTest() throws Exception {
    CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(64);

    HashSet<Long> ids = new HashSet<Long>();
    HashSet<Long> dirIds = new HashSet<Long>();
    for (int i = 0; i < 10; i++) {
      AlluxioURI dir = new AlluxioURI("/i" + i);
      mFsMaster.createDirectory(dir, CreateDirectoryOptions.defaults());
      dirIds.add(mFsMaster.getFileId(dir));
      for (int j = 0; j < 10; j++) {
        ids.add(mFsMaster.createFile(dir.join("j" + j), options));
      }
    }
    HashSet<Long> listedIds = new HashSet<>();
    HashSet<Long> listedDirIds = new HashSet<>();
    List<FileInfo> infoList = mFsMaster.getFileInfoList(new AlluxioURI("/"), true);
    for (FileInfo info : infoList) {
      long id = info.getFileId();
      listedDirIds.add(id);
      for (FileInfo fileInfo : mFsMaster.getFileInfoList(new AlluxioURI(info.getPath()), true)) {
        listedIds.add(fileInfo.getFileId());
      }
    }
    Assert.assertEquals(ids, listedIds);
    Assert.assertEquals(dirIds, listedDirIds);
  }
示例#4
0
  @Override
  public boolean rename(Path src, Path dst) throws IOException {
    LOG.info("rename({}, {})", src, dst);
    if (mStatistics != null) {
      mStatistics.incrementWriteOps(1);
    }

    AlluxioURI srcPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(src));
    AlluxioURI dstPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(dst));
    ensureExists(srcPath);
    URIStatus dstStatus;
    try {
      dstStatus = mFileSystem.getStatus(dstPath);
    } catch (IOException | AlluxioException e) {
      dstStatus = null;
    }
    // If the destination is an existing folder, try to move the src into the folder
    if (dstStatus != null && dstStatus.isFolder()) {
      dstPath = dstPath.join(srcPath.getName());
    }
    try {
      mFileSystem.rename(srcPath, dstPath);
      return true;
    } catch (IOException | AlluxioException e) {
      LOG.error("Failed to rename {} to {}", src, dst, e);
      return false;
    }
  }
  /** Tests load metadata on list. */
  @Test
  public void loadMetadata() throws Exception {
    String dirName = "loadMetaDataRoot";

    String rootDir = PathUtils.concatPath(mUnderfsAddress, dirName);
    mUfs.mkdirs(rootDir, true);

    String rootFile1 = PathUtils.concatPath(rootDir, "file1");
    createEmptyFile(rootFile1);

    String rootFile2 = PathUtils.concatPath(rootDir, "file2");
    createEmptyFile(rootFile2);

    AlluxioURI rootAlluxioURI = new AlluxioURI("/" + dirName);
    FileSystem client = mLocalAlluxioClusterResource.get().getClient();
    client.listStatus(
        rootAlluxioURI, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));

    try {
      client.createDirectory(rootAlluxioURI, CreateDirectoryOptions.defaults());
      Assert.fail("create is expected to fail with FileAlreadyExistsException");
    } catch (FileAlreadyExistsException e) {
      Assert.assertEquals(
          ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(rootAlluxioURI), e.getMessage());
    }

    AlluxioURI file1URI = rootAlluxioURI.join("file1");
    try {
      client.createFile(file1URI, CreateFileOptions.defaults()).close();
      Assert.fail("create is expected to fail with FileAlreadyExistsException");
    } catch (FileAlreadyExistsException e) {
      Assert.assertEquals(
          ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file1URI), e.getMessage());
    }

    AlluxioURI file2URI = rootAlluxioURI.join("file2");
    try {
      client.createFile(file2URI, CreateFileOptions.defaults()).close();
      Assert.fail("create is expected to fail with FileAlreadyExistsException");
    } catch (FileAlreadyExistsException e) {
      Assert.assertEquals(
          ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file2URI), e.getMessage());
    }
  }
    public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
      if (depth < 1) {
        return;
      } else if (depth == 1) {
        long fileId = mFsMaster.createFile(path, CreateFileOptions.defaults());
        Assert.assertEquals(fileId, mFsMaster.getFileId(path));
        // verify the user permission for file
        FileInfo fileInfo = mFsMaster.getFileInfo(fileId);
        Assert.assertEquals("", fileInfo.getUserName());
        Assert.assertEquals(0644, (short) fileInfo.getPermission());
      } else {
        mFsMaster.createDirectory(path, CreateDirectoryOptions.defaults());
        Assert.assertNotNull(mFsMaster.getFileId(path));
        long dirId = mFsMaster.getFileId(path);
        Assert.assertNotEquals(-1, dirId);
        FileInfo dirInfo = mFsMaster.getFileInfo(dirId);
        Assert.assertEquals("", dirInfo.getUserName());
        Assert.assertEquals(0755, (short) dirInfo.getPermission());
      }

      if (concurrencyDepth > 0) {
        ExecutorService executor = Executors.newCachedThreadPool();
        try {
          ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
          for (int i = 0; i < FILES_PER_NODE; i++) {
            Callable<Void> call =
                (new ConcurrentCreator(
                    depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i))));
            futures.add(executor.submit(call));
          }
          for (Future<Void> f : futures) {
            f.get();
          }
        } finally {
          executor.shutdown();
        }
      } else {
        for (int i = 0; i < FILES_PER_NODE; i++) {
          exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
        }
      }
    }