Example #1
0
 /**
  * Returns an {@link AlluxioURI} representation for the {@link UnderFileSystem} given a base UFS
  * URI, and the Alluxio path from the base.
  *
  * <p>The default implementation simply concatenates the path to the base URI. This should be
  * overridden if a subclass needs alternate functionality.
  *
  * @param ufsBaseUri the base {@link AlluxioURI} in the ufs
  * @param alluxioPath the path in Alluxio from the given base
  * @return the UFS {@link AlluxioURI} representing the Alluxio path
  */
 public AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath) {
   return new AlluxioURI(
       ufsBaseUri.getScheme(),
       ufsBaseUri.getAuthority(),
       PathUtils.concatPath(ufsBaseUri.getPath(), alluxioPath),
       ufsBaseUri.getQueryMap());
 }
 /**
  * Transforms the list of {@link AlluxioURI} in a new list of Strings, where each string is {@link
  * AlluxioURI#getPath()}.
  *
  * @param uris the list of {@link AlluxioURI}s to be stripped
  * @return a new list of strings mapping the input URIs to theri path component
  */
 private List<String> stripURIList(List<AlluxioURI> uris) {
   final List<String> pathStrings = new ArrayList<>(uris.size());
   for (final AlluxioURI uri : uris) {
     pathStrings.add(uri.getPath());
   }
   return pathStrings;
 }
Example #3
0
 /**
  * Returns an array of strings naming the files and directories in the directory denoted by this
  * abstract pathname, and all of its subdirectories.
  *
  * <p>If this abstract pathname does not denote a directory, then this method returns {@code
  * null}. Otherwise an array of strings is returned, one for each file or directory in the
  * directory and its subdirectories. Names denoting the directory itself and the directory's
  * parent directory are not included in the result. Each string is a path relative to the given
  * directory.
  *
  * <p>There is no guarantee that the name strings in the resulting array will appear in any
  * specific order; they are not, in particular, guaranteed to appear in alphabetical order.
  *
  * @param path the abstract pathname to list
  * @return An array of strings naming the files and directories in the directory denoted by this
  *     abstract pathname and its subdirectories. The array will be empty if the directory is
  *     empty. Returns {@code null} if this abstract pathname does not denote a directory.
  * @throws IOException if a non-Alluxio error occurs
  */
 public String[] listRecursive(String path) throws IOException {
   // Clean the path by creating a URI and turning it back to a string
   AlluxioURI uri = new AlluxioURI(path);
   path = uri.toString();
   List<String> returnPaths = new ArrayList<>();
   Queue<String> pathsToProcess = new ArrayDeque<>();
   // We call list initially, so we can return null if the path doesn't denote a directory
   String[] subpaths = list(path);
   if (subpaths == null) {
     return null;
   } else {
     for (String subp : subpaths) {
       pathsToProcess.add(PathUtils.concatPath(path, subp));
     }
   }
   while (!pathsToProcess.isEmpty()) {
     String p = pathsToProcess.remove();
     returnPaths.add(p.substring(path.length() + 1));
     // Add all of its subpaths
     subpaths = list(p);
     if (subpaths != null) {
       for (String subp : subpaths) {
         pathsToProcess.add(PathUtils.concatPath(p, subp));
       }
     }
   }
   return returnPaths.toArray(new String[returnPaths.size()]);
 }
  @Test
  public void listFilesTest() throws Exception {
    CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(64);

    HashSet<Long> ids = new HashSet<Long>();
    HashSet<Long> dirIds = new HashSet<Long>();
    for (int i = 0; i < 10; i++) {
      AlluxioURI dir = new AlluxioURI("/i" + i);
      mFsMaster.createDirectory(dir, CreateDirectoryOptions.defaults());
      dirIds.add(mFsMaster.getFileId(dir));
      for (int j = 0; j < 10; j++) {
        ids.add(mFsMaster.createFile(dir.join("j" + j), options));
      }
    }
    HashSet<Long> listedIds = new HashSet<>();
    HashSet<Long> listedDirIds = new HashSet<>();
    List<FileInfo> infoList = mFsMaster.getFileInfoList(new AlluxioURI("/"), true);
    for (FileInfo info : infoList) {
      long id = info.getFileId();
      listedDirIds.add(id);
      for (FileInfo fileInfo : mFsMaster.getFileInfoList(new AlluxioURI(info.getPath()), true)) {
        listedIds.add(fileInfo.getFileId());
      }
    }
    Assert.assertEquals(ids, listedIds);
    Assert.assertEquals(dirIds, listedDirIds);
  }
 public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
   if (depth < 1) {
     return;
   } else if (depth == 1 || (path.hashCode() % 10 == 0)) {
     // Sometimes we want to try deleting a path when we're not all the way down, which is what
     // the second condition is for.
     doDelete(path);
   } else {
     if (concurrencyDepth > 0) {
       ExecutorService executor = Executors.newCachedThreadPool();
       try {
         ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
         for (int i = 0; i < FILES_PER_NODE; i++) {
           Callable<Void> call =
               (new ConcurrentDeleter(
                   depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i))));
           futures.add(executor.submit(call));
         }
         for (Future<Void> f : futures) {
           f.get();
         }
       } finally {
         executor.shutdown();
       }
     } else {
       for (int i = 0; i < FILES_PER_NODE; i++) {
         exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
       }
     }
     doDelete(path);
   }
 }
Example #6
0
  @Override
  public boolean rename(Path src, Path dst) throws IOException {
    LOG.info("rename({}, {})", src, dst);
    if (mStatistics != null) {
      mStatistics.incrementWriteOps(1);
    }

    AlluxioURI srcPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(src));
    AlluxioURI dstPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(dst));
    ensureExists(srcPath);
    URIStatus dstStatus;
    try {
      dstStatus = mFileSystem.getStatus(dstPath);
    } catch (IOException | AlluxioException e) {
      dstStatus = null;
    }
    // If the destination is an existing folder, try to move the src into the folder
    if (dstStatus != null && dstStatus.isFolder()) {
      dstPath = dstPath.join(srcPath.getName());
    }
    try {
      mFileSystem.rename(srcPath, dstPath);
      return true;
    } catch (IOException | AlluxioException e) {
      LOG.error("Failed to rename {} to {}", src, dst, e);
      return false;
    }
  }
  // TODO(calvin): See if this method is still necessary
  public static Pair<String, String> parse(AlluxioURI path) {
    Preconditions.checkArgument(path != null, "path may not be null");

    if (path.hasScheme()) {
      String header = path.getScheme() + "://";
      String authority = (path.hasAuthority()) ? path.getAuthority() : "";
      if (header.equals(Constants.HEADER)
          || header.equals(Constants.HEADER_FT)
          || isHadoopUnderFS(header)
          || header.equals(Constants.HEADER_S3)
          || header.equals(Constants.HEADER_S3A)
          || header.equals(Constants.HEADER_S3N)
          || header.equals(Constants.HEADER_OSS)
          || header.equals(Constants.HEADER_GCS)) {
        if (path.getPath().isEmpty()) {
          return new Pair<>(header + authority, AlluxioURI.SEPARATOR);
        } else {
          return new Pair<>(header + authority, path.getPath());
        }
      } else if (header.equals("file://")) {
        return new Pair<>(AlluxioURI.SEPARATOR, path.getPath());
      }
    } else if (path.isPathAbsolute()) {
      return new Pair<>(AlluxioURI.SEPARATOR, path.getPath());
    }

    return null;
  }
Example #8
0
  /**
   * Constructs a new instance of {@link S3UnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} for this UFS
   * @param conf the configuration for Alluxio
   * @param awsCredentials AWS Credentials configuration for S3 Access
   * @throws ServiceException when a connection to S3 could not be created
   */
  public S3UnderFileSystem(AlluxioURI uri, Configuration conf, AWSCredentials awsCredentials)
      throws ServiceException {
    super(uri, conf);
    String bucketName = uri.getHost();
    mBucketName = bucketName;

    Jets3tProperties props = new Jets3tProperties();
    if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
      props.setProperty("httpclient.proxy-autodetect", "false");
      props.setProperty("httpclient.proxy-host", conf.get(Constants.UNDERFS_S3_PROXY_HOST));
      props.setProperty("httpclient.proxy-port", conf.get(Constants.UNDERFS_S3_PROXY_PORT));
    }
    if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
      props.setProperty(
          "s3service.https-only",
          Boolean.toString(conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
    }
    if (conf.containsKey(Constants.UNDERFS_S3_ENDPOINT)) {
      props.setProperty("s3service.s3-endpoint", conf.get(Constants.UNDERFS_S3_ENDPOINT));
      if (conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
        props.setProperty(
            "s3service.s3-endpoint-https-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTPS_PORT));
      } else {
        props.setProperty(
            "s3service.s3-endpoint-http-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTP_PORT));
      }
    }
    if (conf.containsKey(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
      props.setProperty(
          "s3service.disable-dns-buckets", conf.get(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS));
    }
    LOG.debug("Initializing S3 underFs with properties: {}", props.getProperties());
    mClient = new RestS3Service(awsCredentials, null, null, props);
    mBucketPrefix = PathUtils.normalizePath(Constants.HEADER_S3N + mBucketName, PATH_SEPARATOR);
  }
  /** Tests load metadata on list. */
  @Test
  public void loadMetadata() throws Exception {
    String dirName = "loadMetaDataRoot";

    String rootDir = PathUtils.concatPath(mUnderfsAddress, dirName);
    mUfs.mkdirs(rootDir, true);

    String rootFile1 = PathUtils.concatPath(rootDir, "file1");
    createEmptyFile(rootFile1);

    String rootFile2 = PathUtils.concatPath(rootDir, "file2");
    createEmptyFile(rootFile2);

    AlluxioURI rootAlluxioURI = new AlluxioURI("/" + dirName);
    FileSystem client = mLocalAlluxioClusterResource.get().getClient();
    client.listStatus(
        rootAlluxioURI, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));

    try {
      client.createDirectory(rootAlluxioURI, CreateDirectoryOptions.defaults());
      Assert.fail("create is expected to fail with FileAlreadyExistsException");
    } catch (FileAlreadyExistsException e) {
      Assert.assertEquals(
          ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(rootAlluxioURI), e.getMessage());
    }

    AlluxioURI file1URI = rootAlluxioURI.join("file1");
    try {
      client.createFile(file1URI, CreateFileOptions.defaults()).close();
      Assert.fail("create is expected to fail with FileAlreadyExistsException");
    } catch (FileAlreadyExistsException e) {
      Assert.assertEquals(
          ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file1URI), e.getMessage());
    }

    AlluxioURI file2URI = rootAlluxioURI.join("file2");
    try {
      client.createFile(file2URI, CreateFileOptions.defaults()).close();
      Assert.fail("create is expected to fail with FileAlreadyExistsException");
    } catch (FileAlreadyExistsException e) {
      Assert.assertEquals(
          ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(file2URI), e.getMessage());
    }
  }
    public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
      if (depth < 1) {
        return;
      } else if (depth == 1) {
        long fileId = mFsMaster.createFile(path, CreateFileOptions.defaults());
        Assert.assertEquals(fileId, mFsMaster.getFileId(path));
        // verify the user permission for file
        FileInfo fileInfo = mFsMaster.getFileInfo(fileId);
        Assert.assertEquals("", fileInfo.getUserName());
        Assert.assertEquals(0644, (short) fileInfo.getPermission());
      } else {
        mFsMaster.createDirectory(path, CreateDirectoryOptions.defaults());
        Assert.assertNotNull(mFsMaster.getFileId(path));
        long dirId = mFsMaster.getFileId(path);
        Assert.assertNotEquals(-1, dirId);
        FileInfo dirInfo = mFsMaster.getFileInfo(dirId);
        Assert.assertEquals("", dirInfo.getUserName());
        Assert.assertEquals(0755, (short) dirInfo.getPermission());
      }

      if (concurrencyDepth > 0) {
        ExecutorService executor = Executors.newCachedThreadPool();
        try {
          ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
          for (int i = 0; i < FILES_PER_NODE; i++) {
            Callable<Void> call =
                (new ConcurrentCreator(
                    depth - 1, concurrencyDepth - 1, path.join(Integer.toString(i))));
            futures.add(executor.submit(call));
          }
          for (Future<Void> f : futures) {
            f.get();
          }
        } finally {
          executor.shutdown();
        }
      } else {
        for (int i = 0; i < FILES_PER_NODE; i++) {
          exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
        }
      }
    }
  /**
   * Constructs a new Swift {@link UnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} for this UFS
   */
  public SwiftUnderFileSystem(AlluxioURI uri) {
    super(uri);
    String containerName = uri.getHost();
    LOG.debug("Constructor init: {}", containerName);
    AccountConfig config = new AccountConfig();
    if (Configuration.containsKey(Constants.SWIFT_API_KEY)) {
      config.setPassword(Configuration.get(Constants.SWIFT_API_KEY));
    } else if (Configuration.containsKey(Constants.SWIFT_PASSWORD_KEY)) {
      config.setPassword(Configuration.get(Constants.SWIFT_PASSWORD_KEY));
    }
    config.setAuthUrl(Configuration.get(Constants.SWIFT_AUTH_URL_KEY));
    String authMethod = Configuration.get(Constants.SWIFT_AUTH_METHOD_KEY);
    if (authMethod != null) {
      config.setUsername(Configuration.get(Constants.SWIFT_USER_KEY));
      config.setTenantName(Configuration.get(Constants.SWIFT_TENANT_KEY));
      switch (authMethod) {
        case Constants.SWIFT_AUTH_KEYSTONE:
          config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
          break;
        case Constants.SWIFT_AUTH_SWIFTAUTH:
          // swiftauth authenticates directly against swift
          // note: this method is supported in swift object storage api v1
          config.setAuthenticationMethod(AuthenticationMethod.BASIC);
          break;
        default:
          config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
          // tempauth requires authentication header to be of the form tenant:user.
          // JOSS however generates header of the form user:tenant.
          // To resolve this, we switch user with tenant
          config.setTenantName(Configuration.get(Constants.SWIFT_USER_KEY));
          config.setUsername(Configuration.get(Constants.SWIFT_TENANT_KEY));
      }
    }

    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
    mContainerName = containerName;
    mAccount = new AccountFactory(config).createAccount();
    // Do not allow container cache to avoid stale directory listings
    mAccount.setAllowContainerCaching(false);
    mAccess = mAccount.authenticate();
    Container container = mAccount.getContainer(containerName);
    if (!container.exists()) {
      container.create();
    }
    mContainerPrefix = Constants.HEADER_SWIFT + mContainerName + PATH_SEPARATOR;
  }
 public void exec(int depth, int concurrencyDepth, AlluxioURI path) throws Exception {
   if (depth < 1) {
     return;
   } else if (depth == 1 || (depth < mDepth && path.hashCode() % 10 < 3)) {
     // Sometimes we want to try renaming a path when we're not all the way down, which is what
     // the second condition is for. We have to create the path in the destination up till what
     // we're renaming. This might already exist, so createFile could throw a
     // FileAlreadyExistsException, which we silently handle.
     AlluxioURI srcPath = mRootPath.join(path);
     AlluxioURI dstPath = mRootPath2.join(path);
     long fileId = mFsMaster.getFileId(srcPath);
     try {
       CreateDirectoryOptions options = CreateDirectoryOptions.defaults().setRecursive(true);
       mFsMaster.createDirectory(dstPath.getParent(), options);
     } catch (FileAlreadyExistsException e) {
       // This is an acceptable exception to get, since we don't know if the parent has been
       // created yet by another thread.
     } catch (InvalidPathException e) {
       // This could happen if we are renaming something that's a child of the root.
     }
     mFsMaster.rename(srcPath, dstPath);
     Assert.assertEquals(fileId, mFsMaster.getFileId(dstPath));
   } else if (concurrencyDepth > 0) {
     ExecutorService executor = Executors.newCachedThreadPool();
     try {
       ArrayList<Future<Void>> futures = new ArrayList<Future<Void>>(FILES_PER_NODE);
       for (int i = 0; i < FILES_PER_NODE; i++) {
         Callable<Void> call =
             (new ConcurrentRenamer(
                 depth - 1,
                 concurrencyDepth - 1,
                 mRootPath,
                 mRootPath2,
                 path.join(Integer.toString(i))));
         futures.add(executor.submit(call));
       }
       for (Future<Void> f : futures) {
         f.get();
       }
     } finally {
       executor.shutdown();
     }
   } else {
     for (int i = 0; i < FILES_PER_NODE; i++) {
       exec(depth - 1, concurrencyDepth, path.join(Integer.toString(i)));
     }
   }
 }
Example #13
0
 Key(AlluxioURI uri) {
   mScheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase();
   mAuthority = uri.getAuthority() == null ? "" : uri.getAuthority().toLowerCase();
 }