Ejemplo n.º 1
0
  /**
   * Constructs a new instance of {@link S3UnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} for this UFS
   * @param conf the configuration for Alluxio
   * @param awsCredentials AWS Credentials configuration for S3 Access
   * @throws ServiceException when a connection to S3 could not be created
   */
  public S3UnderFileSystem(AlluxioURI uri, Configuration conf, AWSCredentials awsCredentials)
      throws ServiceException {
    super(uri, conf);
    String bucketName = uri.getHost();
    mBucketName = bucketName;

    Jets3tProperties props = new Jets3tProperties();
    if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
      props.setProperty("httpclient.proxy-autodetect", "false");
      props.setProperty("httpclient.proxy-host", conf.get(Constants.UNDERFS_S3_PROXY_HOST));
      props.setProperty("httpclient.proxy-port", conf.get(Constants.UNDERFS_S3_PROXY_PORT));
    }
    if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
      props.setProperty(
          "s3service.https-only",
          Boolean.toString(conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
    }
    if (conf.containsKey(Constants.UNDERFS_S3_ENDPOINT)) {
      props.setProperty("s3service.s3-endpoint", conf.get(Constants.UNDERFS_S3_ENDPOINT));
      if (conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
        props.setProperty(
            "s3service.s3-endpoint-https-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTPS_PORT));
      } else {
        props.setProperty(
            "s3service.s3-endpoint-http-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTP_PORT));
      }
    }
    if (conf.containsKey(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
      props.setProperty(
          "s3service.disable-dns-buckets", conf.get(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS));
    }
    LOG.debug("Initializing S3 underFs with properties: {}", props.getProperties());
    mClient = new RestS3Service(awsCredentials, null, null, props);
    mBucketPrefix = PathUtils.normalizePath(Constants.HEADER_S3N + mBucketName, PATH_SEPARATOR);
  }
  /** Tests if list correctly returns file or folder names for a large directory. */
  @Test
  public void listLargeDirectory() throws IOException {
    LargeDirectoryConfig config = prepareLargeDirectoryTest();
    String[] children = config.getChildren();

    // Retry for some time to allow list operation eventual consistency for S3 and GCS.
    // See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
    // https://cloud.google.com/storage/docs/consistency for more details.
    // Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
    String[] results = new String[] {};
    for (int i = 0; i < 20; i++) {
      results = mUfs.list(config.getTopLevelDirectory());
      if (children.length == results.length) {
        break;
      }
      CommonUtils.sleepMs(500);
    }
    Assert.assertEquals(children.length, results.length);

    Arrays.sort(results);
    for (int i = 0; i < children.length; ++i) {
      Assert.assertTrue(
          results[i].equals(
              CommonUtils.stripPrefixIfPresent(
                  children[i], PathUtils.normalizePath(config.getTopLevelDirectory(), "/"))));
    }
  }
Ejemplo n.º 3
0
 @Override
 public String[] list(String path) throws IOException {
   // if the path not exists, or it is a file, then should return null
   if (!exists(path) || isFile(path)) {
     return null;
   }
   // Non recursive list
   path = PathUtils.normalizePath(path, PATH_SEPARATOR);
   return listInternal(path, false);
 }
Ejemplo n.º 4
0
 /**
  * Lists the files in the given path, the paths will be their logical names and not contain the
  * folder suffix.
  *
  * @param path the key to list
  * @param recursive if true will list children directories as well
  * @return an array of the file and folder names in this directory
  * @throws IOException if an I/O error occurs
  */
 private String[] listInternal(String path, boolean recursive) throws IOException {
   try {
     path = stripPrefixIfPresent(path);
     path = PathUtils.normalizePath(path, PATH_SEPARATOR);
     path = path.equals(PATH_SEPARATOR) ? "" : path;
     // Gets all the objects under the path, because we have no idea if there are non Alluxio
     // managed "directories"
     S3Object[] objs = mClient.listObjects(mBucketName, path, "");
     if (recursive) {
       List<String> ret = new ArrayList<>();
       for (S3Object obj : objs) {
         // Remove parent portion of the key
         String child = getChildName(obj.getKey(), path);
         // Prune the special folder suffix
         child = stripFolderSuffixIfPresent(child);
         // Only add if the path is not empty (removes results equal to the path)
         if (!child.isEmpty()) {
           ret.add(child);
         }
       }
       return ret.toArray(new String[ret.size()]);
     }
     // Non recursive list
     Set<String> children = new HashSet<String>();
     for (S3Object obj : objs) {
       // Remove parent portion of the key
       String child = getChildName(obj.getKey(), path);
       // Remove any portion after the path delimiter
       int childNameIndex = child.indexOf(PATH_SEPARATOR);
       child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
       // Prune the special folder suffix
       child = stripFolderSuffixIfPresent(child);
       // Add to the set of children, the set will deduplicate.
       if (!child.isEmpty()) {
         children.add(child);
       }
     }
     return children.toArray(new String[children.size()]);
   } catch (ServiceException e) {
     LOG.error("Failed to list path {}", path, e);
     return null;
   }
 }
Ejemplo n.º 5
0
 /**
  * A trailing {@link SwiftUnderFileSystem#FOLDER_SUFFIX} is added if not present.
  *
  * @param path URI to the object
  * @return folder path
  */
 private String addFolderSuffixIfNotPresent(String path) {
   return PathUtils.normalizePath(path, FOLDER_SUFFIX);
 }
Ejemplo n.º 6
0
 /**
  * Checks if the key is the root.
  *
  * @param key the key to check
  * @return true if the key is the root, false otherwise
  */
 private boolean isRoot(String key) {
   return PathUtils.normalizePath(key, PATH_SEPARATOR)
       .equals(PathUtils.normalizePath(Constants.HEADER_S3N + mBucketName, PATH_SEPARATOR));
 }