コード例 #1
0
  @Override
  public String[] list(String path) throws IOException {
    String prefix = addFolderSuffixIfNotPresent(stripContainerPrefixIfPresent(path));
    prefix = prefix.equals(PATH_SEPARATOR) ? "" : prefix;

    Collection<DirectoryOrObject> objects = listInternal(prefix);
    Set<String> children = new HashSet<>();
    final String self = stripFolderSuffixIfPresent(prefix);
    boolean foundSelf = false;
    for (DirectoryOrObject object : objects) {
      String child = stripFolderSuffixIfPresent(object.getName());
      String noPrefix = CommonUtils.stripPrefixIfPresent(child, prefix);
      if (!noPrefix.equals(self)) {
        children.add(noPrefix);
      } else {
        foundSelf = true;
      }
    }

    if (!foundSelf) {
      // Path does not exist
      return null;
    }

    return children.toArray(new String[children.size()]);
  }
コード例 #2
0
  /** Tests if list correctly returns file or folder names for a large directory. */
  @Test
  public void listLargeDirectory() throws IOException {
    LargeDirectoryConfig config = prepareLargeDirectoryTest();
    String[] children = config.getChildren();

    // Retry for some time to allow list operation eventual consistency for S3 and GCS.
    // See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
    // https://cloud.google.com/storage/docs/consistency for more details.
    // Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
    String[] results = new String[] {};
    for (int i = 0; i < 20; i++) {
      results = mUfs.list(config.getTopLevelDirectory());
      if (children.length == results.length) {
        break;
      }
      CommonUtils.sleepMs(500);
    }
    Assert.assertEquals(children.length, results.length);

    Arrays.sort(results);
    for (int i = 0; i < children.length; ++i) {
      Assert.assertTrue(
          results[i].equals(
              CommonUtils.stripPrefixIfPresent(
                  children[i], PathUtils.normalizePath(config.getTopLevelDirectory(), "/"))));
    }
  }
コード例 #3
0
 /**
  * Creates a directory flagged file with the folder suffix.
  *
  * @param path the path to create a folder
  * @return true if the operation was successful, false otherwise
  */
 private boolean mkdirsInternal(String path) {
   try {
     String keyAsFolder =
         addFolderSuffixIfNotPresent(
             CommonUtils.stripPrefixIfPresent(path, Constants.HEADER_SWIFT));
     // We do not check if a file with same name exists, i.e. a file with name
     // 'swift://swift-container/path' and a folder with name 'swift://swift-container/path/'
     // may both exist simultaneously
     SwiftDirectClient.put(mAccess, keyAsFolder).close();
     return true;
   } catch (IOException e) {
     LOG.error("Failed to create directory: {}", path, e);
     return false;
   }
 }
コード例 #4
0
  @Override
  public OutputStream create(String path, CreateOptions options) throws IOException {
    LOG.debug("Create method: {}", path);

    // create will attempt to create the parent directory if it does not already exist
    if (!mkdirs(getParentPath(path), true)) {
      // fail if the parent directory does not exist and creation was unsuccessful
      LOG.error("Parent directory creation unsuccessful for {}", path);
      return null;
    }

    final String strippedPath = CommonUtils.stripPrefixIfPresent(path, Constants.HEADER_SWIFT);
    // TODO(adit): remove special handling of */_SUCCESS objects
    if (strippedPath.endsWith("_SUCCESS")) {
      // when path/_SUCCESS is created, there is need to create path as
      // an empty object. This is required by Spark in case Spark
      // accesses path directly, bypassing Alluxio
      String plainName = CommonUtils.stripSuffixIfPresent(strippedPath, "_SUCCESS");
      SwiftDirectClient.put(mAccess, plainName).close();
    }
    // We do not check if a folder with the same name exists
    return SwiftDirectClient.put(mAccess, strippedPath);
  }
コード例 #5
0
 /**
  * Strips the Swift container prefix from the path if it is present. For example, for input path
  * swift://my-container-name/my-path/file, the output would be my-path/file. This method will
  * leave paths without a prefix unaltered, ie. my-path/file returns my-path/file.
  *
  * @param path the path to strip
  * @return the path without the Swift container prefix
  */
 private String stripContainerPrefixIfPresent(final String path) {
   return CommonUtils.stripPrefixIfPresent(path, mContainerPrefix);
 }