Example #1
0
 @Test
 public void mustDeleteEmptyMetadata() {
   Account account = new AccountMock();
   Container container = account.getContainer("use-this");
   container.create();
   container.setMetadata(convertToMetadata(new String[][] {{"Field-1", "value 1"}}));
   container.setMetadata(convertToMetadata(new String[][] {{"Field-1", ""}}));
   assertEquals(0, container.getMetadata().size());
 }
Example #2
0
 @Test
 public void fetchMetadataAfterSet() {
   Swift swift = new Swift();
   Account account = new AccountMock(swift).setAllowContainerCaching(false);
   Container container = account.getContainer("use-this");
   container.create();
   container.setMetadata(convertToMetadata(new String[][] {{"Field-1", "value 1"}}));
   Container container2 = account.getContainer("use-this");
   container2.setMetadata(convertToMetadata(new String[][] {{"Field-2", "value 2"}}));
   assertEquals(2, container2.getMetadata().size());
 }
 /**
  * Copies an object to another name. Destination will be overwritten if it already exists.
  *
  * @param source the source path to copy
  * @param destination the destination path to copy to
  * @return true if the operation was successful, false otherwise
  */
 private boolean copy(String source, String destination) {
   LOG.debug("copy from {} to {}", source, destination);
   final String strippedSourcePath = stripContainerPrefixIfPresent(source);
   final String strippedDestinationPath = stripContainerPrefixIfPresent(destination);
   // Retry copy for a few times, in case some Swift internal errors happened during copy.
   for (int i = 0; i < NUM_RETRIES; i++) {
     try {
       Container container = mAccount.getContainer(mContainerName);
       container
           .getObject(strippedSourcePath)
           .copyObject(container, container.getObject(strippedDestinationPath));
       return true;
     } catch (NotFoundException e) {
       LOG.error("Source path {} does not exist", source);
       return false;
     } catch (Exception e) {
       LOG.error("Failed to copy file {} to {}", source, destination, e.getMessage());
       if (i != NUM_RETRIES - 1) {
         LOG.error("Retrying copying file {} to {}", source, destination);
       }
     }
   }
   LOG.error("Failed to copy file {} to {}, after {} retries", source, destination, NUM_RETRIES);
   return false;
 }
  @Override
  public boolean delete(String path, boolean recursive) throws IOException {
    LOG.debug("Delete method: {}, recursive {}", path, recursive);
    final String strippedPath = stripContainerPrefixIfPresent(path);
    Container container = mAccount.getContainer(mContainerName);
    if (recursive) {
      // For a file, recursive delete will not find any children
      PaginationMap paginationMap =
          container.getPaginationMap(addFolderSuffixIfNotPresent(strippedPath), DIR_PAGE_SIZE);
      for (int page = 0; page < paginationMap.getNumberOfPages(); page++) {
        for (StoredObject childObject : container.list(paginationMap, page)) {
          deleteObject(childObject);
        }
      }
    } else {
      String[] children = list(path);
      if (children != null && children.length != 0) {
        LOG.error("Attempting to non-recursively delete a non-empty directory.");
        return false;
      }
    }

    // Path is a file or folder with no children
    if (!deleteObject(container.getObject(strippedPath))) {
      // Path may be a folder
      final String strippedFolderPath = addFolderSuffixIfNotPresent(strippedPath);
      if (strippedFolderPath.equals(strippedPath)) {
        return false;
      }
      return deleteObject(container.getObject(strippedFolderPath));
    }
    return true;
  }
  /**
   * Constructs a new Swift {@link UnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} for this UFS
   */
  public SwiftUnderFileSystem(AlluxioURI uri) {
    super(uri);
    String containerName = uri.getHost();
    LOG.debug("Constructor init: {}", containerName);
    AccountConfig config = new AccountConfig();
    if (Configuration.containsKey(Constants.SWIFT_API_KEY)) {
      config.setPassword(Configuration.get(Constants.SWIFT_API_KEY));
    } else if (Configuration.containsKey(Constants.SWIFT_PASSWORD_KEY)) {
      config.setPassword(Configuration.get(Constants.SWIFT_PASSWORD_KEY));
    }
    config.setAuthUrl(Configuration.get(Constants.SWIFT_AUTH_URL_KEY));
    String authMethod = Configuration.get(Constants.SWIFT_AUTH_METHOD_KEY);
    if (authMethod != null) {
      config.setUsername(Configuration.get(Constants.SWIFT_USER_KEY));
      config.setTenantName(Configuration.get(Constants.SWIFT_TENANT_KEY));
      switch (authMethod) {
        case Constants.SWIFT_AUTH_KEYSTONE:
          config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
          break;
        case Constants.SWIFT_AUTH_SWIFTAUTH:
          // swiftauth authenticates directly against swift
          // note: this method is supported in swift object storage api v1
          config.setAuthenticationMethod(AuthenticationMethod.BASIC);
          break;
        default:
          config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
          // tempauth requires authentication header to be of the form tenant:user.
          // JOSS however generates header of the form user:tenant.
          // To resolve this, we switch user with tenant
          config.setTenantName(Configuration.get(Constants.SWIFT_USER_KEY));
          config.setUsername(Configuration.get(Constants.SWIFT_TENANT_KEY));
      }
    }

    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
    mContainerName = containerName;
    mAccount = new AccountFactory(config).createAccount();
    // Do not allow container cache to avoid stale directory listings
    mAccount.setAllowContainerCaching(false);
    mAccess = mAccount.authenticate();
    Container container = mAccount.getContainer(containerName);
    if (!container.exists()) {
      container.create();
    }
    mContainerPrefix = Constants.HEADER_SWIFT + mContainerName + PATH_SEPARATOR;
  }
  /**
   * Constructor. Sets up the container mostly.
   *
   * @param settings Settings for our repository. Only care about buffer size.
   * @param auth
   * @param container
   * @param executor
   */
  public SwiftBlobStore(Settings settings, Account auth, String container, Executor executor) {
    super(settings);
    this.executor = executor;
    this.bufferSizeInBytes =
        (int)
            settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes();

    swift = auth.getContainer(container);
    if (!swift.exists()) {
      swift.create();
      swift.makePublic();
    }
  }
 /** Logs into the Object store, but note that the token only works for 24 hours. FIXME: Test */
 private void login() {
   if (swiftUsername == null || swiftPassword == null || swiftAuthUrl == null) {
     System.out.println("ERROR: Swift storage account is not configured");
     logger.error("Swift storage account is not configured");
   }
   config = new AccountConfig();
   config.setUsername(swiftUsername);
   config.setPassword(swiftPassword);
   config.setAuthUrl(swiftAuthUrl);
   config.setTenantId(swiftTenantId);
   //      config.setTenantName(swiftTenantName);
   if (swiftMock != null) {
     config.setMock(Boolean.valueOf(swiftMock));
   }
   account = new AccountFactory(config).createAccount();
   container = account.getContainer(swiftContainer);
   if (!container.exists()) {
     container.create();
   }
   // container.makePublic();
 }
 /**
  * Retrieves a handle to an object identified by the given path.
  *
  * @param path the path to retrieve an object handle for
  * @return the object handle
  */
 private StoredObject getObject(final String path) {
   Container container = mAccount.getContainer(mContainerName);
   return container.getObject(stripContainerPrefixIfPresent(path));
 }
 /**
  * Lists the files or folders which match the given prefix.
  *
  * @param prefix the prefix to match
  * @return a collection of the files or folders matching the prefix, or null if not found
  * @throws IOException if path is not accessible, e.g. network issues
  */
 private Collection<DirectoryOrObject> listInternal(final String prefix) throws IOException {
   // TODO(adit): UnderFileSystem interface should be changed to support pagination
   Directory directory = new Directory(prefix, PATH_SEPARATOR_CHAR);
   Container container = mAccount.getContainer(mContainerName);
   return container.listDirectory(directory);
 }