@Test public void mustDeleteEmptyMetadata() { Account account = new AccountMock(); Container container = account.getContainer("use-this"); container.create(); container.setMetadata(convertToMetadata(new String[][] {{"Field-1", "value 1"}})); container.setMetadata(convertToMetadata(new String[][] {{"Field-1", ""}})); assertEquals(0, container.getMetadata().size()); }
@Test public void fetchMetadataAfterSet() { Swift swift = new Swift(); Account account = new AccountMock(swift).setAllowContainerCaching(false); Container container = account.getContainer("use-this"); container.create(); container.setMetadata(convertToMetadata(new String[][] {{"Field-1", "value 1"}})); Container container2 = account.getContainer("use-this"); container2.setMetadata(convertToMetadata(new String[][] {{"Field-2", "value 2"}})); assertEquals(2, container2.getMetadata().size()); }
/** * Copies an object to another name. Destination will be overwritten if it already exists. * * @param source the source path to copy * @param destination the destination path to copy to * @return true if the operation was successful, false otherwise */ private boolean copy(String source, String destination) { LOG.debug("copy from {} to {}", source, destination); final String strippedSourcePath = stripContainerPrefixIfPresent(source); final String strippedDestinationPath = stripContainerPrefixIfPresent(destination); // Retry copy for a few times, in case some Swift internal errors happened during copy. for (int i = 0; i < NUM_RETRIES; i++) { try { Container container = mAccount.getContainer(mContainerName); container .getObject(strippedSourcePath) .copyObject(container, container.getObject(strippedDestinationPath)); return true; } catch (NotFoundException e) { LOG.error("Source path {} does not exist", source); return false; } catch (Exception e) { LOG.error("Failed to copy file {} to {}", source, destination, e.getMessage()); if (i != NUM_RETRIES - 1) { LOG.error("Retrying copying file {} to {}", source, destination); } } } LOG.error("Failed to copy file {} to {}, after {} retries", source, destination, NUM_RETRIES); return false; }
@Override public boolean delete(String path, boolean recursive) throws IOException { LOG.debug("Delete method: {}, recursive {}", path, recursive); final String strippedPath = stripContainerPrefixIfPresent(path); Container container = mAccount.getContainer(mContainerName); if (recursive) { // For a file, recursive delete will not find any children PaginationMap paginationMap = container.getPaginationMap(addFolderSuffixIfNotPresent(strippedPath), DIR_PAGE_SIZE); for (int page = 0; page < paginationMap.getNumberOfPages(); page++) { for (StoredObject childObject : container.list(paginationMap, page)) { deleteObject(childObject); } } } else { String[] children = list(path); if (children != null && children.length != 0) { LOG.error("Attempting to non-recursively delete a non-empty directory."); return false; } } // Path is a file or folder with no children if (!deleteObject(container.getObject(strippedPath))) { // Path may be a folder final String strippedFolderPath = addFolderSuffixIfNotPresent(strippedPath); if (strippedFolderPath.equals(strippedPath)) { return false; } return deleteObject(container.getObject(strippedFolderPath)); } return true; }
/** * Constructs a new Swift {@link UnderFileSystem}. * * @param uri the {@link AlluxioURI} for this UFS */ public SwiftUnderFileSystem(AlluxioURI uri) { super(uri); String containerName = uri.getHost(); LOG.debug("Constructor init: {}", containerName); AccountConfig config = new AccountConfig(); if (Configuration.containsKey(Constants.SWIFT_API_KEY)) { config.setPassword(Configuration.get(Constants.SWIFT_API_KEY)); } else if (Configuration.containsKey(Constants.SWIFT_PASSWORD_KEY)) { config.setPassword(Configuration.get(Constants.SWIFT_PASSWORD_KEY)); } config.setAuthUrl(Configuration.get(Constants.SWIFT_AUTH_URL_KEY)); String authMethod = Configuration.get(Constants.SWIFT_AUTH_METHOD_KEY); if (authMethod != null) { config.setUsername(Configuration.get(Constants.SWIFT_USER_KEY)); config.setTenantName(Configuration.get(Constants.SWIFT_TENANT_KEY)); switch (authMethod) { case Constants.SWIFT_AUTH_KEYSTONE: config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE); break; case Constants.SWIFT_AUTH_SWIFTAUTH: // swiftauth authenticates directly against swift // note: this method is supported in swift object storage api v1 config.setAuthenticationMethod(AuthenticationMethod.BASIC); break; default: config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH); // tempauth requires authentication header to be of the form tenant:user. // JOSS however generates header of the form user:tenant. // To resolve this, we switch user with tenant config.setTenantName(Configuration.get(Constants.SWIFT_USER_KEY)); config.setUsername(Configuration.get(Constants.SWIFT_TENANT_KEY)); } } ObjectMapper mapper = new ObjectMapper(); mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true); mContainerName = containerName; mAccount = new AccountFactory(config).createAccount(); // Do not allow container cache to avoid stale directory listings mAccount.setAllowContainerCaching(false); mAccess = mAccount.authenticate(); Container container = mAccount.getContainer(containerName); if (!container.exists()) { container.create(); } mContainerPrefix = Constants.HEADER_SWIFT + mContainerName + PATH_SEPARATOR; }
/** * Constructor. Sets up the container mostly. * * @param settings Settings for our repository. Only care about buffer size. * @param auth * @param container * @param executor */ public SwiftBlobStore(Settings settings, Account auth, String container, Executor executor) { super(settings); this.executor = executor; this.bufferSizeInBytes = (int) settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes(); swift = auth.getContainer(container); if (!swift.exists()) { swift.create(); swift.makePublic(); } }
/** Logs into the Object store, but note that the token only works for 24 hours. FIXME: Test */ private void login() { if (swiftUsername == null || swiftPassword == null || swiftAuthUrl == null) { System.out.println("ERROR: Swift storage account is not configured"); logger.error("Swift storage account is not configured"); } config = new AccountConfig(); config.setUsername(swiftUsername); config.setPassword(swiftPassword); config.setAuthUrl(swiftAuthUrl); config.setTenantId(swiftTenantId); // config.setTenantName(swiftTenantName); if (swiftMock != null) { config.setMock(Boolean.valueOf(swiftMock)); } account = new AccountFactory(config).createAccount(); container = account.getContainer(swiftContainer); if (!container.exists()) { container.create(); } // container.makePublic(); }
/** * Retrieves a handle to an object identified by the given path. * * @param path the path to retrieve an object handle for * @return the object handle */ private StoredObject getObject(final String path) { Container container = mAccount.getContainer(mContainerName); return container.getObject(stripContainerPrefixIfPresent(path)); }
/** * Lists the files or folders which match the given prefix. * * @param prefix the prefix to match * @return a collection of the files or folders matching the prefix, or null if not found * @throws IOException if path is not accessible, e.g. network issues */ private Collection<DirectoryOrObject> listInternal(final String prefix) throws IOException { // TODO(adit): UnderFileSystem interface should be changed to support pagination Directory directory = new Directory(prefix, PATH_SEPARATOR_CHAR); Container container = mAccount.getContainer(mContainerName); return container.listDirectory(directory); }