@Test
public void fetchMetadataAfterSet() {
Swift swift = new Swift();
Account account = new AccountMock(swift).setAllowContainerCaching(false);
Container container = account.getContainer("use-this");
container.create();
container.setMetadata(convertToMetadata(new String[][] {{"Field-1", "value 1"}}));
Container container2 = account.getContainer("use-this");
container2.setMetadata(convertToMetadata(new String[][] {{"Field-2", "value 2"}}));
assertEquals(2, container2.getMetadata().size());
}
/**
* Copies an object to another name. Destination will be overwritten if it already exists.
*
* @param source the source path to copy
* @param destination the destination path to copy to
* @return true if the operation was successful, false otherwise
*/
private boolean copy(String source, String destination) {
LOG.debug("copy from {} to {}", source, destination);
final String strippedSourcePath = stripContainerPrefixIfPresent(source);
final String strippedDestinationPath = stripContainerPrefixIfPresent(destination);
// Retry copy for a few times, in case some Swift internal errors happened during copy.
for (int i = 0; i < NUM_RETRIES; i++) {
try {
Container container = mAccount.getContainer(mContainerName);
container
.getObject(strippedSourcePath)
.copyObject(container, container.getObject(strippedDestinationPath));
return true;
} catch (NotFoundException e) {
LOG.error("Source path {} does not exist", source);
return false;
} catch (Exception e) {
LOG.error("Failed to copy file {} to {}", source, destination, e.getMessage());
if (i != NUM_RETRIES - 1) {
LOG.error("Retrying copying file {} to {}", source, destination);
}
}
}
LOG.error("Failed to copy file {} to {}, after {} retries", source, destination, NUM_RETRIES);
return false;
}
@Override
public boolean delete(String path, boolean recursive) throws IOException {
LOG.debug("Delete method: {}, recursive {}", path, recursive);
final String strippedPath = stripContainerPrefixIfPresent(path);
Container container = mAccount.getContainer(mContainerName);
if (recursive) {
// For a file, recursive delete will not find any children
PaginationMap paginationMap =
container.getPaginationMap(addFolderSuffixIfNotPresent(strippedPath), DIR_PAGE_SIZE);
for (int page = 0; page < paginationMap.getNumberOfPages(); page++) {
for (StoredObject childObject : container.list(paginationMap, page)) {
deleteObject(childObject);
}
}
} else {
String[] children = list(path);
if (children != null && children.length != 0) {
LOG.error("Attempting to non-recursively delete a non-empty directory.");
return false;
}
}
// Path is a file or folder with no children
if (!deleteObject(container.getObject(strippedPath))) {
// Path may be a folder
final String strippedFolderPath = addFolderSuffixIfNotPresent(strippedPath);
if (strippedFolderPath.equals(strippedPath)) {
return false;
}
return deleteObject(container.getObject(strippedFolderPath));
}
return true;
}
@Test
public void mustDeleteEmptyMetadata() {
Account account = new AccountMock();
Container container = account.getContainer("use-this");
container.create();
container.setMetadata(convertToMetadata(new String[][] {{"Field-1", "value 1"}}));
container.setMetadata(convertToMetadata(new String[][] {{"Field-1", ""}}));
assertEquals(0, container.getMetadata().size());
}
/**
* Constructor. Sets up the container mostly.
*
* @param settings Settings for our repository. Only care about buffer size.
* @param auth
* @param container
* @param executor
*/
public SwiftBlobStore(Settings settings, Account auth, String container, Executor executor) {
super(settings);
this.executor = executor;
this.bufferSizeInBytes =
(int)
settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes();
swift = auth.getContainer(container);
if (!swift.exists()) {
swift.create();
swift.makePublic();
}
}
/**
* Constructs a new Swift {@link UnderFileSystem}.
*
* @param uri the {@link AlluxioURI} for this UFS
*/
public SwiftUnderFileSystem(AlluxioURI uri) {
super(uri);
String containerName = uri.getHost();
LOG.debug("Constructor init: {}", containerName);
AccountConfig config = new AccountConfig();
if (Configuration.containsKey(Constants.SWIFT_API_KEY)) {
config.setPassword(Configuration.get(Constants.SWIFT_API_KEY));
} else if (Configuration.containsKey(Constants.SWIFT_PASSWORD_KEY)) {
config.setPassword(Configuration.get(Constants.SWIFT_PASSWORD_KEY));
}
config.setAuthUrl(Configuration.get(Constants.SWIFT_AUTH_URL_KEY));
String authMethod = Configuration.get(Constants.SWIFT_AUTH_METHOD_KEY);
if (authMethod != null) {
config.setUsername(Configuration.get(Constants.SWIFT_USER_KEY));
config.setTenantName(Configuration.get(Constants.SWIFT_TENANT_KEY));
switch (authMethod) {
case Constants.SWIFT_AUTH_KEYSTONE:
config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
break;
case Constants.SWIFT_AUTH_SWIFTAUTH:
// swiftauth authenticates directly against swift
// note: this method is supported in swift object storage api v1
config.setAuthenticationMethod(AuthenticationMethod.BASIC);
break;
default:
config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
// tempauth requires authentication header to be of the form tenant:user.
// JOSS however generates header of the form user:tenant.
// To resolve this, we switch user with tenant
config.setTenantName(Configuration.get(Constants.SWIFT_USER_KEY));
config.setUsername(Configuration.get(Constants.SWIFT_TENANT_KEY));
}
}
ObjectMapper mapper = new ObjectMapper();
mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
mContainerName = containerName;
mAccount = new AccountFactory(config).createAccount();
// Do not allow container cache to avoid stale directory listings
mAccount.setAllowContainerCaching(false);
mAccess = mAccount.authenticate();
Container container = mAccount.getContainer(containerName);
if (!container.exists()) {
container.create();
}
mContainerPrefix = Constants.HEADER_SWIFT + mContainerName + PATH_SEPARATOR;
}
/** Logs into the Object store, but note that the token only works for 24 hours. FIXME: Test */
private void login() {
if (swiftUsername == null || swiftPassword == null || swiftAuthUrl == null) {
System.out.println("ERROR: Swift storage account is not configured");
logger.error("Swift storage account is not configured");
}
config = new AccountConfig();
config.setUsername(swiftUsername);
config.setPassword(swiftPassword);
config.setAuthUrl(swiftAuthUrl);
config.setTenantId(swiftTenantId);
// config.setTenantName(swiftTenantName);
if (swiftMock != null) {
config.setMock(Boolean.valueOf(swiftMock));
}
account = new AccountFactory(config).createAccount();
container = account.getContainer(swiftContainer);
if (!container.exists()) {
container.create();
}
// container.makePublic();
}
/**
* Retrieves a handle to an object identified by the given path.
*
* @param path the path to retrieve an object handle for
* @return the object handle
*/
private StoredObject getObject(final String path) {
Container container = mAccount.getContainer(mContainerName);
return container.getObject(stripContainerPrefixIfPresent(path));
}
/**
* Lists the files or folders which match the given prefix.
*
* @param prefix the prefix to match
* @return a collection of the files or folders matching the prefix, or null if not found
* @throws IOException if path is not accessible, e.g. network issues
*/
private Collection<DirectoryOrObject> listInternal(final String prefix) throws IOException {
// TODO(adit): UnderFileSystem interface should be changed to support pagination
Directory directory = new Directory(prefix, PATH_SEPARATOR_CHAR);
Container container = mAccount.getContainer(mContainerName);
return container.listDirectory(directory);
}