/**
* Constructs a new instance of {@link S3UnderFileSystem}.
*
* @param uri the {@link AlluxioURI} for this UFS
* @param conf the configuration for Alluxio
* @param awsCredentials AWS Credentials configuration for S3 Access
* @throws ServiceException when a connection to S3 could not be created
*/
public S3UnderFileSystem(AlluxioURI uri, Configuration conf, AWSCredentials awsCredentials)
throws ServiceException {
super(uri, conf);
String bucketName = uri.getHost();
mBucketName = bucketName;
Jets3tProperties props = new Jets3tProperties();
if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
props.setProperty("httpclient.proxy-autodetect", "false");
props.setProperty("httpclient.proxy-host", conf.get(Constants.UNDERFS_S3_PROXY_HOST));
props.setProperty("httpclient.proxy-port", conf.get(Constants.UNDERFS_S3_PROXY_PORT));
}
if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.https-only",
Boolean.toString(conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
}
if (conf.containsKey(Constants.UNDERFS_S3_ENDPOINT)) {
props.setProperty("s3service.s3-endpoint", conf.get(Constants.UNDERFS_S3_ENDPOINT));
if (conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.s3-endpoint-https-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTPS_PORT));
} else {
props.setProperty(
"s3service.s3-endpoint-http-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTP_PORT));
}
}
if (conf.containsKey(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
props.setProperty(
"s3service.disable-dns-buckets", conf.get(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS));
}
LOG.debug("Initializing S3 underFs with properties: {}", props.getProperties());
mClient = new RestS3Service(awsCredentials, null, null, props);
mBucketPrefix = PathUtils.normalizePath(Constants.HEADER_S3N + mBucketName, PATH_SEPARATOR);
}
/**
* Constructs a new Swift {@link UnderFileSystem}.
*
* @param uri the {@link AlluxioURI} for this UFS
*/
public SwiftUnderFileSystem(AlluxioURI uri) {
super(uri);
String containerName = uri.getHost();
LOG.debug("Constructor init: {}", containerName);
AccountConfig config = new AccountConfig();
if (Configuration.containsKey(Constants.SWIFT_API_KEY)) {
config.setPassword(Configuration.get(Constants.SWIFT_API_KEY));
} else if (Configuration.containsKey(Constants.SWIFT_PASSWORD_KEY)) {
config.setPassword(Configuration.get(Constants.SWIFT_PASSWORD_KEY));
}
config.setAuthUrl(Configuration.get(Constants.SWIFT_AUTH_URL_KEY));
String authMethod = Configuration.get(Constants.SWIFT_AUTH_METHOD_KEY);
if (authMethod != null) {
config.setUsername(Configuration.get(Constants.SWIFT_USER_KEY));
config.setTenantName(Configuration.get(Constants.SWIFT_TENANT_KEY));
switch (authMethod) {
case Constants.SWIFT_AUTH_KEYSTONE:
config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
break;
case Constants.SWIFT_AUTH_SWIFTAUTH:
// swiftauth authenticates directly against swift
// note: this method is supported in swift object storage api v1
config.setAuthenticationMethod(AuthenticationMethod.BASIC);
break;
default:
config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
// tempauth requires authentication header to be of the form tenant:user.
// JOSS however generates header of the form user:tenant.
// To resolve this, we switch user with tenant
config.setTenantName(Configuration.get(Constants.SWIFT_USER_KEY));
config.setUsername(Configuration.get(Constants.SWIFT_TENANT_KEY));
}
}
ObjectMapper mapper = new ObjectMapper();
mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
mContainerName = containerName;
mAccount = new AccountFactory(config).createAccount();
// Do not allow container cache to avoid stale directory listings
mAccount.setAllowContainerCaching(false);
mAccess = mAccount.authenticate();
Container container = mAccount.getContainer(containerName);
if (!container.exists()) {
container.create();
}
mContainerPrefix = Constants.HEADER_SWIFT + mContainerName + PATH_SEPARATOR;
}