/**
* Constructs new shared file system repository
*
* @param name repository name
* @param repositorySettings repository settings
* @param indexShardRepository index shard repository
* @param s3Service S3 service
* @throws IOException
*/
@Inject
public S3Repository(
RepositoryName name,
RepositorySettings repositorySettings,
IndexShardRepository indexShardRepository,
AwsS3Service s3Service)
throws IOException {
super(name.getName(), repositorySettings, indexShardRepository);
String bucket = repositorySettings.settings().get("bucket", componentSettings.get("bucket"));
if (bucket == null) {
throw new RepositoryException(name.name(), "No bucket defined for s3 gateway");
}
String region = repositorySettings.settings().get("region", componentSettings.get("region"));
if (region == null) {
// Bucket setting is not set - use global region setting
String regionSetting =
repositorySettings.settings().get("cloud.aws.region", settings.get("cloud.aws.region"));
if (regionSetting != null) {
regionSetting = regionSetting.toLowerCase(Locale.ENGLISH);
if ("us-east".equals(regionSetting)) {
// Default bucket - setting region to null
region = null;
} else if ("us-east-1".equals(regionSetting)) {
region = null;
} else if ("us-west".equals(regionSetting)) {
region = "us-west-1";
} else if ("us-west-1".equals(regionSetting)) {
region = "us-west-1";
} else if ("us-west-2".equals(regionSetting)) {
region = "us-west-2";
} else if ("ap-southeast".equals(regionSetting)) {
region = "ap-southeast-1";
} else if ("ap-southeast-1".equals(regionSetting)) {
region = "ap-southeast-1";
} else if ("ap-southeast-2".equals(regionSetting)) {
region = "ap-southeast-2";
} else if ("ap-northeast".equals(regionSetting)) {
region = "ap-northeast-1";
} else if ("ap-northeast-1".equals(regionSetting)) {
region = "ap-northeast-1";
} else if ("eu-west".equals(regionSetting)) {
region = "EU";
} else if ("eu-west-1".equals(regionSetting)) {
region = "EU";
} else if ("sa-east".equals(regionSetting)) {
region = "sa-east-1";
} else if ("sa-east-1".equals(regionSetting)) {
region = "sa-east-1";
}
}
}
int concurrentStreams =
repositorySettings
.settings()
.getAsInt("concurrent_streams", componentSettings.getAsInt("concurrent_streams", 5));
ExecutorService concurrentStreamPool =
EsExecutors.newScaling(
1,
concurrentStreams,
5,
TimeUnit.SECONDS,
EsExecutors.daemonThreadFactory(settings, "[s3_stream]"));
boolean serverSideEncryption =
repositorySettings
.settings()
.getAsBoolean(
"server_side_encryption",
componentSettings.getAsBoolean("server_side_encryption", false));
logger.debug(
"using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}], server_side_encryption [{}]",
bucket,
region,
chunkSize,
concurrentStreams,
serverSideEncryption);
blobStore =
new S3BlobStore(
settings,
s3Service.client(
region,
repositorySettings.settings().get("access_key"),
repositorySettings.settings().get("secret_key")),
bucket,
region,
concurrentStreamPool,
serverSideEncryption);
this.chunkSize =
repositorySettings
.settings()
.getAsBytesSize(
"chunk_size",
componentSettings.getAsBytesSize(
"chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB)));
this.compress =
repositorySettings
.settings()
.getAsBoolean("compress", componentSettings.getAsBoolean("compress", false));
String basePath = repositorySettings.settings().get("base_path", null);
if (Strings.hasLength(basePath)) {
BlobPath path = new BlobPath();
for (String elem : Strings.splitStringToArray(basePath, '/')) {
path = path.add(elem);
}
this.basePath = path;
} else {
this.basePath = BlobPath.cleanPath();
}
}
@Inject
public S3Gateway(
Settings settings,
ThreadPool threadPool,
ClusterService clusterService,
ClusterName clusterName,
AwsS3Service s3Service)
throws IOException {
super(settings, threadPool, clusterService);
String bucket = componentSettings.get("bucket");
if (bucket == null) {
throw new ElasticSearchIllegalArgumentException("No bucket defined for s3 gateway");
}
String region = componentSettings.get("region");
if (region == null) {
if (settings.get("cloud.aws.region") != null) {
String regionSetting = settings.get("cloud.aws.region");
if ("us-east".equals(regionSetting.toLowerCase())) {
region = null;
} else if ("us-east-1".equals(regionSetting.toLowerCase())) {
region = null;
} else if ("us-west".equals(regionSetting.toLowerCase())) {
region = "us-west-1";
} else if ("us-west-1".equals(regionSetting.toLowerCase())) {
region = "us-west-1";
} else if ("us-west-2".equals(regionSetting.toLowerCase())) {
region = "us-west-2";
} else if ("ap-southeast".equals(regionSetting.toLowerCase())) {
region = "ap-southeast-1";
} else if ("ap-southeast-1".equals(regionSetting.toLowerCase())) {
region = "ap-southeast-1";
} else if ("ap-southeast-2".equals(regionSetting.toLowerCase())) {
region = "ap-southeast-2";
} else if ("eu-west".equals(regionSetting.toLowerCase())) {
region = "EU";
} else if ("eu-west-1".equals(regionSetting.toLowerCase())) {
region = "EU";
} else if ("sa-east".equals(regionSetting.toLowerCase())) {
region = "sa-east-1";
} else if ("sa-east-1".equals(regionSetting.toLowerCase())) {
region = "sa-east-1";
}
}
}
ByteSizeValue chunkSize =
componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB));
int concurrentStreams = componentSettings.getAsInt("concurrent_streams", 5);
this.concurrentStreamPool =
EsExecutors.newScalingExecutorService(
1,
concurrentStreams,
5,
TimeUnit.SECONDS,
EsExecutors.daemonThreadFactory(settings, "[s3_stream]"));
logger.debug(
"using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}]",
bucket,
region,
chunkSize,
concurrentStreams);
initialize(
new S3BlobStore(settings, s3Service.client(), bucket, region, concurrentStreamPool),
clusterName,
chunkSize);
}