protected void initialize(
BlobStore blobStore, ClusterName clusterName, @Nullable ByteSizeValue defaultChunkSize)
throws IOException {
this.blobStore = blobStore;
this.chunkSize = componentSettings.getAsBytesSize("chunk_size", defaultChunkSize);
this.basePath = BlobPath.cleanPath().add(clusterName.value());
this.metaDataBlobContainer = blobStore.immutableBlobContainer(basePath.add("metadata"));
this.currentIndex = findLatestIndex();
this.compress = componentSettings.getAsBoolean("compress", true);
logger.debug("Latest metadata found at index [" + currentIndex + "]");
}
/**
* Delete an arbitrary BlobPath from our store.
*
* @param path The blob path to delete
*/
@Override
public void delete(BlobPath path) {
String keyPath = path.buildAsString("/");
if (!keyPath.isEmpty()) {
keyPath = keyPath + "/";
}
StoredObject obj = swift().getObject(keyPath);
if (obj.exists()) {
obj.delete();
}
}
/**
* Builds URL using base URL and specified path
*
* @param path relative path
* @return Base URL + path
* @throws MalformedURLException
*/
private URL buildPath(BlobPath path) throws MalformedURLException {
String[] paths = path.toArray();
if (paths.length == 0) {
return path();
}
URL blobPath = new URL(this.path, paths[0] + "/");
if (paths.length > 1) {
for (int i = 1; i < paths.length; i++) {
blobPath = new URL(blobPath, paths[i] + "/");
}
}
return blobPath;
}
/**
* Constructs new shared file system repository
*
* @param name repository name
* @param repositorySettings repository settings
* @param indexShardRepository index shard repository
* @param s3Service S3 service
* @throws IOException
*/
@Inject
public S3Repository(
RepositoryName name,
RepositorySettings repositorySettings,
IndexShardRepository indexShardRepository,
AwsS3Service s3Service)
throws IOException {
super(name.getName(), repositorySettings, indexShardRepository);
String bucket = repositorySettings.settings().get("bucket", componentSettings.get("bucket"));
if (bucket == null) {
throw new RepositoryException(name.name(), "No bucket defined for s3 gateway");
}
String region = repositorySettings.settings().get("region", componentSettings.get("region"));
if (region == null) {
// Bucket setting is not set - use global region setting
String regionSetting =
repositorySettings.settings().get("cloud.aws.region", settings.get("cloud.aws.region"));
if (regionSetting != null) {
regionSetting = regionSetting.toLowerCase(Locale.ENGLISH);
if ("us-east".equals(regionSetting)) {
// Default bucket - setting region to null
region = null;
} else if ("us-east-1".equals(regionSetting)) {
region = null;
} else if ("us-west".equals(regionSetting)) {
region = "us-west-1";
} else if ("us-west-1".equals(regionSetting)) {
region = "us-west-1";
} else if ("us-west-2".equals(regionSetting)) {
region = "us-west-2";
} else if ("ap-southeast".equals(regionSetting)) {
region = "ap-southeast-1";
} else if ("ap-southeast-1".equals(regionSetting)) {
region = "ap-southeast-1";
} else if ("ap-southeast-2".equals(regionSetting)) {
region = "ap-southeast-2";
} else if ("ap-northeast".equals(regionSetting)) {
region = "ap-northeast-1";
} else if ("ap-northeast-1".equals(regionSetting)) {
region = "ap-northeast-1";
} else if ("eu-west".equals(regionSetting)) {
region = "EU";
} else if ("eu-west-1".equals(regionSetting)) {
region = "EU";
} else if ("sa-east".equals(regionSetting)) {
region = "sa-east-1";
} else if ("sa-east-1".equals(regionSetting)) {
region = "sa-east-1";
}
}
}
int concurrentStreams =
repositorySettings
.settings()
.getAsInt("concurrent_streams", componentSettings.getAsInt("concurrent_streams", 5));
ExecutorService concurrentStreamPool =
EsExecutors.newScaling(
1,
concurrentStreams,
5,
TimeUnit.SECONDS,
EsExecutors.daemonThreadFactory(settings, "[s3_stream]"));
boolean serverSideEncryption =
repositorySettings
.settings()
.getAsBoolean(
"server_side_encryption",
componentSettings.getAsBoolean("server_side_encryption", false));
logger.debug(
"using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}], server_side_encryption [{}]",
bucket,
region,
chunkSize,
concurrentStreams,
serverSideEncryption);
blobStore =
new S3BlobStore(
settings,
s3Service.client(
region,
repositorySettings.settings().get("access_key"),
repositorySettings.settings().get("secret_key")),
bucket,
region,
concurrentStreamPool,
serverSideEncryption);
this.chunkSize =
repositorySettings
.settings()
.getAsBytesSize(
"chunk_size",
componentSettings.getAsBytesSize(
"chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB)));
this.compress =
repositorySettings
.settings()
.getAsBoolean("compress", componentSettings.getAsBoolean("compress", false));
String basePath = repositorySettings.settings().get("base_path", null);
if (Strings.hasLength(basePath)) {
BlobPath path = new BlobPath();
for (String elem : Strings.splitStringToArray(basePath, '/')) {
path = path.add(elem);
}
this.basePath = path;
} else {
this.basePath = BlobPath.cleanPath();
}
}
public BlobPath shardPath(int shardId) {
return indexPath.add(Integer.toString(shardId));
}
@Override
public void reset() throws Exception {
blobStore.delete(BlobPath.cleanPath());
}