@Override
public void recursiveDelete(String keyPrefix, FilenameFilter except) throws IOException {
if (keyPrefix.charAt(keyPrefix.length() - 1) == '/') {
// Need to delete the dir too, and to list it, can't specify its trailing slash
keyPrefix = keyPrefix.substring(0, keyPrefix.length() - 1);
}
boolean truncated = true;
String marker = null;
String bucket = Namespaces.get().getBucket();
while (truncated) {
ListObjectsRequest listRequest =
new ListObjectsRequest(bucket, keyPrefix, marker, null, null);
ObjectListing listing;
try {
listing = s3Client.listObjects(listRequest);
Collection<S3ObjectSummary> summaries = listing.getObjectSummaries();
if (summaries.isEmpty()) {
return;
}
List<DeleteObjectsRequest.KeyVersion> keysToDelete =
Lists.newArrayListWithCapacity(summaries.size());
for (S3ObjectSummary summary : summaries) {
String key = summary.getKey();
if (except == null || !except.accept(null, key)) {
keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key));
}
}
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket);
deleteObjectsRequest.setKeys(keysToDelete);
s3Client.deleteObjects(deleteObjectsRequest);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
truncated = listing.isTruncated();
marker = listing.getNextMarker();
}
}
/** Deletes content of the repository files in the bucket */
public void cleanRepositoryFiles(String basePath) {
Settings settings = internalCluster().getInstance(Settings.class);
Settings[] buckets = {
settings.getByPrefix("repositories.s3."),
settings.getByPrefix("repositories.s3.private-bucket."),
settings.getByPrefix("repositories.s3.remote-bucket."),
settings.getByPrefix("repositories.s3.external-bucket.")
};
for (Settings bucket : buckets) {
String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint"));
String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol"));
String region = bucket.get("region", settings.get("repositories.s3.region"));
String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key"));
String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key"));
String bucketName = bucket.get("bucket");
// We check that settings has been set in elasticsearch.yml integration test file
// as described in README
assertThat(
"Your settings in elasticsearch.yml are incorrects. Check README file.",
bucketName,
notNullValue());
AmazonS3 client =
internalCluster()
.getInstance(AwsS3Service.class)
.client(endpoint, protocol, region, accessKey, secretKey);
try {
ObjectListing prevListing = null;
// From
// http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
// we can do at most 1K objects per delete
// We don't know the bucket name until first object listing
DeleteObjectsRequest multiObjectDeleteRequest = null;
ArrayList<DeleteObjectsRequest.KeyVersion> keys =
new ArrayList<DeleteObjectsRequest.KeyVersion>();
while (true) {
ObjectListing list;
if (prevListing != null) {
list = client.listNextBatchOfObjects(prevListing);
} else {
list = client.listObjects(bucketName, basePath);
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
}
for (S3ObjectSummary summary : list.getObjectSummaries()) {
keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
// Every 500 objects batch the delete request
if (keys.size() > 500) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
keys.clear();
}
}
if (list.isTruncated()) {
prevListing = list;
} else {
break;
}
}
if (!keys.isEmpty()) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
}
} catch (Throwable ex) {
logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region);
}
}
}
