@Override public void recursiveDelete(String keyPrefix, FilenameFilter except) throws IOException { if (keyPrefix.charAt(keyPrefix.length() - 1) == '/') { // Need to delete the dir too, and to list it, can't specify its trailing slash keyPrefix = keyPrefix.substring(0, keyPrefix.length() - 1); } boolean truncated = true; String marker = null; String bucket = Namespaces.get().getBucket(); while (truncated) { ListObjectsRequest listRequest = new ListObjectsRequest(bucket, keyPrefix, marker, null, null); ObjectListing listing; try { listing = s3Client.listObjects(listRequest); Collection<S3ObjectSummary> summaries = listing.getObjectSummaries(); if (summaries.isEmpty()) { return; } List<DeleteObjectsRequest.KeyVersion> keysToDelete = Lists.newArrayListWithCapacity(summaries.size()); for (S3ObjectSummary summary : summaries) { String key = summary.getKey(); if (except == null || !except.accept(null, key)) { keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key)); } } DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket); deleteObjectsRequest.setKeys(keysToDelete); s3Client.deleteObjects(deleteObjectsRequest); } catch (AmazonClientException ace) { throw new IOException(ace); } truncated = listing.isTruncated(); marker = listing.getNextMarker(); } }
/** Deletes content of the repository files in the bucket */ public void cleanRepositoryFiles(String basePath) { Settings settings = internalCluster().getInstance(Settings.class); Settings[] buckets = { settings.getByPrefix("repositories.s3."), settings.getByPrefix("repositories.s3.private-bucket."), settings.getByPrefix("repositories.s3.remote-bucket."), settings.getByPrefix("repositories.s3.external-bucket.") }; for (Settings bucket : buckets) { String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint")); String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol")); String region = bucket.get("region", settings.get("repositories.s3.region")); String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key")); String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key")); String bucketName = bucket.get("bucket"); // We check that settings has been set in elasticsearch.yml integration test file // as described in README assertThat( "Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue()); AmazonS3 client = internalCluster() .getInstance(AwsS3Service.class) .client(endpoint, protocol, region, accessKey, secretKey); try { ObjectListing prevListing = null; // From // http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html // we can do at most 1K objects per delete // We don't know the bucket name until first object listing DeleteObjectsRequest multiObjectDeleteRequest = null; ArrayList<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>(); while (true) { ObjectListing list; if (prevListing != null) { list = client.listNextBatchOfObjects(prevListing); } else { list = client.listObjects(bucketName, basePath); multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); } for (S3ObjectSummary summary : list.getObjectSummaries()) { keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); // Every 500 objects batch the delete request if (keys.size() > 500) { multiObjectDeleteRequest.setKeys(keys); client.deleteObjects(multiObjectDeleteRequest); multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); keys.clear(); } } if (list.isTruncated()) { prevListing = list; } else { break; } } if (!keys.isEmpty()) { multiObjectDeleteRequest.setKeys(keys); client.deleteObjects(multiObjectDeleteRequest); } } catch (Throwable ex) { logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region); } } }