// 列出某个bucket的所有objects
public static void listObjectInBucket(Bucket bucket) {
List<Bucket> buckets = conn.listBuckets();
ObjectListing objects = conn.listObjects(bucket.getName());
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
System.out.println(
objectSummary.getKey()
+ "\t"
+ objectSummary.getSize()
+ "\t"
+ StringUtils.fromDate(objectSummary.getLastModified()));
}
objects = conn.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
// 列出有prefix前缀的所有objects
public static void listObjectInBucketWithPrefix(Bucket bucket, String prefix) {
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucket.getName()).withPrefix(prefix);
ObjectListing objects = conn.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
System.out.println(
objectSummary.getKey()
+ "\t"
+ objectSummary.getSize()
+ "\t"
+ StringUtils.fromDate(objectSummary.getLastModified()));
}
objects = conn.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
String key = user + "/" + "notebook" + "/" + noteId;
final ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucketName).withPrefix(key);
try {
ObjectListing objects = s3client.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
s3client.deleteObject(bucketName, objectSummary.getKey());
}
objects = s3client.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to remove note in S3: " + ace, ace);
}
}
@Override
protected Response serve() {
JsonObject json = new JsonObject();
JsonArray succ = new JsonArray();
JsonArray fail = new JsonArray();
String bucket = _bucket.value();
AmazonS3 s3 = PersistS3.getClient();
ObjectListing currentList = s3.listObjects(bucket);
processListing(currentList, succ, fail);
while (currentList.isTruncated()) {
currentList = s3.listNextBatchOfObjects(currentList);
processListing(currentList, succ, fail);
}
json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size()));
json.add(SUCCEEDED, succ);
json.add(NUM_FAILED, new JsonPrimitive(fail.size()));
json.add(FAILED, fail);
DKV.write_barrier();
Response r = Response.done(json);
r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder());
return r;
}
public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: localPath bucketname <bucketPrefix>");
System.exit(1);
}
String localPath = args[0];
String bucketName = args[1];
String bucketPrefix = "";
// add on extra slash
if (!localPath.endsWith("/")) localPath += "/";
if (args.length == 3) bucketPrefix = args[2];
// check local dir, make if it does not exist
File localDir = new File(localPath);
if (!localDir.exists()) localDir.mkdirs();
if (!localDir.isDirectory()) {
System.out.println("Local Dir is not a dir: " + localPath);
System.exit(1);
}
long totalBytes = 0;
long start = System.currentTimeMillis();
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
ObjectListing listObjects = s3.listObjects(bucketName, bucketPrefix);
do {
for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) {
S3Object object = s3.getObject(bucketName, objectSummary.getKey());
S3ObjectInputStream inputStream = object.getObjectContent();
if ("gzip".equals(object.getObjectMetadata().getContentEncoding())) {
InputStream in = null;
try {
totalBytes += object.getObjectMetadata().getContentLength();
in = new GZIPInputStream(inputStream);
// write this sucker out
String path = localPath + object.getKey();
// have to take the gz off since this is not downloading compressed!
if (path.endsWith(".gz")) path = path.substring(0, path.length() - 3);
System.out.print("Writing file: " + path);
File check = new File(path);
File parentFile = check.getParentFile();
if (!parentFile.exists()) parentFile.mkdirs();
FileOutputStream out = new FileOutputStream(path);
IOUtils.copy(in, out);
System.out.println(" written.");
} catch (IOException e) {
System.out.println("crap");
e.printStackTrace();
throw new IllegalStateException("files are too hard", e);
} finally {
IOUtils.closeQuietly(in);
}
} else {
System.out.println(
"unhandled content encoding: " + object.getObjectMetadata().getContentEncoding());
}
}
listObjects = s3.listNextBatchOfObjects(listObjects);
} while (listObjects.isTruncated());
long now = System.currentTimeMillis();
System.out.println(
(totalBytes / 1000.0 / 1000.0)
+ " mb downloaded in "
+ ((now - start) / 1000)
+ " seconds.");
}
/** Deletes content of the repository files in the bucket */
public void cleanRepositoryFiles(String basePath) {
Settings settings = internalCluster().getInstance(Settings.class);
Settings[] buckets = {
settings.getByPrefix("repositories.s3."),
settings.getByPrefix("repositories.s3.private-bucket."),
settings.getByPrefix("repositories.s3.remote-bucket."),
settings.getByPrefix("repositories.s3.external-bucket.")
};
for (Settings bucket : buckets) {
String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint"));
String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol"));
String region = bucket.get("region", settings.get("repositories.s3.region"));
String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key"));
String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key"));
String bucketName = bucket.get("bucket");
// We check that settings has been set in elasticsearch.yml integration test file
// as described in README
assertThat(
"Your settings in elasticsearch.yml are incorrects. Check README file.",
bucketName,
notNullValue());
AmazonS3 client =
internalCluster()
.getInstance(AwsS3Service.class)
.client(endpoint, protocol, region, accessKey, secretKey);
try {
ObjectListing prevListing = null;
// From
// http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
// we can do at most 1K objects per delete
// We don't know the bucket name until first object listing
DeleteObjectsRequest multiObjectDeleteRequest = null;
ArrayList<DeleteObjectsRequest.KeyVersion> keys =
new ArrayList<DeleteObjectsRequest.KeyVersion>();
while (true) {
ObjectListing list;
if (prevListing != null) {
list = client.listNextBatchOfObjects(prevListing);
} else {
list = client.listObjects(bucketName, basePath);
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
}
for (S3ObjectSummary summary : list.getObjectSummaries()) {
keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
// Every 500 objects batch the delete request
if (keys.size() > 500) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
keys.clear();
}
}
if (list.isTruncated()) {
prevListing = list;
} else {
break;
}
}
if (!keys.isEmpty()) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
}
} catch (Throwable ex) {
logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region);
}
}
}
