@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
List<NoteInfo> infos = new LinkedList<>();
NoteInfo info;
try {
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().endsWith("note.json")) {
info = getNoteInfo(objectSummary.getKey());
if (info != null) {
infos.add(info);
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to list objects in S3: " + ace, ace);
}
return infos;
}
@Override
protected List<String> listDirectory(String directory) throws ResourceDoesNotExistException {
List<String> directoryContents = new ArrayList<String>();
try {
String prefix = getKey(directory);
Pattern pattern = Pattern.compile(String.format(RESOURCE_FORMAT, prefix));
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest() //
.withBucketName(this.bucketName) //
.withPrefix(prefix) //
.withDelimiter("/");
ObjectListing objectListing;
objectListing = this.amazonS3.listObjects(listObjectsRequest);
directoryContents.addAll(getResourceNames(objectListing, pattern));
while (objectListing.isTruncated()) {
objectListing = this.amazonS3.listObjects(listObjectsRequest);
directoryContents.addAll(getResourceNames(objectListing, pattern));
}
return directoryContents;
} catch (AmazonServiceException e) {
throw new ResourceDoesNotExistException(String.format("'%s' does not exist", directory), e);
}
}
/**
* Reads next key from S3
*
* @return {@link S3ObjectSummary} of the key or <code>null</code> if there are no more keys to
* read
*/
public S3ObjectSummary getNextKey() {
if (objectListing == null) {
LOG.debug("Listing objects");
objectListing = s3Client.listObjects(listObjectsRequest);
}
// we have more elements in list
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else if (objectListing.isTruncated()) {
LOG.debug("Current batch reached its end. Fetching next page.");
// when we're at the end, check if there's more data to read
listObjectsRequest.setMarker(objectListing.getNextMarker());
LOG.debug("New marker is set to {}", listObjectsRequest.getMarker());
objectListing = s3Client.listObjects(listObjectsRequest);
currentPosition = 0;
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else {
LOG.debug("No more objects to read");
currentObject = null;
}
} else {
currentObject = null;
}
return currentObject;
}
// 列出某个bucket的所有objects
public static void listObjectInBucket(Bucket bucket) {
List<Bucket> buckets = conn.listBuckets();
ObjectListing objects = conn.listObjects(bucket.getName());
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
System.out.println(
objectSummary.getKey()
+ "\t"
+ objectSummary.getSize()
+ "\t"
+ StringUtils.fromDate(objectSummary.getLastModified()));
}
objects = conn.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
// 列出有prefix前缀的所有objects
public static void listObjectInBucketWithPrefix(Bucket bucket, String prefix) {
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucket.getName()).withPrefix(prefix);
ObjectListing objects = conn.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
System.out.println(
objectSummary.getKey()
+ "\t"
+ objectSummary.getSize()
+ "\t"
+ StringUtils.fromDate(objectSummary.getLastModified()));
}
objects = conn.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
String key = user + "/" + "notebook" + "/" + noteId;
final ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucketName).withPrefix(key);
try {
ObjectListing objects = s3client.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
s3client.deleteObject(bucketName, objectSummary.getKey());
}
objects = s3client.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to remove note in S3: " + ace, ace);
}
}
@Override
public void recursiveDelete(String keyPrefix, FilenameFilter except) throws IOException {
if (keyPrefix.charAt(keyPrefix.length() - 1) == '/') {
// Need to delete the dir too, and to list it, can't specify its trailing slash
keyPrefix = keyPrefix.substring(0, keyPrefix.length() - 1);
}
boolean truncated = true;
String marker = null;
String bucket = Namespaces.get().getBucket();
while (truncated) {
ListObjectsRequest listRequest =
new ListObjectsRequest(bucket, keyPrefix, marker, null, null);
ObjectListing listing;
try {
listing = s3Client.listObjects(listRequest);
Collection<S3ObjectSummary> summaries = listing.getObjectSummaries();
if (summaries.isEmpty()) {
return;
}
List<DeleteObjectsRequest.KeyVersion> keysToDelete =
Lists.newArrayListWithCapacity(summaries.size());
for (S3ObjectSummary summary : summaries) {
String key = summary.getKey();
if (except == null || !except.accept(null, key)) {
keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key));
}
}
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket);
deleteObjectsRequest.setKeys(keysToDelete);
s3Client.deleteObjects(deleteObjectsRequest);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
truncated = listing.isTruncated();
marker = listing.getNextMarker();
}
}
@Override
public InputStream next() {
if (currObject < currObjects.size()) {
InputStream ret = s3.objectForKey(bucket, currObjects.get(currObject).getKey());
currObject++;
return ret;
} else if (currList.isTruncated()) {
currList = s3.nextList(currList);
currObjects = currList.getObjectSummaries();
currObject = 0;
InputStream ret = s3.objectForKey(bucket, currObjects.get(currObject).getKey());
currObject++;
return ret;
}
return null;
}
public static List<String> listBucket(String bucketName, String prefix) {
List<String> ret = new ArrayList<String>();
ObjectListing listing =
getInstance()
.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix));
for (S3ObjectSummary s : listing.getObjectSummaries()) {
StringBuilder sb = new StringBuilder();
sb.append("s3://").append(bucketName).append("/").append(s.getKey());
ret.add(sb.toString());
}
while (listing.isTruncated()) {
listing = getInstance().listNextBatchOfObjects(listing);
for (S3ObjectSummary s : listing.getObjectSummaries()) {
StringBuilder sb = new StringBuilder();
sb.append("s3://").append(bucketName).append("/").append(s.getKey());
ret.add(sb.toString());
}
}
return ret;
}
@Override
protected Response serve() {
JsonObject json = new JsonObject();
JsonArray succ = new JsonArray();
JsonArray fail = new JsonArray();
String bucket = _bucket.value();
AmazonS3 s3 = PersistS3.getClient();
ObjectListing currentList = s3.listObjects(bucket);
processListing(currentList, succ, fail);
while (currentList.isTruncated()) {
currentList = s3.listNextBatchOfObjects(currentList);
processListing(currentList, succ, fail);
}
json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size()));
json.add(SUCCEEDED, succ);
json.add(NUM_FAILED, new JsonPrimitive(fail.size()));
json.add(FAILED, fail);
DKV.write_barrier();
Response r = Response.done(json);
r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder());
return r;
}
@Override
public List<String> list(String prefix, boolean files) throws IOException {
boolean truncated = true;
String marker = null;
List<String> result = Lists.newArrayList();
while (truncated) {
if (marker != null) {
log.info("Next page after {}", marker);
}
ListObjectsRequest listRequest =
new ListObjectsRequest(Namespaces.get().getBucket(), prefix, marker, "/", null);
ObjectListing listing;
try {
listing = s3Client.listObjects(listRequest);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
truncated = listing.isTruncated();
marker = listing.getNextMarker();
if (!truncated) {
if (files) {
for (S3ObjectSummary summary : listing.getObjectSummaries()) {
String key = summary.getKey();
if (!key.endsWith("_SUCCESS")) {
result.add(key);
}
}
} else {
for (String key : listing.getCommonPrefixes()) {
if (!key.endsWith("_SUCCESS")) {
result.add(key);
}
}
}
}
}
Collections.sort(result);
return result;
}
public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: localPath bucketname <bucketPrefix>");
System.exit(1);
}
String localPath = args[0];
String bucketName = args[1];
String bucketPrefix = "";
// add on extra slash
if (!localPath.endsWith("/")) localPath += "/";
if (args.length == 3) bucketPrefix = args[2];
// check local dir, make if it does not exist
File localDir = new File(localPath);
if (!localDir.exists()) localDir.mkdirs();
if (!localDir.isDirectory()) {
System.out.println("Local Dir is not a dir: " + localPath);
System.exit(1);
}
long totalBytes = 0;
long start = System.currentTimeMillis();
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
ObjectListing listObjects = s3.listObjects(bucketName, bucketPrefix);
do {
for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) {
S3Object object = s3.getObject(bucketName, objectSummary.getKey());
S3ObjectInputStream inputStream = object.getObjectContent();
if ("gzip".equals(object.getObjectMetadata().getContentEncoding())) {
InputStream in = null;
try {
totalBytes += object.getObjectMetadata().getContentLength();
in = new GZIPInputStream(inputStream);
// write this sucker out
String path = localPath + object.getKey();
// have to take the gz off since this is not downloading compressed!
if (path.endsWith(".gz")) path = path.substring(0, path.length() - 3);
System.out.print("Writing file: " + path);
File check = new File(path);
File parentFile = check.getParentFile();
if (!parentFile.exists()) parentFile.mkdirs();
FileOutputStream out = new FileOutputStream(path);
IOUtils.copy(in, out);
System.out.println(" written.");
} catch (IOException e) {
System.out.println("crap");
e.printStackTrace();
throw new IllegalStateException("files are too hard", e);
} finally {
IOUtils.closeQuietly(in);
}
} else {
System.out.println(
"unhandled content encoding: " + object.getObjectMetadata().getContentEncoding());
}
}
listObjects = s3.listNextBatchOfObjects(listObjects);
} while (listObjects.isTruncated());
long now = System.currentTimeMillis();
System.out.println(
(totalBytes / 1000.0 / 1000.0)
+ " mb downloaded in "
+ ((now - start) / 1000)
+ " seconds.");
}
@Override
public boolean hasNext() {
return !currList.isTruncated() && currObject < currObjects.size();
}
/** Deletes content of the repository files in the bucket */
public void cleanRepositoryFiles(String basePath) {
Settings settings = internalCluster().getInstance(Settings.class);
Settings[] buckets = {
settings.getByPrefix("repositories.s3."),
settings.getByPrefix("repositories.s3.private-bucket."),
settings.getByPrefix("repositories.s3.remote-bucket."),
settings.getByPrefix("repositories.s3.external-bucket.")
};
for (Settings bucket : buckets) {
String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint"));
String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol"));
String region = bucket.get("region", settings.get("repositories.s3.region"));
String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key"));
String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key"));
String bucketName = bucket.get("bucket");
// We check that settings has been set in elasticsearch.yml integration test file
// as described in README
assertThat(
"Your settings in elasticsearch.yml are incorrects. Check README file.",
bucketName,
notNullValue());
AmazonS3 client =
internalCluster()
.getInstance(AwsS3Service.class)
.client(endpoint, protocol, region, accessKey, secretKey);
try {
ObjectListing prevListing = null;
// From
// http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
// we can do at most 1K objects per delete
// We don't know the bucket name until first object listing
DeleteObjectsRequest multiObjectDeleteRequest = null;
ArrayList<DeleteObjectsRequest.KeyVersion> keys =
new ArrayList<DeleteObjectsRequest.KeyVersion>();
while (true) {
ObjectListing list;
if (prevListing != null) {
list = client.listNextBatchOfObjects(prevListing);
} else {
list = client.listObjects(bucketName, basePath);
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
}
for (S3ObjectSummary summary : list.getObjectSummaries()) {
keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
// Every 500 objects batch the delete request
if (keys.size() > 500) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
keys.clear();
}
}
if (list.isTruncated()) {
prevListing = list;
} else {
break;
}
}
if (!keys.isEmpty()) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
}
} catch (Throwable ex) {
logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region);
}
}
}