@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
List<NoteInfo> infos = new LinkedList<>();
NoteInfo info;
try {
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().endsWith("note.json")) {
info = getNoteInfo(objectSummary.getKey());
if (info != null) {
infos.add(info);
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to list objects in S3: " + ace, ace);
}
return infos;
}
/**
* Reads next key from S3
*
* @return {@link S3ObjectSummary} of the key or <code>null</code> if there are no more keys to
* read
*/
public S3ObjectSummary getNextKey() {
if (objectListing == null) {
LOG.debug("Listing objects");
objectListing = s3Client.listObjects(listObjectsRequest);
}
// we have more elements in list
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else if (objectListing.isTruncated()) {
LOG.debug("Current batch reached its end. Fetching next page.");
// when we're at the end, check if there's more data to read
listObjectsRequest.setMarker(objectListing.getNextMarker());
LOG.debug("New marker is set to {}", listObjectsRequest.getMarker());
objectListing = s3Client.listObjects(listObjectsRequest);
currentPosition = 0;
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else {
LOG.debug("No more objects to read");
currentObject = null;
}
} else {
currentObject = null;
}
return currentObject;
}
@Override
public void recursiveDelete(String keyPrefix, FilenameFilter except) throws IOException {
if (keyPrefix.charAt(keyPrefix.length() - 1) == '/') {
// Need to delete the dir too, and to list it, can't specify its trailing slash
keyPrefix = keyPrefix.substring(0, keyPrefix.length() - 1);
}
boolean truncated = true;
String marker = null;
String bucket = Namespaces.get().getBucket();
while (truncated) {
ListObjectsRequest listRequest =
new ListObjectsRequest(bucket, keyPrefix, marker, null, null);
ObjectListing listing;
try {
listing = s3Client.listObjects(listRequest);
Collection<S3ObjectSummary> summaries = listing.getObjectSummaries();
if (summaries.isEmpty()) {
return;
}
List<DeleteObjectsRequest.KeyVersion> keysToDelete =
Lists.newArrayListWithCapacity(summaries.size());
for (S3ObjectSummary summary : summaries) {
String key = summary.getKey();
if (except == null || !except.accept(null, key)) {
keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key));
}
}
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket);
deleteObjectsRequest.setKeys(keysToDelete);
s3Client.deleteObjects(deleteObjectsRequest);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
truncated = listing.isTruncated();
marker = listing.getNextMarker();
}
}
private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
ListObjectsRequest listRequest = new ListObjectsRequest();
listRequest.setBucketName(s3BucketName);
// listRequest.setGeneralProgressListener(this);
listRequest.setMaxKeys(1000);
String nextMarker = null;
ObjectListing objectListing = null;
while (true) {
objectListing = s3Client.listObjects(listRequest);
List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
for (S3ObjectSummary objSummary : objectSummaries) {
String key = objSummary.getKey();
TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
// is it a "dir/" ?
if (key.lastIndexOf("/") == (key.length() - 1)) {
tocInfo.isDirectory = true;
} else {
tocInfo.isDirectory = false;
}
toc.add(tocInfo);
tocQueue.add(tocInfo);
tocInfosGenerated++; // increment for logging
}
// for pagination
nextMarker = objectListing.getNextMarker();
if (nextMarker == null) {
break;
} else {
listRequest.setMarker(nextMarker);
logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
}
}
}
@Override
public List<String> list(String prefix, boolean files) throws IOException {
boolean truncated = true;
String marker = null;
List<String> result = Lists.newArrayList();
while (truncated) {
if (marker != null) {
log.info("Next page after {}", marker);
}
ListObjectsRequest listRequest =
new ListObjectsRequest(Namespaces.get().getBucket(), prefix, marker, "/", null);
ObjectListing listing;
try {
listing = s3Client.listObjects(listRequest);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
truncated = listing.isTruncated();
marker = listing.getNextMarker();
if (!truncated) {
if (files) {
for (S3ObjectSummary summary : listing.getObjectSummaries()) {
String key = summary.getKey();
if (!key.endsWith("_SUCCESS")) {
result.add(key);
}
}
} else {
for (String key : listing.getCommonPrefixes()) {
if (!key.endsWith("_SUCCESS")) {
result.add(key);
}
}
}
}
}
Collections.sort(result);
return result;
}