@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
List<NoteInfo> infos = new LinkedList<>();
NoteInfo info;
try {
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().endsWith("note.json")) {
info = getNoteInfo(objectSummary.getKey());
if (info != null) {
infos.add(info);
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to list objects in S3: " + ace, ace);
}
return infos;
}
/**
* Reads next key from S3
*
* @return {@link S3ObjectSummary} of the key or <code>null</code> if there are no more keys to
* read
*/
public S3ObjectSummary getNextKey() {
if (objectListing == null) {
LOG.debug("Listing objects");
objectListing = s3Client.listObjects(listObjectsRequest);
}
// we have more elements in list
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else if (objectListing.isTruncated()) {
LOG.debug("Current batch reached its end. Fetching next page.");
// when we're at the end, check if there's more data to read
listObjectsRequest.setMarker(objectListing.getNextMarker());
LOG.debug("New marker is set to {}", listObjectsRequest.getMarker());
objectListing = s3Client.listObjects(listObjectsRequest);
currentPosition = 0;
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else {
LOG.debug("No more objects to read");
currentObject = null;
}
} else {
currentObject = null;
}
return currentObject;
}
private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
ListObjectsRequest listRequest = new ListObjectsRequest();
listRequest.setBucketName(s3BucketName);
// listRequest.setGeneralProgressListener(this);
listRequest.setMaxKeys(1000);
String nextMarker = null;
ObjectListing objectListing = null;
while (true) {
objectListing = s3Client.listObjects(listRequest);
List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
for (S3ObjectSummary objSummary : objectSummaries) {
String key = objSummary.getKey();
TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
// is it a "dir/" ?
if (key.lastIndexOf("/") == (key.length() - 1)) {
tocInfo.isDirectory = true;
} else {
tocInfo.isDirectory = false;
}
toc.add(tocInfo);
tocQueue.add(tocInfo);
tocInfosGenerated++; // increment for logging
}
// for pagination
nextMarker = objectListing.getNextMarker();
if (nextMarker == null) {
break;
} else {
listRequest.setMarker(nextMarker);
logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
}
}
}