@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
List<NoteInfo> infos = new LinkedList<>();
NoteInfo info;
try {
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().endsWith("note.json")) {
info = getNoteInfo(objectSummary.getKey());
if (info != null) {
infos.add(info);
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to list objects in S3: " + ace, ace);
}
return infos;
}
/**
* Reads next key from S3
*
* @return {@link S3ObjectSummary} of the key or <code>null</code> if there are no more keys to
* read
*/
public S3ObjectSummary getNextKey() {
if (objectListing == null) {
LOG.debug("Listing objects");
objectListing = s3Client.listObjects(listObjectsRequest);
}
// we have more elements in list
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else if (objectListing.isTruncated()) {
LOG.debug("Current batch reached its end. Fetching next page.");
// when we're at the end, check if there's more data to read
listObjectsRequest.setMarker(objectListing.getNextMarker());
LOG.debug("New marker is set to {}", listObjectsRequest.getMarker());
objectListing = s3Client.listObjects(listObjectsRequest);
currentPosition = 0;
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else {
LOG.debug("No more objects to read");
currentObject = null;
}
} else {
currentObject = null;
}
return currentObject;
}
@Override
public List<BackupMetaData> getAvailableBackups(
Exhibitor exhibitor, Map<String, String> configValues) throws Exception {
ListObjectsRequest request = new ListObjectsRequest();
request.setBucketName(configValues.get(CONFIG_BUCKET.getKey()));
ObjectListing listing = s3Client.listObjects(request);
return Lists.transform(
listing.getObjectSummaries(),
new Function<S3ObjectSummary, BackupMetaData>() {
@Override
public BackupMetaData apply(S3ObjectSummary summary) {
return fromKey(summary.getKey());
}
});
}
public static void main(final String[] args) {
DesktopSetup.deploy();
final RelativePath target = JUtils.newRelativePath();
final AmazonS3 s3 = new AmazonS3Client(new ProfileCredentialsProvider());
final ListObjectsRequest request = new ListObjectsRequest().withBucketName("jfix.by");
final String prefix = "/";
request.withPrefix("wp-content");
request.setDelimiter("");
final ObjectListing objectListing = s3.listObjects(request);
final List<String> prefixes = Collections.newList(objectListing.getCommonPrefixes());
prefixes.print("prefixes");
final List<S3ObjectSummary> summs = Collections.newList(objectListing.getObjectSummaries());
}
private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
ListObjectsRequest listRequest = new ListObjectsRequest();
listRequest.setBucketName(s3BucketName);
// listRequest.setGeneralProgressListener(this);
listRequest.setMaxKeys(1000);
String nextMarker = null;
ObjectListing objectListing = null;
while (true) {
objectListing = s3Client.listObjects(listRequest);
List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
for (S3ObjectSummary objSummary : objectSummaries) {
String key = objSummary.getKey();
TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
// is it a "dir/" ?
if (key.lastIndexOf("/") == (key.length() - 1)) {
tocInfo.isDirectory = true;
} else {
tocInfo.isDirectory = false;
}
toc.add(tocInfo);
tocQueue.add(tocInfo);
tocInfosGenerated++; // increment for logging
}
// for pagination
nextMarker = objectListing.getNextMarker();
if (nextMarker == null) {
break;
} else {
listRequest.setMarker(nextMarker);
logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
}
}
}
@Override
public boolean matches(final Object arg) {
boolean result = false;
if (arg instanceof ListObjectsRequest) {
final ListObjectsRequest obj = (ListObjectsRequest) arg;
result = true;
if (!this.bucket.equals(obj.getBucketName())) {
result = false;
}
if (!this.prefix.equals(obj.getPrefix())) {
result = false;
}
if (this.marker == null && obj.getMarker() != null) {
result = false;
}
if (this.marker != null && !this.marker.equals(obj.getMarker())) {
result = false;
}
}
return result;
}
@Override
protected int poll() throws Exception {
// must reset for each poll
shutdownRunningTask = null;
pendingExchanges = 0;
String fileName = getConfiguration().getFileName();
String bucketName = getConfiguration().getBucketName();
Queue<Exchange> exchanges = null;
if (fileName != null) {
LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);
S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
exchanges = createExchanges(s3Object);
} else {
LOG.trace("Queueing objects in bucket [{}]...", bucketName);
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName(bucketName);
listObjectsRequest.setPrefix(getConfiguration().getPrefix());
listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
if (LOG.isTraceEnabled()) {
LOG.trace(
"Found {} objects in bucket [{}]...",
listObjects.getObjectSummaries().size(),
bucketName);
}
exchanges = createExchanges(listObjects.getObjectSummaries());
}
return processBatch(CastUtils.cast(exchanges));
}