示例#1
0
 @Override
 public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
   List<NoteInfo> infos = new LinkedList<>();
   NoteInfo info;
   try {
     ListObjectsRequest listObjectsRequest =
         new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
     ObjectListing objectListing;
     do {
       objectListing = s3client.listObjects(listObjectsRequest);
       for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
         if (objectSummary.getKey().endsWith("note.json")) {
           info = getNoteInfo(objectSummary.getKey());
           if (info != null) {
             infos.add(info);
           }
         }
       }
       listObjectsRequest.setMarker(objectListing.getNextMarker());
     } while (objectListing.isTruncated());
   } catch (AmazonClientException ace) {
     throw new IOException("Unable to list objects in S3: " + ace, ace);
   }
   return infos;
 }
示例#2
0
  /**
   * Reads next key from S3
   *
   * @return {@link S3ObjectSummary} of the key or <code>null</code> if there are no more keys to
   *     read
   */
  public S3ObjectSummary getNextKey() {
    if (objectListing == null) {
      LOG.debug("Listing objects");
      objectListing = s3Client.listObjects(listObjectsRequest);
    }

    // we have more elements in list
    if (currentPosition < objectListing.getObjectSummaries().size()) {
      currentObject = objectListing.getObjectSummaries().get(currentPosition++);
    } else if (objectListing.isTruncated()) {
      LOG.debug("Current batch reached its end. Fetching next page.");
      // when we're at the end, check if there's more data to read
      listObjectsRequest.setMarker(objectListing.getNextMarker());
      LOG.debug("New marker is set to {}", listObjectsRequest.getMarker());

      objectListing = s3Client.listObjects(listObjectsRequest);
      currentPosition = 0;

      if (currentPosition < objectListing.getObjectSummaries().size()) {
        currentObject = objectListing.getObjectSummaries().get(currentPosition++);
      } else {
        LOG.debug("No more objects to read");
        currentObject = null;
      }
    } else {
      currentObject = null;
    }

    return currentObject;
  }
  private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {

    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(s3BucketName);
    // listRequest.setGeneralProgressListener(this);
    listRequest.setMaxKeys(1000);

    String nextMarker = null;
    ObjectListing objectListing = null;

    while (true) {

      objectListing = s3Client.listObjects(listRequest);

      List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();

      for (S3ObjectSummary objSummary : objectSummaries) {
        String key = objSummary.getKey();

        TocInfo tocInfo = new TocInfo(key, objSummary.getSize());

        // is it a "dir/" ?
        if (key.lastIndexOf("/") == (key.length() - 1)) {
          tocInfo.isDirectory = true;
        } else {
          tocInfo.isDirectory = false;
        }

        toc.add(tocInfo);
        tocQueue.add(tocInfo);
        tocInfosGenerated++; // increment for logging
      }

      // for pagination
      nextMarker = objectListing.getNextMarker();
      if (nextMarker == null) {
        break;
      } else {
        listRequest.setMarker(nextMarker);
        logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
      }
    }
  }