Пример #1
0
 @Override
 public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
   List<NoteInfo> infos = new LinkedList<>();
   NoteInfo info;
   try {
     ListObjectsRequest listObjectsRequest =
         new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
     ObjectListing objectListing;
     do {
       objectListing = s3client.listObjects(listObjectsRequest);
       for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
         if (objectSummary.getKey().endsWith("note.json")) {
           info = getNoteInfo(objectSummary.getKey());
           if (info != null) {
             infos.add(info);
           }
         }
       }
       listObjectsRequest.setMarker(objectListing.getNextMarker());
     } while (objectListing.isTruncated());
   } catch (AmazonClientException ace) {
     throw new IOException("Unable to list objects in S3: " + ace, ace);
   }
   return infos;
 }
Пример #2
0
  /**
   * Reads next key from S3
   *
   * @return {@link S3ObjectSummary} of the key or <code>null</code> if there are no more keys to
   *     read
   */
  public S3ObjectSummary getNextKey() {
    if (objectListing == null) {
      LOG.debug("Listing objects");
      objectListing = s3Client.listObjects(listObjectsRequest);
    }

    // we have more elements in list
    if (currentPosition < objectListing.getObjectSummaries().size()) {
      currentObject = objectListing.getObjectSummaries().get(currentPosition++);
    } else if (objectListing.isTruncated()) {
      LOG.debug("Current batch reached its end. Fetching next page.");
      // when we're at the end, check if there's more data to read
      listObjectsRequest.setMarker(objectListing.getNextMarker());
      LOG.debug("New marker is set to {}", listObjectsRequest.getMarker());

      objectListing = s3Client.listObjects(listObjectsRequest);
      currentPosition = 0;

      if (currentPosition < objectListing.getObjectSummaries().size()) {
        currentObject = objectListing.getObjectSummaries().get(currentPosition++);
      } else {
        LOG.debug("No more objects to read");
        currentObject = null;
      }
    } else {
      currentObject = null;
    }

    return currentObject;
  }
  @Override
  public void recursiveDelete(String keyPrefix, FilenameFilter except) throws IOException {

    if (keyPrefix.charAt(keyPrefix.length() - 1) == '/') {
      // Need to delete the dir too, and to list it, can't specify its trailing slash
      keyPrefix = keyPrefix.substring(0, keyPrefix.length() - 1);
    }

    boolean truncated = true;
    String marker = null;

    String bucket = Namespaces.get().getBucket();
    while (truncated) {

      ListObjectsRequest listRequest =
          new ListObjectsRequest(bucket, keyPrefix, marker, null, null);
      ObjectListing listing;
      try {
        listing = s3Client.listObjects(listRequest);
        Collection<S3ObjectSummary> summaries = listing.getObjectSummaries();
        if (summaries.isEmpty()) {
          return;
        }

        List<DeleteObjectsRequest.KeyVersion> keysToDelete =
            Lists.newArrayListWithCapacity(summaries.size());
        for (S3ObjectSummary summary : summaries) {
          String key = summary.getKey();
          if (except == null || !except.accept(null, key)) {
            keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key));
          }
        }

        DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket);
        deleteObjectsRequest.setKeys(keysToDelete);
        s3Client.deleteObjects(deleteObjectsRequest);
      } catch (AmazonClientException ace) {
        throw new IOException(ace);
      }

      truncated = listing.isTruncated();
      marker = listing.getNextMarker();
    }
  }
  private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {

    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(s3BucketName);
    // listRequest.setGeneralProgressListener(this);
    listRequest.setMaxKeys(1000);

    String nextMarker = null;
    ObjectListing objectListing = null;

    while (true) {

      objectListing = s3Client.listObjects(listRequest);

      List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();

      for (S3ObjectSummary objSummary : objectSummaries) {
        String key = objSummary.getKey();

        TocInfo tocInfo = new TocInfo(key, objSummary.getSize());

        // is it a "dir/" ?
        if (key.lastIndexOf("/") == (key.length() - 1)) {
          tocInfo.isDirectory = true;
        } else {
          tocInfo.isDirectory = false;
        }

        toc.add(tocInfo);
        tocQueue.add(tocInfo);
        tocInfosGenerated++; // increment for logging
      }

      // for pagination
      nextMarker = objectListing.getNextMarker();
      if (nextMarker == null) {
        break;
      } else {
        listRequest.setMarker(nextMarker);
        logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
      }
    }
  }
 @Override
 public List<String> list(String prefix, boolean files) throws IOException {
   boolean truncated = true;
   String marker = null;
   List<String> result = Lists.newArrayList();
   while (truncated) {
     if (marker != null) {
       log.info("Next page after {}", marker);
     }
     ListObjectsRequest listRequest =
         new ListObjectsRequest(Namespaces.get().getBucket(), prefix, marker, "/", null);
     ObjectListing listing;
     try {
       listing = s3Client.listObjects(listRequest);
     } catch (AmazonClientException ace) {
       throw new IOException(ace);
     }
     truncated = listing.isTruncated();
     marker = listing.getNextMarker();
     if (!truncated) {
       if (files) {
         for (S3ObjectSummary summary : listing.getObjectSummaries()) {
           String key = summary.getKey();
           if (!key.endsWith("_SUCCESS")) {
             result.add(key);
           }
         }
       } else {
         for (String key : listing.getCommonPrefixes()) {
           if (!key.endsWith("_SUCCESS")) {
             result.add(key);
           }
         }
       }
     }
   }
   Collections.sort(result);
   return result;
 }