示例#1
0
 S3ObjectSummary findAndQueueObjects(AmazonS3Source.S3Offset s3offset, boolean checkCurrent)
     throws AmazonClientException {
   List<S3ObjectSummary> s3ObjectSummaries;
   ObjectOrdering objectOrdering = s3ConfigBean.s3FileConfig.objectOrdering;
   switch (objectOrdering) {
     case TIMESTAMP:
       s3ObjectSummaries =
           AmazonS3Util.listObjectsChronologically(
               s3Client, s3ConfigBean, pathMatcher, s3offset, objectQueue.remainingCapacity());
       break;
     case LEXICOGRAPHICAL:
       s3ObjectSummaries =
           AmazonS3Util.listObjectsLexicographically(
               s3Client, s3ConfigBean, pathMatcher, s3offset, objectQueue.remainingCapacity());
       break;
     default:
       throw new IllegalArgumentException("Unknown ordering: " + objectOrdering.getLabel());
   }
   for (S3ObjectSummary objectSummary : s3ObjectSummaries) {
     addObjectToQueue(objectSummary, checkCurrent);
   }
   spoolQueueMeter.mark(objectQueue.size());
   LOG.debug("Found '{}' files", objectQueue.size());
   return (s3ObjectSummaries.isEmpty())
       ? null
       : s3ObjectSummaries.get(s3ObjectSummaries.size() - 1);
 }
示例#2
0
  public void postProcessOlderObjectIfNeeded(AmazonS3Source.S3Offset s3Offset) {
    // If sdc was shutdown after reading an object but before post processing it, handle it now.

    // The scenario is detected as follows:
    //  1. the current key must not be null
    //  2. offset must be -1
    //  3. An object with same key must exist in s3
    //  4. The timestamp of the object ins3 must be same as that of the timestamp in offset [It is
    // possible that one
    //    uploads another object with the same name. We can avoid post processing it without
    // producing records by
    //    comparing the timestamp on that object

    if (s3Offset.getKey() != null && "-1".equals(s3Offset.getOffset())) {
      // conditions 1, 2 are met. Check for 3 and 4.
      S3ObjectSummary objectSummary =
          AmazonS3Util.getObjectSummary(s3Client, s3ConfigBean.s3Config.bucket, s3Offset.getKey());
      if (objectSummary != null
          && objectSummary
                  .getLastModified()
                  .compareTo(new Date(Long.parseLong(s3Offset.getTimestamp())))
              == 0) {
        postProcessOrErrorHandle(
            s3Offset.getKey(),
            s3ConfigBean.postProcessingConfig.postProcessing,
            s3ConfigBean.postProcessingConfig.postProcessBucket,
            s3ConfigBean.postProcessingConfig.postProcessPrefix,
            s3ConfigBean.postProcessingConfig.archivingOption);
      }
    }
    currentObject = null;
  }
示例#3
0
 private void archive(
     String postProcessObjectKey,
     String postProcessBucket,
     String postProcessFolder,
     S3ArchivingOption archivingOption) {
   boolean isMove = true;
   String destBucket = s3ConfigBean.s3Config.bucket;
   switch (archivingOption) {
     case MOVE_TO_PREFIX:
       break;
     case MOVE_TO_BUCKET:
       destBucket = postProcessBucket;
       break;
     case COPY_TO_PREFIX:
       isMove = false;
       break;
     case COPY_TO_BUCKET:
       isMove = false;
       destBucket = postProcessBucket;
       break;
     default:
       throw new IllegalStateException("Invalid Archive option : " + archivingOption.name());
   }
   String srcObjKey =
       postProcessObjectKey.substring(
           postProcessObjectKey.lastIndexOf(s3ConfigBean.s3Config.delimiter) + 1);
   String destKey = postProcessFolder + srcObjKey;
   AmazonS3Util.copy(
       s3Client, s3ConfigBean.s3Config.bucket, postProcessObjectKey, destBucket, destKey, isMove);
 }