Пример #1
0
 private void completeUpload(InitiateMultipartUploadResult initResponse, List<PartETag> eTags)
     throws Exception {
   CompleteMultipartUploadRequest completeRequest =
       new CompleteMultipartUploadRequest(
           initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId(), eTags);
   s3Client.completeMultipartUpload(completeRequest);
 }
Пример #2
0
  @Override
  public UploadResult uploadBackup(
      Exhibitor exhibitor,
      BackupMetaData backup,
      File source,
      final Map<String, String> configValues)
      throws Exception {
    List<BackupMetaData> availableBackups = getAvailableBackups(exhibitor, configValues);
    if (availableBackups.contains(backup)) {
      return UploadResult.DUPLICATE;
    }

    RetryPolicy retryPolicy = makeRetryPolicy(configValues);
    Throttle throttle = makeThrottle(configValues);

    String key = toKey(backup);
    InitiateMultipartUploadRequest initRequest =
        new InitiateMultipartUploadRequest(configValues.get(CONFIG_BUCKET.getKey()), key);
    InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);

    CompressorIterator compressorIterator = compressor.compress(source);
    try {
      List<PartETag> eTags = Lists.newArrayList();
      int index = 0;
      for (; ; ) {
        ByteBuffer chunk = compressorIterator.next();
        if (chunk == null) {
          break;
        }
        throttle.throttle(chunk.limit());

        PartETag eTag = uploadChunkWithRetry(chunk, initResponse, index++, retryPolicy);
        eTags.add(eTag);
      }

      completeUpload(initResponse, eTags);
    } catch (Exception e) {
      abortUpload(initResponse);
      throw e;
    } finally {
      Closeables.closeQuietly(compressorIterator);
    }

    UploadResult result = UploadResult.SUCCEEDED;
    for (BackupMetaData existing : availableBackups) {
      if (existing.getName().equals(backup.getName())) {
        deleteBackup(exhibitor, existing, configValues);
        result = UploadResult.REPLACED_OLD_VERSION;
      }
    }
    return result;
  }
Пример #3
0
 private S3Object getConfigObject() throws Exception {
   try {
     S3Object object = s3Client.getObject(arguments.getBucket(), arguments.getKey());
     if (object.getObjectMetadata().getContentLength() > 0) {
       return object;
     }
   } catch (AmazonS3Exception e) {
     if (!isNotFoundError(e)) {
       throw e;
     }
   }
   return null;
 }
Пример #4
0
 private ObjectMetadata getConfigMetadata() throws Exception {
   try {
     ObjectMetadata metadata =
         s3Client.getObjectMetadata(arguments.getBucket(), arguments.getKey());
     if (metadata.getContentLength() > 0) {
       return metadata;
     }
   } catch (AmazonS3Exception e) {
     if (!isNotFoundError(e)) {
       throw e;
     }
   }
   return null;
 }
Пример #5
0
 @Override
 public List<BackupMetaData> getAvailableBackups(
     Exhibitor exhibitor, Map<String, String> configValues) throws Exception {
   ListObjectsRequest request = new ListObjectsRequest();
   request.setBucketName(configValues.get(CONFIG_BUCKET.getKey()));
   ObjectListing listing = s3Client.listObjects(request);
   return Lists.transform(
       listing.getObjectSummaries(),
       new Function<S3ObjectSummary, BackupMetaData>() {
         @Override
         public BackupMetaData apply(S3ObjectSummary summary) {
           return fromKey(summary.getKey());
         }
       });
 }
Пример #6
0
  @Override
  public void downloadBackup(
      Exhibitor exhibitor,
      BackupMetaData backup,
      File destination,
      Map<String, String> configValues)
      throws Exception {
    S3Object object = s3Client.getObject(configValues.get(CONFIG_BUCKET.getKey()), toKey(backup));

    long startMs = System.currentTimeMillis();
    RetryPolicy retryPolicy = makeRetryPolicy(configValues);
    int retryCount = 0;
    boolean done = false;
    while (!done) {
      Throttle throttle = makeThrottle(configValues);
      InputStream in = null;
      FileOutputStream out = null;
      try {
        out = new FileOutputStream(destination);
        in = object.getObjectContent();

        FileChannel channel = out.getChannel();
        CompressorIterator compressorIterator = compressor.decompress(in);
        for (; ; ) {
          ByteBuffer bytes = compressorIterator.next();
          if (bytes == null) {
            break;
          }

          throttle.throttle(bytes.limit());
          channel.write(bytes);
        }

        done = true;
      } catch (Exception e) {
        if (!retryPolicy.allowRetry(retryCount++, System.currentTimeMillis() - startMs)) {
          done = true;
        }
      } finally {
        Closeables.closeQuietly(in);
        Closeables.closeQuietly(out);
      }
    }
  }
Пример #7
0
  private PartETag uploadChunk(
      ByteBuffer bytes, InitiateMultipartUploadResult initResponse, int index) throws Exception {
    byte[] md5 = S3Utils.md5(bytes);

    UploadPartRequest request = new UploadPartRequest();
    request.setBucketName(initResponse.getBucketName());
    request.setKey(initResponse.getKey());
    request.setUploadId(initResponse.getUploadId());
    request.setPartNumber(index);
    request.setPartSize(bytes.limit());
    request.setMd5Digest(S3Utils.toBase64(md5));
    request.setInputStream(new ByteBufferInputStream(bytes));

    UploadPartResult response = s3Client.uploadPart(request);
    PartETag partETag = response.getPartETag();
    if (!response.getPartETag().getETag().equals(S3Utils.toHex(md5))) {
      throw new Exception("Unable to match MD5 for part " + index);
    }

    return partETag;
  }
Пример #8
0
 @Override
 public void close() throws IOException {
   s3Client.close();
 }
Пример #9
0
 private void abortUpload(InitiateMultipartUploadResult initResponse) throws Exception {
   AbortMultipartUploadRequest abortRequest =
       new AbortMultipartUploadRequest(
           initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId());
   s3Client.abortMultipartUpload(abortRequest);
 }
Пример #10
0
 @Override
 public void deleteBackup(
     Exhibitor exhibitor, BackupMetaData backup, Map<String, String> configValues)
     throws Exception {
   s3Client.deleteObject(configValues.get(CONFIG_BUCKET.getKey()), toKey(backup));
 }