Esempio n. 1
0
 private void completeUpload(InitiateMultipartUploadResult initResponse, List<PartETag> eTags)
     throws Exception {
   CompleteMultipartUploadRequest completeRequest =
       new CompleteMultipartUploadRequest(
           initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId(), eTags);
   s3Client.completeMultipartUpload(completeRequest);
 }
Esempio n. 2
0
  private PartETag uploadChunk(
      ByteBuffer bytes, InitiateMultipartUploadResult initResponse, int index) throws Exception {
    byte[] md5 = S3Utils.md5(bytes);

    UploadPartRequest request = new UploadPartRequest();
    request.setBucketName(initResponse.getBucketName());
    request.setKey(initResponse.getKey());
    request.setUploadId(initResponse.getUploadId());
    request.setPartNumber(index);
    request.setPartSize(bytes.limit());
    request.setMd5Digest(S3Utils.toBase64(md5));
    request.setInputStream(new ByteBufferInputStream(bytes));

    UploadPartResult response = s3Client.uploadPart(request);
    PartETag partETag = response.getPartETag();
    if (!response.getPartETag().getETag().equals(S3Utils.toHex(md5))) {
      throw new Exception("Unable to match MD5 for part " + index);
    }

    return partETag;
  }
  @Override
  boolean keyCopied(ObjectMetadata sourceMetadata, AccessControlList objectAcl) {
    long objectSize = summary.getSize();
    MirrorOptions options = context.getOptions();
    String sourceBucketName = options.getSourceBucket();
    int maxPartRetries = options.getMaxRetries();
    String targetBucketName = options.getDestinationBucket();
    List<CopyPartResult> copyResponses = new ArrayList<CopyPartResult>();
    if (options.isVerbose()) {
      log.info("Initiating multipart upload request for " + summary.getKey());
    }
    InitiateMultipartUploadRequest initiateRequest =
        new InitiateMultipartUploadRequest(targetBucketName, keydest)
            .withObjectMetadata(sourceMetadata);

    if (options.isCrossAccountCopy()) {
      initiateRequest.withCannedACL(CannedAccessControlList.BucketOwnerFullControl);
    } else {
      initiateRequest.withAccessControlList(objectAcl);
    }

    InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initiateRequest);

    long partSize = options.getUploadPartSize();
    long bytePosition = 0;

    for (int i = 1; bytePosition < objectSize; i++) {
      long lastByte =
          bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1;
      String infoMessage = "copying : " + bytePosition + " to " + lastByte;
      if (options.isVerbose()) {
        log.info(infoMessage);
      }
      CopyPartRequest copyRequest =
          new CopyPartRequest()
              .withDestinationBucketName(targetBucketName)
              .withDestinationKey(keydest)
              .withSourceBucketName(sourceBucketName)
              .withSourceKey(summary.getKey())
              .withUploadId(initResult.getUploadId())
              .withFirstByte(bytePosition)
              .withLastByte(lastByte)
              .withPartNumber(i);

      for (int tries = 1; tries <= maxPartRetries; tries++) {
        try {
          if (options.isVerbose()) log.info("try :" + tries);
          context.getStats().s3copyCount.incrementAndGet();
          CopyPartResult copyPartResult = client.copyPart(copyRequest);
          copyResponses.add(copyPartResult);
          if (options.isVerbose()) log.info("completed " + infoMessage);
          break;
        } catch (Exception e) {
          if (tries == maxPartRetries) {
            client.abortMultipartUpload(
                new AbortMultipartUploadRequest(
                    targetBucketName, keydest, initResult.getUploadId()));
            log.error("Exception while doing multipart copy", e);
            return false;
          }
        }
      }
      bytePosition += partSize;
    }
    CompleteMultipartUploadRequest completeRequest =
        new CompleteMultipartUploadRequest(
            targetBucketName, keydest, initResult.getUploadId(), getETags(copyResponses));
    client.completeMultipartUpload(completeRequest);
    if (options.isVerbose()) {
      log.info("completed multipart request for : " + summary.getKey());
    }
    context.getStats().bytesCopied.addAndGet(objectSize);
    return true;
  }
Esempio n. 4
0
 private void abortUpload(InitiateMultipartUploadResult initResponse) throws Exception {
   AbortMultipartUploadRequest abortRequest =
       new AbortMultipartUploadRequest(
           initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId());
   s3Client.abortMultipartUpload(abortRequest);
 }