private void completeUpload(InitiateMultipartUploadResult initResponse, List<PartETag> eTags)
throws Exception {
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest(
initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId(), eTags);
s3Client.completeMultipartUpload(completeRequest);
}
private PartETag uploadChunk(
ByteBuffer bytes, InitiateMultipartUploadResult initResponse, int index) throws Exception {
byte[] md5 = S3Utils.md5(bytes);
UploadPartRequest request = new UploadPartRequest();
request.setBucketName(initResponse.getBucketName());
request.setKey(initResponse.getKey());
request.setUploadId(initResponse.getUploadId());
request.setPartNumber(index);
request.setPartSize(bytes.limit());
request.setMd5Digest(S3Utils.toBase64(md5));
request.setInputStream(new ByteBufferInputStream(bytes));
UploadPartResult response = s3Client.uploadPart(request);
PartETag partETag = response.getPartETag();
if (!response.getPartETag().getETag().equals(S3Utils.toHex(md5))) {
throw new Exception("Unable to match MD5 for part " + index);
}
return partETag;
}
@Override
boolean keyCopied(ObjectMetadata sourceMetadata, AccessControlList objectAcl) {
long objectSize = summary.getSize();
MirrorOptions options = context.getOptions();
String sourceBucketName = options.getSourceBucket();
int maxPartRetries = options.getMaxRetries();
String targetBucketName = options.getDestinationBucket();
List<CopyPartResult> copyResponses = new ArrayList<CopyPartResult>();
if (options.isVerbose()) {
log.info("Initiating multipart upload request for " + summary.getKey());
}
InitiateMultipartUploadRequest initiateRequest =
new InitiateMultipartUploadRequest(targetBucketName, keydest)
.withObjectMetadata(sourceMetadata);
if (options.isCrossAccountCopy()) {
initiateRequest.withCannedACL(CannedAccessControlList.BucketOwnerFullControl);
} else {
initiateRequest.withAccessControlList(objectAcl);
}
InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initiateRequest);
long partSize = options.getUploadPartSize();
long bytePosition = 0;
for (int i = 1; bytePosition < objectSize; i++) {
long lastByte =
bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1;
String infoMessage = "copying : " + bytePosition + " to " + lastByte;
if (options.isVerbose()) {
log.info(infoMessage);
}
CopyPartRequest copyRequest =
new CopyPartRequest()
.withDestinationBucketName(targetBucketName)
.withDestinationKey(keydest)
.withSourceBucketName(sourceBucketName)
.withSourceKey(summary.getKey())
.withUploadId(initResult.getUploadId())
.withFirstByte(bytePosition)
.withLastByte(lastByte)
.withPartNumber(i);
for (int tries = 1; tries <= maxPartRetries; tries++) {
try {
if (options.isVerbose()) log.info("try :" + tries);
context.getStats().s3copyCount.incrementAndGet();
CopyPartResult copyPartResult = client.copyPart(copyRequest);
copyResponses.add(copyPartResult);
if (options.isVerbose()) log.info("completed " + infoMessage);
break;
} catch (Exception e) {
if (tries == maxPartRetries) {
client.abortMultipartUpload(
new AbortMultipartUploadRequest(
targetBucketName, keydest, initResult.getUploadId()));
log.error("Exception while doing multipart copy", e);
return false;
}
}
}
bytePosition += partSize;
}
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest(
targetBucketName, keydest, initResult.getUploadId(), getETags(copyResponses));
client.completeMultipartUpload(completeRequest);
if (options.isVerbose()) {
log.info("completed multipart request for : " + summary.getKey());
}
context.getStats().bytesCopied.addAndGet(objectSize);
return true;
}
private void abortUpload(InitiateMultipartUploadResult initResponse) throws Exception {
AbortMultipartUploadRequest abortRequest =
new AbortMultipartUploadRequest(
initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId());
s3Client.abortMultipartUpload(abortRequest);
}