private void completeUpload(InitiateMultipartUploadResult initResponse, List<PartETag> eTags)
throws Exception {
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest(
initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId(), eTags);
s3Client.completeMultipartUpload(completeRequest);
}
/*
@author: Nitish
*/
public static String UploadToS3() throws IOException {
String existingBucketName = "s3-bucket-zerorpc";
String keyName = "image.jpeg";
String filePath = "C:\\Users\\Nitish\\Desktop\\IMG_2695.JPG";
AmazonS3 s3Client = new AmazonS3Client(new BasicAWSCredentials("", ""));
// Create a list of UploadPartResponse objects. You get one of these
// for each part upload.
List<PartETag> partETags = new ArrayList<PartETag>();
// Step 1: Initialize.
InitiateMultipartUploadRequest initRequest =
new InitiateMultipartUploadRequest(existingBucketName, keyName);
InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
File file = new File(filePath);
long contentLength = file.length();
long partSize = 5242880; // Set part size to 5 MB.
try {
// Step 2: Upload parts.
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++) {
// Last part can be less than 5 MB. Adjust part size.
partSize = Math.min(partSize, (contentLength - filePosition));
// Create request to upload a part.
UploadPartRequest uploadRequest =
new UploadPartRequest()
.withBucketName(existingBucketName)
.withKey(keyName)
.withUploadId(initResponse.getUploadId())
.withPartNumber(i)
.withFileOffset(filePosition)
.withFile(file)
.withPartSize(partSize);
// Upload part and add response to our list.
partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
filePosition += partSize;
}
// Step 3: Complete.
CompleteMultipartUploadRequest compRequest =
new CompleteMultipartUploadRequest(
existingBucketName, keyName, initResponse.getUploadId(), partETags);
s3Client.completeMultipartUpload(compRequest);
} catch (Exception e) {
s3Client.abortMultipartUpload(
new AbortMultipartUploadRequest(existingBucketName, keyName, initResponse.getUploadId()));
}
return "Uploaded successfully";
}
/* (non-Javadoc)
* @see com.amazonaws.services.s3.AmazonS3Client#initiateMultipartUpload(com.amazonaws.services.s3.model.InitiateMultipartUploadRequest)
*/
@Override
public InitiateMultipartUploadResult initiateMultipartUpload(
InitiateMultipartUploadRequest initiateMultipartUploadRequest)
throws AmazonClientException, AmazonServiceException {
appendUserAgent(initiateMultipartUploadRequest, USER_AGENT);
// Generate a one-time use symmetric key and initialize a cipher to encrypt object data
SecretKey envelopeSymmetricKey = EncryptionUtils.generateOneTimeUseSymmetricKey();
Cipher symmetricCipher =
EncryptionUtils.createSymmetricCipher(
envelopeSymmetricKey, Cipher.ENCRYPT_MODE, cryptoConfig.getCryptoProvider(), null);
if (cryptoConfig.getStorageMode() == CryptoStorageMode.ObjectMetadata) {
EncryptionMaterials encryptionMaterials =
encryptionMaterialsProvider.getEncryptionMaterials();
// Encrypt the envelope symmetric key
byte[] encryptedEnvelopeSymmetricKey =
EncryptionUtils.getEncryptedSymmetricKey(
envelopeSymmetricKey, encryptionMaterials, cryptoConfig.getCryptoProvider());
// Store encryption info in metadata
ObjectMetadata metadata =
EncryptionUtils.updateMetadataWithEncryptionInfo(
initiateMultipartUploadRequest,
encryptedEnvelopeSymmetricKey,
symmetricCipher,
encryptionMaterials.getMaterialsDescription());
// Update the request's metadata to the updated metadata
initiateMultipartUploadRequest.setObjectMetadata(metadata);
}
InitiateMultipartUploadResult result =
super.initiateMultipartUpload(initiateMultipartUploadRequest);
EncryptedUploadContext encryptedUploadContext =
new EncryptedUploadContext(
initiateMultipartUploadRequest.getBucketName(),
initiateMultipartUploadRequest.getKey(),
envelopeSymmetricKey);
encryptedUploadContext.setNextInitializationVector(symmetricCipher.getIV());
encryptedUploadContext.setFirstInitializationVector(symmetricCipher.getIV());
currentMultipartUploadSecretKeys.put(result.getUploadId(), encryptedUploadContext);
return result;
}
private PartETag uploadChunk(
ByteBuffer bytes, InitiateMultipartUploadResult initResponse, int index) throws Exception {
byte[] md5 = S3Utils.md5(bytes);
UploadPartRequest request = new UploadPartRequest();
request.setBucketName(initResponse.getBucketName());
request.setKey(initResponse.getKey());
request.setUploadId(initResponse.getUploadId());
request.setPartNumber(index);
request.setPartSize(bytes.limit());
request.setMd5Digest(S3Utils.toBase64(md5));
request.setInputStream(new ByteBufferInputStream(bytes));
UploadPartResult response = s3Client.uploadPart(request);
PartETag partETag = response.getPartETag();
if (!response.getPartETag().getETag().equals(S3Utils.toHex(md5))) {
throw new Exception("Unable to match MD5 for part " + index);
}
return partETag;
}
public static String xs3_init_multi_upload(String xs3_objname, int file_size, String file_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
InitiateMultipartUploadRequest xs3_multi_req =
new InitiateMultipartUploadRequest(xs3_bucketname, xs3_objname);
xs3_multi_req.setCannedACL(CannedAccessControlList.PublicRead);
ObjectMetadata xs3_meta = new ObjectMetadata();
xs3_meta.setContentType(file_type);
xs3_multi_req.setObjectMetadata(xs3_meta);
InitiateMultipartUploadResult xs3_multi_res =
xs3_client.initiateMultipartUpload(xs3_multi_req);
String xs3_multi_uploadid = xs3_multi_res.getUploadId();
String json_urls = gen_part_url(xs3_multi_uploadid, file_size, xs3_objname, file_type);
return json_urls;
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
@Override
boolean keyCopied(ObjectMetadata sourceMetadata, AccessControlList objectAcl) {
long objectSize = summary.getSize();
MirrorOptions options = context.getOptions();
String sourceBucketName = options.getSourceBucket();
int maxPartRetries = options.getMaxRetries();
String targetBucketName = options.getDestinationBucket();
List<CopyPartResult> copyResponses = new ArrayList<CopyPartResult>();
if (options.isVerbose()) {
log.info("Initiating multipart upload request for " + summary.getKey());
}
InitiateMultipartUploadRequest initiateRequest =
new InitiateMultipartUploadRequest(targetBucketName, keydest)
.withObjectMetadata(sourceMetadata);
if (options.isCrossAccountCopy()) {
initiateRequest.withCannedACL(CannedAccessControlList.BucketOwnerFullControl);
} else {
initiateRequest.withAccessControlList(objectAcl);
}
InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initiateRequest);
long partSize = options.getUploadPartSize();
long bytePosition = 0;
for (int i = 1; bytePosition < objectSize; i++) {
long lastByte =
bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1;
String infoMessage = "copying : " + bytePosition + " to " + lastByte;
if (options.isVerbose()) {
log.info(infoMessage);
}
CopyPartRequest copyRequest =
new CopyPartRequest()
.withDestinationBucketName(targetBucketName)
.withDestinationKey(keydest)
.withSourceBucketName(sourceBucketName)
.withSourceKey(summary.getKey())
.withUploadId(initResult.getUploadId())
.withFirstByte(bytePosition)
.withLastByte(lastByte)
.withPartNumber(i);
for (int tries = 1; tries <= maxPartRetries; tries++) {
try {
if (options.isVerbose()) log.info("try :" + tries);
context.getStats().s3copyCount.incrementAndGet();
CopyPartResult copyPartResult = client.copyPart(copyRequest);
copyResponses.add(copyPartResult);
if (options.isVerbose()) log.info("completed " + infoMessage);
break;
} catch (Exception e) {
if (tries == maxPartRetries) {
client.abortMultipartUpload(
new AbortMultipartUploadRequest(
targetBucketName, keydest, initResult.getUploadId()));
log.error("Exception while doing multipart copy", e);
return false;
}
}
}
bytePosition += partSize;
}
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest(
targetBucketName, keydest, initResult.getUploadId(), getETags(copyResponses));
client.completeMultipartUpload(completeRequest);
if (options.isVerbose()) {
log.info("completed multipart request for : " + summary.getKey());
}
context.getStats().bytesCopied.addAndGet(objectSize);
return true;
}
private void abortUpload(InitiateMultipartUploadResult initResponse) throws Exception {
AbortMultipartUploadRequest abortRequest =
new AbortMultipartUploadRequest(
initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId());
s3Client.abortMultipartUpload(abortRequest);
}
private void doParallelUpload(String bucket, String key, File file, ObjectMetadata metadata)
throws IOException {
long contentLength = file.length();
long numParts = (contentLength + PARALLEL_UPLOAD_SIZE - 1) / PARALLEL_UPLOAD_SIZE;
long partSize = (contentLength / numParts) + 1;
log.info("Uploading {} in {} parts with part size {}", file, numParts, partSize);
final List<PartETag> partETags = Lists.newArrayList();
InitiateMultipartUploadRequest initRequest =
new InitiateMultipartUploadRequest(bucket, key, metadata);
InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
String uploadID = initResponse.getUploadId();
try {
long filePosition = 0;
int partNumber = 1;
Collection<Future<?>> futures = Lists.newArrayList();
while (filePosition < contentLength) {
final UploadPartRequest uploadRequest =
new UploadPartRequest()
.withBucketName(bucket)
.withKey(key)
.withUploadId(uploadID)
.withPartNumber(partNumber)
.withFileOffset(filePosition)
.withFile(file)
.withPartSize(FastMath.min(partSize, contentLength - filePosition));
futures.add(
executorService.submit(
new Callable<Object>() {
@Override
public Object call() {
String theKey = uploadRequest.getKey();
int thePartNumber = uploadRequest.getPartNumber();
log.info("Starting {} part {}", theKey, thePartNumber);
int failures = 0;
UploadPartResult uploadPartResult;
while (true) {
try {
uploadPartResult = s3Client.uploadPart(uploadRequest);
break;
} catch (AmazonClientException ace) {
if (++failures >= MAX_RETRIES) {
throw ace;
}
log.warn(
"Upload {} part {} failed ({}); retrying",
theKey,
thePartNumber,
ace.getMessage());
}
}
partETags.add(uploadPartResult.getPartETag());
log.info("Finished {} part {}", theKey, thePartNumber);
return null;
}
}));
filePosition += partSize;
partNumber++;
}
for (Future<?> future : futures) {
try {
future.get();
} catch (InterruptedException e) {
throw new IOException(e);
} catch (ExecutionException e) {
throw new IOException(e.getCause());
}
}
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest(bucket, key, uploadID, partETags);
s3Client.completeMultipartUpload(completeRequest);
} catch (AmazonClientException ace) {
AbortMultipartUploadRequest abortRequest =
new AbortMultipartUploadRequest(bucket, key, uploadID);
s3Client.abortMultipartUpload(abortRequest);
throw ace;
}
}
