private void completeUpload(InitiateMultipartUploadResult initResponse, List<PartETag> eTags)
throws Exception {
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest(
initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId(), eTags);
s3Client.completeMultipartUpload(completeRequest);
}
@Override
public UploadResult uploadBackup(
Exhibitor exhibitor,
BackupMetaData backup,
File source,
final Map<String, String> configValues)
throws Exception {
List<BackupMetaData> availableBackups = getAvailableBackups(exhibitor, configValues);
if (availableBackups.contains(backup)) {
return UploadResult.DUPLICATE;
}
RetryPolicy retryPolicy = makeRetryPolicy(configValues);
Throttle throttle = makeThrottle(configValues);
String key = toKey(backup);
InitiateMultipartUploadRequest initRequest =
new InitiateMultipartUploadRequest(configValues.get(CONFIG_BUCKET.getKey()), key);
InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
CompressorIterator compressorIterator = compressor.compress(source);
try {
List<PartETag> eTags = Lists.newArrayList();
int index = 0;
for (; ; ) {
ByteBuffer chunk = compressorIterator.next();
if (chunk == null) {
break;
}
throttle.throttle(chunk.limit());
PartETag eTag = uploadChunkWithRetry(chunk, initResponse, index++, retryPolicy);
eTags.add(eTag);
}
completeUpload(initResponse, eTags);
} catch (Exception e) {
abortUpload(initResponse);
throw e;
} finally {
Closeables.closeQuietly(compressorIterator);
}
UploadResult result = UploadResult.SUCCEEDED;
for (BackupMetaData existing : availableBackups) {
if (existing.getName().equals(backup.getName())) {
deleteBackup(exhibitor, existing, configValues);
result = UploadResult.REPLACED_OLD_VERSION;
}
}
return result;
}
private S3Object getConfigObject() throws Exception {
try {
S3Object object = s3Client.getObject(arguments.getBucket(), arguments.getKey());
if (object.getObjectMetadata().getContentLength() > 0) {
return object;
}
} catch (AmazonS3Exception e) {
if (!isNotFoundError(e)) {
throw e;
}
}
return null;
}
private ObjectMetadata getConfigMetadata() throws Exception {
try {
ObjectMetadata metadata =
s3Client.getObjectMetadata(arguments.getBucket(), arguments.getKey());
if (metadata.getContentLength() > 0) {
return metadata;
}
} catch (AmazonS3Exception e) {
if (!isNotFoundError(e)) {
throw e;
}
}
return null;
}
@Override
public List<BackupMetaData> getAvailableBackups(
Exhibitor exhibitor, Map<String, String> configValues) throws Exception {
ListObjectsRequest request = new ListObjectsRequest();
request.setBucketName(configValues.get(CONFIG_BUCKET.getKey()));
ObjectListing listing = s3Client.listObjects(request);
return Lists.transform(
listing.getObjectSummaries(),
new Function<S3ObjectSummary, BackupMetaData>() {
@Override
public BackupMetaData apply(S3ObjectSummary summary) {
return fromKey(summary.getKey());
}
});
}
@Override
public void downloadBackup(
Exhibitor exhibitor,
BackupMetaData backup,
File destination,
Map<String, String> configValues)
throws Exception {
S3Object object = s3Client.getObject(configValues.get(CONFIG_BUCKET.getKey()), toKey(backup));
long startMs = System.currentTimeMillis();
RetryPolicy retryPolicy = makeRetryPolicy(configValues);
int retryCount = 0;
boolean done = false;
while (!done) {
Throttle throttle = makeThrottle(configValues);
InputStream in = null;
FileOutputStream out = null;
try {
out = new FileOutputStream(destination);
in = object.getObjectContent();
FileChannel channel = out.getChannel();
CompressorIterator compressorIterator = compressor.decompress(in);
for (; ; ) {
ByteBuffer bytes = compressorIterator.next();
if (bytes == null) {
break;
}
throttle.throttle(bytes.limit());
channel.write(bytes);
}
done = true;
} catch (Exception e) {
if (!retryPolicy.allowRetry(retryCount++, System.currentTimeMillis() - startMs)) {
done = true;
}
} finally {
Closeables.closeQuietly(in);
Closeables.closeQuietly(out);
}
}
}
private PartETag uploadChunk(
ByteBuffer bytes, InitiateMultipartUploadResult initResponse, int index) throws Exception {
byte[] md5 = S3Utils.md5(bytes);
UploadPartRequest request = new UploadPartRequest();
request.setBucketName(initResponse.getBucketName());
request.setKey(initResponse.getKey());
request.setUploadId(initResponse.getUploadId());
request.setPartNumber(index);
request.setPartSize(bytes.limit());
request.setMd5Digest(S3Utils.toBase64(md5));
request.setInputStream(new ByteBufferInputStream(bytes));
UploadPartResult response = s3Client.uploadPart(request);
PartETag partETag = response.getPartETag();
if (!response.getPartETag().getETag().equals(S3Utils.toHex(md5))) {
throw new Exception("Unable to match MD5 for part " + index);
}
return partETag;
}
@Override
public void close() throws IOException {
s3Client.close();
}
private void abortUpload(InitiateMultipartUploadResult initResponse) throws Exception {
AbortMultipartUploadRequest abortRequest =
new AbortMultipartUploadRequest(
initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId());
s3Client.abortMultipartUpload(abortRequest);
}
@Override
public void deleteBackup(
Exhibitor exhibitor, BackupMetaData backup, Map<String, String> configValues)
throws Exception {
s3Client.deleteObject(configValues.get(CONFIG_BUCKET.getKey()), toKey(backup));
}