private StoredObject createCombinedObjectSmall(CombinedStoredObject combinedObject) {
ImmutableList.Builder<InputSupplier<InputStream>> builder = ImmutableList.builder();
List<URI> sourceParts =
Lists.transform(combinedObject.getSourceParts(), StoredObject.GET_LOCATION_FUNCTION);
for (URI sourcePart : sourceParts) {
builder.add(getInputSupplier(sourcePart));
}
InputSupplier<InputStream> source = ByteStreams.join(builder.build());
File tempFile = null;
try {
tempFile =
File.createTempFile(
S3StorageHelper.getS3FileName(combinedObject.getLocation()), ".small.s3.data");
Files.copy(source, tempFile);
StoredObject result = putObject(combinedObject.getLocation(), tempFile);
return result;
} catch (IOException e) {
throw Throwables.propagate(e);
} finally {
if (tempFile != null) {
tempFile.delete();
}
}
}
private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) {
URI location = combinedObject.getLocation();
log.info("starting multipart upload: %s", location);
String bucket = getS3Bucket(location);
String key = getS3ObjectKey(location);
String uploadId =
s3Service
.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key))
.getUploadId();
try {
List<PartETag> parts = newArrayList();
int partNumber = 1;
for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
CopyPartResult part =
s3Service.copyPart(
new CopyPartRequest()
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withDestinationBucketName(bucket)
.withDestinationKey(key)
.withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation()))
.withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation())));
parts.add(new PartETag(partNumber, part.getETag()));
partNumber++;
}
String etag =
s3Service
.completeMultipartUpload(
new CompleteMultipartUploadRequest(bucket, key, uploadId, parts))
.getETag();
ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key);
log.info("completed multipart upload: %s", location);
if (!etag.equals(newObject.getETag())) {
// this might happen in rare cases due to S3's eventual consistency
throw new IllegalStateException("completed etag is different from combined object etag");
}
return updateStoredObject(location, newObject);
} catch (AmazonClientException e) {
try {
s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
} catch (AmazonClientException ignored) {
}
throw Throwables.propagate(e);
}
}
@Override
public StoredObject createCombinedObject(CombinedStoredObject combinedObject) {
Preconditions.checkNotNull(combinedObject, "combinedObject is null");
Preconditions.checkArgument(
!combinedObject.getSourceParts().isEmpty(), "combinedObject sourceParts is empty");
boolean setIsSmall = combinedObject.getSourceParts().get(0).getSize() < 5 * 1024 * 1024;
// verify size
for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
boolean fileIsSmall = newCombinedObjectPart.getSize() < 5 * 1024 * 1024;
Preconditions.checkArgument(
fileIsSmall == setIsSmall,
"combinedObject sourceParts contains mixed large and small files");
}
return setIsSmall
? createCombinedObjectSmall(combinedObject)
: createCombinedObjectLarge(combinedObject);
}