private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) {
URI location = combinedObject.getLocation();
log.info("starting multipart upload: %s", location);
String bucket = getS3Bucket(location);
String key = getS3ObjectKey(location);
String uploadId =
s3Service
.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key))
.getUploadId();
try {
List<PartETag> parts = newArrayList();
int partNumber = 1;
for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
CopyPartResult part =
s3Service.copyPart(
new CopyPartRequest()
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withDestinationBucketName(bucket)
.withDestinationKey(key)
.withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation()))
.withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation())));
parts.add(new PartETag(partNumber, part.getETag()));
partNumber++;
}
String etag =
s3Service
.completeMultipartUpload(
new CompleteMultipartUploadRequest(bucket, key, uploadId, parts))
.getETag();
ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key);
log.info("completed multipart upload: %s", location);
if (!etag.equals(newObject.getETag())) {
// this might happen in rare cases due to S3's eventual consistency
throw new IllegalStateException("completed etag is different from combined object etag");
}
return updateStoredObject(location, newObject);
} catch (AmazonClientException e) {
try {
s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
} catch (AmazonClientException ignored) {
}
throw Throwables.propagate(e);
}
}
public class Main {
private static final Logger log = Logger.get(Main.class);
public static void main(String[] args) {
try {
Bootstrap app =
new Bootstrap(
new CassandraModule(),
new NodeModule(),
new HttpServerModule(),
new JaxrsModule(),
new JsonModule(),
new MBeanModule(),
new JmxModule(),
new DiscoveryModule(),
new HttpEventModule())
.strictConfig();
app.initialize();
} catch (Exception e) {
log.error(e);
// Cassandra prevents the vm from shutting down on its own
System.exit(1);
}
}
}
public class Share implements Resource {
public static final String SERVICE_NAME = "SHARE_SERVICE";
private final Logger log = Logger.get(Share.class);
private final HttpServiceSelector serviceSelector;
@Inject
public Share(@ServiceType(SERVICE_NAME) HttpServiceSelector serviceSelector) {
Preconditions.checkNotNull(serviceSelector, "Customer serviceSelector must not be null");
this.serviceSelector = serviceSelector;
}
@Override
public List<URI> getServices() {
try {
// TODO: FIXME - This is a hack to get consistent results from discovery
Thread.sleep(500);
return serviceSelector.selectHttpService();
} catch (InterruptedException e) {
throw new NoSuchElementException(
SERVICE_NAME + " Service not found - " + serviceSelector.toString());
}
}
@Override
public URI getService() throws NoSuchElementException {
List<URI> services = getServices();
if (services.size() > 0) {
return services.get(0);
}
throw new NoSuchElementException(
SERVICE_NAME + " Service not found - " + serviceSelector.toString());
}
}
@Override
public StoredObject putObject(URI location, File source) {
try {
log.info("starting upload: %s", location);
PutObjectResult result =
s3Service.putObject(getS3Bucket(location), getS3ObjectKey(location), source);
ObjectMetadata metadata =
s3Service.getObjectMetadata(getS3Bucket(location), getS3ObjectKey(location));
if (!result.getETag().equals(metadata.getETag())) {
// this might happen in rare cases due to S3's eventual consistency
throw new IllegalStateException("uploaded etag is different from retrieved object etag");
}
log.info("completed upload: %s", location);
return updateStoredObject(location, metadata);
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
@PreDestroy
public void destroy() {
executor.shutdownNow();
try {
executor.awaitTermination(30, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
// unannounce
try {
getFutureResult(announcementClient.unannounce(), DiscoveryException.class);
} catch (DiscoveryException e) {
if (e.getCause() instanceof ConnectException) {
log.error(
"Cannot connect to discovery server for unannounce: %s", e.getCause().getMessage());
} else {
log.error(e);
}
}
}
public class S3StorageSystem {
private static final Logger log = Logger.get(S3StorageSystem.class);
private final AmazonS3 s3Service;
@Inject
public S3StorageSystem(AmazonS3 s3Service) {
Preconditions.checkNotNull(s3Service, "s3Service is null");
this.s3Service = s3Service;
}
public List<URI> listDirectoriesNewestFirst(URI storageArea) {
S3StorageHelper.checkValidS3Uri(storageArea);
String bucket = getS3Bucket(storageArea);
String key = getS3ObjectKey(storageArea);
Iterator<String> iter =
new S3PrefixListing(s3Service, new ListObjectsRequest(bucket, key, null, "/", null))
.iterator();
ImmutableList.Builder<URI> builder = ImmutableList.builder();
while (iter.hasNext()) {
builder.add(buildS3Location("s3://", bucket, iter.next()));
}
return builder.build().reverse();
}
public List<URI> listObjects(URI storageArea) {
S3StorageHelper.checkValidS3Uri(storageArea);
String s3Path = getS3ObjectKey(storageArea);
Iterable<S3ObjectSummary> objectListing =
new S3ObjectListing(
s3Service, new ListObjectsRequest(getS3Bucket(storageArea), s3Path, null, "/", null));
ImmutableList.Builder<URI> builder = ImmutableList.builder();
for (S3ObjectSummary summary : objectListing) {
builder.add(buildS3Location(storageArea, summary.getKey().substring(s3Path.length())));
}
return builder.build();
}
public InputSupplier<InputStream> getInputSupplier(URI target) {
return new S3InputSupplier(s3Service, target);
}
}
public class S3StorageSystem implements StorageSystem {
private static final Logger log = Logger.get(S3StorageSystem.class);
private final AmazonS3 s3Service;
@Inject
public S3StorageSystem(AmazonS3 s3Service) {
Preconditions.checkNotNull(s3Service, "s3Service is null");
this.s3Service = s3Service;
}
@Override
public List<URI> listDirectories(URI storageArea) {
S3StorageHelper.checkValidS3Uri(storageArea);
String bucket = getS3Bucket(storageArea);
String key = getS3ObjectKey(storageArea);
Iterator<String> iter =
new S3PrefixListing(s3Service, new ListObjectsRequest(bucket, key, null, "/", null))
.iterator();
ImmutableList.Builder<URI> builder = ImmutableList.builder();
while (iter.hasNext()) {
builder.add(buildS3Location("s3://", bucket, iter.next()));
}
return builder.build();
}
@Override
public List<StoredObject> listObjects(URI storageArea) {
S3StorageHelper.checkValidS3Uri(storageArea);
String s3Path = getS3ObjectKey(storageArea);
Iterator<S3ObjectSummary> iter =
new S3ObjectListing(
s3Service,
new ListObjectsRequest(getS3Bucket(storageArea), s3Path, null, "/", null))
.iterator();
ImmutableList.Builder<StoredObject> builder = ImmutableList.builder();
while (iter.hasNext()) {
S3ObjectSummary summary = iter.next();
builder.add(
new StoredObject(
buildS3Location(storageArea, summary.getKey().substring(s3Path.length())),
summary.getETag(),
summary.getSize(),
summary.getLastModified().getTime()));
}
return builder.build();
}
@Override
public StoredObject createCombinedObject(CombinedStoredObject combinedObject) {
Preconditions.checkNotNull(combinedObject, "combinedObject is null");
Preconditions.checkArgument(
!combinedObject.getSourceParts().isEmpty(), "combinedObject sourceParts is empty");
boolean setIsSmall = combinedObject.getSourceParts().get(0).getSize() < 5 * 1024 * 1024;
// verify size
for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
boolean fileIsSmall = newCombinedObjectPart.getSize() < 5 * 1024 * 1024;
Preconditions.checkArgument(
fileIsSmall == setIsSmall,
"combinedObject sourceParts contains mixed large and small files");
}
return setIsSmall
? createCombinedObjectSmall(combinedObject)
: createCombinedObjectLarge(combinedObject);
}
private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) {
URI location = combinedObject.getLocation();
log.info("starting multipart upload: %s", location);
String bucket = getS3Bucket(location);
String key = getS3ObjectKey(location);
String uploadId =
s3Service
.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key))
.getUploadId();
try {
List<PartETag> parts = newArrayList();
int partNumber = 1;
for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
CopyPartResult part =
s3Service.copyPart(
new CopyPartRequest()
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withDestinationBucketName(bucket)
.withDestinationKey(key)
.withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation()))
.withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation())));
parts.add(new PartETag(partNumber, part.getETag()));
partNumber++;
}
String etag =
s3Service
.completeMultipartUpload(
new CompleteMultipartUploadRequest(bucket, key, uploadId, parts))
.getETag();
ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key);
log.info("completed multipart upload: %s", location);
if (!etag.equals(newObject.getETag())) {
// this might happen in rare cases due to S3's eventual consistency
throw new IllegalStateException("completed etag is different from combined object etag");
}
return updateStoredObject(location, newObject);
} catch (AmazonClientException e) {
try {
s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
} catch (AmazonClientException ignored) {
}
throw Throwables.propagate(e);
}
}
private StoredObject createCombinedObjectSmall(CombinedStoredObject combinedObject) {
ImmutableList.Builder<InputSupplier<InputStream>> builder = ImmutableList.builder();
List<URI> sourceParts =
Lists.transform(combinedObject.getSourceParts(), StoredObject.GET_LOCATION_FUNCTION);
for (URI sourcePart : sourceParts) {
builder.add(getInputSupplier(sourcePart));
}
InputSupplier<InputStream> source = ByteStreams.join(builder.build());
File tempFile = null;
try {
tempFile =
File.createTempFile(
S3StorageHelper.getS3FileName(combinedObject.getLocation()), ".small.s3.data");
Files.copy(source, tempFile);
StoredObject result = putObject(combinedObject.getLocation(), tempFile);
return result;
} catch (IOException e) {
throw Throwables.propagate(e);
} finally {
if (tempFile != null) {
tempFile.delete();
}
}
}
@Override
public StoredObject putObject(URI location, File source) {
try {
log.info("starting upload: %s", location);
PutObjectResult result =
s3Service.putObject(getS3Bucket(location), getS3ObjectKey(location), source);
ObjectMetadata metadata =
s3Service.getObjectMetadata(getS3Bucket(location), getS3ObjectKey(location));
if (!result.getETag().equals(metadata.getETag())) {
// this might happen in rare cases due to S3's eventual consistency
throw new IllegalStateException("uploaded etag is different from retrieved object etag");
}
log.info("completed upload: %s", location);
return updateStoredObject(location, metadata);
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
public StoredObject getObjectDetails(URI target) {
StoredObject storedObject = new StoredObject(target);
ObjectMetadata metadata =
s3Service.getObjectMetadata(getS3Bucket(target), getS3ObjectKey(target));
return updateStoredObject(storedObject.getLocation(), metadata);
}
public InputSupplier<InputStream> getInputSupplier(URI target) {
return new S3InputSupplier(s3Service, target);
}
}
public class Announcer {
private static final Logger log = Logger.get(Announcer.class);
private final ConcurrentMap<UUID, ServiceAnnouncement> announcements = new MapMaker().makeMap();
private final DiscoveryAnnouncementClient announcementClient;
private final ScheduledExecutorService executor;
private final AtomicBoolean started = new AtomicBoolean(false);
private final ExponentialBackOff errorBackOff =
new ExponentialBackOff(
new Duration(1, MILLISECONDS),
new Duration(1, SECONDS),
"Discovery server connect succeeded for announce",
"Cannot connect to discovery server for announce",
log);
@Inject
public Announcer(
DiscoveryAnnouncementClient announcementClient,
Set<ServiceAnnouncement> serviceAnnouncements) {
checkNotNull(announcementClient, "client is null");
checkNotNull(serviceAnnouncements, "serviceAnnouncements is null");
this.announcementClient = announcementClient;
for (ServiceAnnouncement serviceAnnouncement : serviceAnnouncements) {
announcements.put(serviceAnnouncement.getId(), serviceAnnouncement);
}
executor = new ScheduledThreadPoolExecutor(5, daemonThreadsNamed("Announcer-%s"));
}
public void start() {
Preconditions.checkState(!executor.isShutdown(), "Announcer has been destroyed");
if (started.compareAndSet(false, true)) {
// announce immediately, if discovery is running
announce();
}
}
@PreDestroy
public void destroy() {
executor.shutdownNow();
try {
executor.awaitTermination(30, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
// unannounce
try {
getFutureResult(announcementClient.unannounce(), DiscoveryException.class);
} catch (DiscoveryException e) {
if (e.getCause() instanceof ConnectException) {
log.error(
"Cannot connect to discovery server for unannounce: %s", e.getCause().getMessage());
} else {
log.error(e);
}
}
}
public void addServiceAnnouncement(ServiceAnnouncement serviceAnnouncement) {
checkNotNull(serviceAnnouncement, "serviceAnnouncement is null");
announcements.put(serviceAnnouncement.getId(), serviceAnnouncement);
}
public void removeServiceAnnouncement(UUID serviceId) {
announcements.remove(serviceId);
}
private ListenableFuture<Duration> announce() {
final ListenableFuture<Duration> future =
announcementClient.announce(ImmutableSet.copyOf(announcements.values()));
Futures.addCallback(
future,
new FutureCallback<Duration>() {
@Override
public void onSuccess(Duration duration) {
errorBackOff.success();
// wait 80% of the suggested delay
duration = new Duration(duration.toMillis() * 0.8, MILLISECONDS);
scheduleNextAnnouncement(duration);
}
@Override
public void onFailure(Throwable t) {
Duration duration = errorBackOff.failed(t);
scheduleNextAnnouncement(duration);
}
},
executor);
return future;
}
private void scheduleNextAnnouncement(Duration delay) {
// already stopped? avoids rejection exception
if (executor.isShutdown()) {
return;
}
executor.schedule(
new Runnable() {
@Override
public void run() {
announce();
}
},
delay.toMillis(),
MILLISECONDS);
}
// TODO: move this to a utility package
private static <T, X extends Throwable> T getFutureResult(Future<T> future, Class<X> type)
throws X {
try {
return future.get();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw Throwables.propagate(e);
} catch (ExecutionException e) {
Throwables.propagateIfPossible(e.getCause(), type);
throw Throwables.propagate(e.getCause());
}
}
}
