Beispiel #1
0
public class Main {
  private static final Logger log = Logger.get(Main.class);

  public static void main(String[] args) {
    try {
      Bootstrap app =
          new Bootstrap(
                  new CassandraModule(),
                  new NodeModule(),
                  new HttpServerModule(),
                  new JaxrsModule(),
                  new JsonModule(),
                  new MBeanModule(),
                  new JmxModule(),
                  new DiscoveryModule(),
                  new HttpEventModule())
              .strictConfig();
      app.initialize();
    } catch (Exception e) {
      log.error(e);
      // Cassandra prevents the vm from shutting down on its own
      System.exit(1);
    }
  }
}
public class Share implements Resource {
  public static final String SERVICE_NAME = "SHARE_SERVICE";

  private final Logger log = Logger.get(Share.class);
  private final HttpServiceSelector serviceSelector;

  @Inject
  public Share(@ServiceType(SERVICE_NAME) HttpServiceSelector serviceSelector) {
    Preconditions.checkNotNull(serviceSelector, "Customer serviceSelector must not be null");
    this.serviceSelector = serviceSelector;
  }

  @Override
  public List<URI> getServices() {
    try {
      // TODO: FIXME - This is a hack to get consistent results from discovery
      Thread.sleep(500);
      return serviceSelector.selectHttpService();
    } catch (InterruptedException e) {
      throw new NoSuchElementException(
          SERVICE_NAME + " Service not found - " + serviceSelector.toString());
    }
  }

  @Override
  public URI getService() throws NoSuchElementException {
    List<URI> services = getServices();

    if (services.size() > 0) {
      return services.get(0);
    }
    throw new NoSuchElementException(
        SERVICE_NAME + " Service not found - " + serviceSelector.toString());
  }
}
public class S3StorageSystem {
  private static final Logger log = Logger.get(S3StorageSystem.class);
  private final AmazonS3 s3Service;

  @Inject
  public S3StorageSystem(AmazonS3 s3Service) {
    Preconditions.checkNotNull(s3Service, "s3Service is null");
    this.s3Service = s3Service;
  }

  public List<URI> listDirectoriesNewestFirst(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String bucket = getS3Bucket(storageArea);
    String key = getS3ObjectKey(storageArea);
    Iterator<String> iter =
        new S3PrefixListing(s3Service, new ListObjectsRequest(bucket, key, null, "/", null))
            .iterator();

    ImmutableList.Builder<URI> builder = ImmutableList.builder();
    while (iter.hasNext()) {
      builder.add(buildS3Location("s3://", bucket, iter.next()));
    }
    return builder.build().reverse();
  }

  public List<URI> listObjects(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String s3Path = getS3ObjectKey(storageArea);
    Iterable<S3ObjectSummary> objectListing =
        new S3ObjectListing(
            s3Service, new ListObjectsRequest(getS3Bucket(storageArea), s3Path, null, "/", null));

    ImmutableList.Builder<URI> builder = ImmutableList.builder();
    for (S3ObjectSummary summary : objectListing) {
      builder.add(buildS3Location(storageArea, summary.getKey().substring(s3Path.length())));
    }
    return builder.build();
  }

  public InputSupplier<InputStream> getInputSupplier(URI target) {
    return new S3InputSupplier(s3Service, target);
  }
}
public class S3StorageSystem implements StorageSystem {
  private static final Logger log = Logger.get(S3StorageSystem.class);
  private final AmazonS3 s3Service;

  @Inject
  public S3StorageSystem(AmazonS3 s3Service) {
    Preconditions.checkNotNull(s3Service, "s3Service is null");
    this.s3Service = s3Service;
  }

  @Override
  public List<URI> listDirectories(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String bucket = getS3Bucket(storageArea);
    String key = getS3ObjectKey(storageArea);
    Iterator<String> iter =
        new S3PrefixListing(s3Service, new ListObjectsRequest(bucket, key, null, "/", null))
            .iterator();

    ImmutableList.Builder<URI> builder = ImmutableList.builder();
    while (iter.hasNext()) {
      builder.add(buildS3Location("s3://", bucket, iter.next()));
    }
    return builder.build();
  }

  @Override
  public List<StoredObject> listObjects(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String s3Path = getS3ObjectKey(storageArea);
    Iterator<S3ObjectSummary> iter =
        new S3ObjectListing(
                s3Service,
                new ListObjectsRequest(getS3Bucket(storageArea), s3Path, null, "/", null))
            .iterator();

    ImmutableList.Builder<StoredObject> builder = ImmutableList.builder();
    while (iter.hasNext()) {
      S3ObjectSummary summary = iter.next();
      builder.add(
          new StoredObject(
              buildS3Location(storageArea, summary.getKey().substring(s3Path.length())),
              summary.getETag(),
              summary.getSize(),
              summary.getLastModified().getTime()));
    }
    return builder.build();
  }

  @Override
  public StoredObject createCombinedObject(CombinedStoredObject combinedObject) {
    Preconditions.checkNotNull(combinedObject, "combinedObject is null");
    Preconditions.checkArgument(
        !combinedObject.getSourceParts().isEmpty(), "combinedObject sourceParts is empty");

    boolean setIsSmall = combinedObject.getSourceParts().get(0).getSize() < 5 * 1024 * 1024;

    // verify size
    for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
      boolean fileIsSmall = newCombinedObjectPart.getSize() < 5 * 1024 * 1024;
      Preconditions.checkArgument(
          fileIsSmall == setIsSmall,
          "combinedObject sourceParts contains mixed large and small files");
    }

    return setIsSmall
        ? createCombinedObjectSmall(combinedObject)
        : createCombinedObjectLarge(combinedObject);
  }

  private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) {
    URI location = combinedObject.getLocation();
    log.info("starting multipart upload: %s", location);

    String bucket = getS3Bucket(location);
    String key = getS3ObjectKey(location);

    String uploadId =
        s3Service
            .initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key))
            .getUploadId();

    try {
      List<PartETag> parts = newArrayList();
      int partNumber = 1;
      for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
        CopyPartResult part =
            s3Service.copyPart(
                new CopyPartRequest()
                    .withUploadId(uploadId)
                    .withPartNumber(partNumber)
                    .withDestinationBucketName(bucket)
                    .withDestinationKey(key)
                    .withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation()))
                    .withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation())));
        parts.add(new PartETag(partNumber, part.getETag()));
        partNumber++;
      }

      String etag =
          s3Service
              .completeMultipartUpload(
                  new CompleteMultipartUploadRequest(bucket, key, uploadId, parts))
              .getETag();

      ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key);
      log.info("completed multipart upload: %s", location);

      if (!etag.equals(newObject.getETag())) {
        // this might happen in rare cases due to S3's eventual consistency
        throw new IllegalStateException("completed etag is different from combined object etag");
      }

      return updateStoredObject(location, newObject);
    } catch (AmazonClientException e) {
      try {
        s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
      } catch (AmazonClientException ignored) {
      }
      throw Throwables.propagate(e);
    }
  }

  private StoredObject createCombinedObjectSmall(CombinedStoredObject combinedObject) {
    ImmutableList.Builder<InputSupplier<InputStream>> builder = ImmutableList.builder();
    List<URI> sourceParts =
        Lists.transform(combinedObject.getSourceParts(), StoredObject.GET_LOCATION_FUNCTION);
    for (URI sourcePart : sourceParts) {
      builder.add(getInputSupplier(sourcePart));
    }
    InputSupplier<InputStream> source = ByteStreams.join(builder.build());

    File tempFile = null;
    try {
      tempFile =
          File.createTempFile(
              S3StorageHelper.getS3FileName(combinedObject.getLocation()), ".small.s3.data");
      Files.copy(source, tempFile);
      StoredObject result = putObject(combinedObject.getLocation(), tempFile);
      return result;
    } catch (IOException e) {
      throw Throwables.propagate(e);
    } finally {
      if (tempFile != null) {
        tempFile.delete();
      }
    }
  }

  @Override
  public StoredObject putObject(URI location, File source) {
    try {
      log.info("starting upload: %s", location);
      PutObjectResult result =
          s3Service.putObject(getS3Bucket(location), getS3ObjectKey(location), source);
      ObjectMetadata metadata =
          s3Service.getObjectMetadata(getS3Bucket(location), getS3ObjectKey(location));
      if (!result.getETag().equals(metadata.getETag())) {
        // this might happen in rare cases due to S3's eventual consistency
        throw new IllegalStateException("uploaded etag is different from retrieved object etag");
      }
      log.info("completed upload: %s", location);
      return updateStoredObject(location, metadata);
    } catch (Exception e) {
      throw Throwables.propagate(e);
    }
  }

  public StoredObject getObjectDetails(URI target) {
    StoredObject storedObject = new StoredObject(target);
    ObjectMetadata metadata =
        s3Service.getObjectMetadata(getS3Bucket(target), getS3ObjectKey(target));
    return updateStoredObject(storedObject.getLocation(), metadata);
  }

  public InputSupplier<InputStream> getInputSupplier(URI target) {
    return new S3InputSupplier(s3Service, target);
  }
}
Beispiel #5
0
public class Announcer {
  private static final Logger log = Logger.get(Announcer.class);
  private final ConcurrentMap<UUID, ServiceAnnouncement> announcements = new MapMaker().makeMap();

  private final DiscoveryAnnouncementClient announcementClient;
  private final ScheduledExecutorService executor;
  private final AtomicBoolean started = new AtomicBoolean(false);

  private final ExponentialBackOff errorBackOff =
      new ExponentialBackOff(
          new Duration(1, MILLISECONDS),
          new Duration(1, SECONDS),
          "Discovery server connect succeeded for announce",
          "Cannot connect to discovery server for announce",
          log);

  @Inject
  public Announcer(
      DiscoveryAnnouncementClient announcementClient,
      Set<ServiceAnnouncement> serviceAnnouncements) {
    checkNotNull(announcementClient, "client is null");
    checkNotNull(serviceAnnouncements, "serviceAnnouncements is null");

    this.announcementClient = announcementClient;
    for (ServiceAnnouncement serviceAnnouncement : serviceAnnouncements) {
      announcements.put(serviceAnnouncement.getId(), serviceAnnouncement);
    }
    executor = new ScheduledThreadPoolExecutor(5, daemonThreadsNamed("Announcer-%s"));
  }

  public void start() {
    Preconditions.checkState(!executor.isShutdown(), "Announcer has been destroyed");
    if (started.compareAndSet(false, true)) {
      // announce immediately, if discovery is running
      announce();
    }
  }

  @PreDestroy
  public void destroy() {
    executor.shutdownNow();
    try {
      executor.awaitTermination(30, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    }

    // unannounce
    try {
      getFutureResult(announcementClient.unannounce(), DiscoveryException.class);
    } catch (DiscoveryException e) {
      if (e.getCause() instanceof ConnectException) {
        log.error(
            "Cannot connect to discovery server for unannounce: %s", e.getCause().getMessage());
      } else {
        log.error(e);
      }
    }
  }

  public void addServiceAnnouncement(ServiceAnnouncement serviceAnnouncement) {
    checkNotNull(serviceAnnouncement, "serviceAnnouncement is null");
    announcements.put(serviceAnnouncement.getId(), serviceAnnouncement);
  }

  public void removeServiceAnnouncement(UUID serviceId) {
    announcements.remove(serviceId);
  }

  private ListenableFuture<Duration> announce() {
    final ListenableFuture<Duration> future =
        announcementClient.announce(ImmutableSet.copyOf(announcements.values()));

    Futures.addCallback(
        future,
        new FutureCallback<Duration>() {
          @Override
          public void onSuccess(Duration duration) {
            errorBackOff.success();

            // wait 80% of the suggested delay
            duration = new Duration(duration.toMillis() * 0.8, MILLISECONDS);
            scheduleNextAnnouncement(duration);
          }

          @Override
          public void onFailure(Throwable t) {
            Duration duration = errorBackOff.failed(t);
            scheduleNextAnnouncement(duration);
          }
        },
        executor);

    return future;
  }

  private void scheduleNextAnnouncement(Duration delay) {
    // already stopped?  avoids rejection exception
    if (executor.isShutdown()) {
      return;
    }
    executor.schedule(
        new Runnable() {
          @Override
          public void run() {
            announce();
          }
        },
        delay.toMillis(),
        MILLISECONDS);
  }

  // TODO: move this to a utility package
  private static <T, X extends Throwable> T getFutureResult(Future<T> future, Class<X> type)
      throws X {
    try {
      return future.get();
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
      throw Throwables.propagate(e);
    } catch (ExecutionException e) {
      Throwables.propagateIfPossible(e.getCause(), type);
      throw Throwables.propagate(e.getCause());
    }
  }
}