コード例 #1
0
  private static void createResources() {
    ClasspathPropertiesFileCredentialsProvider provider =
        new ClasspathPropertiesFileCredentialsProvider();

    AmazonSQS sqs = new AmazonSQSClient(provider);
    sqs.setEndpoint(SQS_ENDPOINT);
    sqs.createQueue(new CreateQueueRequest().withQueueName(SQS_QUEUE));

    AmazonS3 s3 = new AmazonS3Client(provider);
    if (!s3.doesBucketExist(BUCKET_NAME)) {
      s3.createBucket(new CreateBucketRequest(BUCKET_NAME));
    }

    AmazonDynamoDBClient dynamo = new AmazonDynamoDBClient(provider);
    dynamo.setEndpoint(DYNAMO_ENDPOINT);

    if (!doesTableExist(dynamo, DYNAMO_TABLE_NAME)) {
      dynamo.createTable(
          new CreateTableRequest()
              .withTableName(DYNAMO_TABLE_NAME)
              .withProvisionedThroughput(
                  new ProvisionedThroughput()
                      .withReadCapacityUnits(50l)
                      .withWriteCapacityUnits(50l))
              .withKeySchema(
                  new KeySchema()
                      .withHashKeyElement(
                          new KeySchemaElement()
                              .withAttributeName("id")
                              .withAttributeType(ScalarAttributeType.S))));
      waitForTableToBecomeAvailable(dynamo, DYNAMO_TABLE_NAME);
    }
  }
コード例 #2
0
ファイル: UploadToS3.java プロジェクト: jq/zhong
  public static boolean upload(MusicInfo music) {
    String key = "";
    try {
      // upload  mp3 ring
      key = music.getUUID() + music.getRingName();
      File file = new File(Consts.NEW_DOWNLOAD_DIR + music.getRingName());
      s3.putObject(new PutObjectRequest(Consts.AMAZON_RING_BUCKET, key, file)); // upload ring
      s3.setObjectAcl(
          Consts.AMAZON_RING_BUCKET, key, CannedAccessControlList.PublicRead); // set access

      // upload m4r ring
      //			String m4r = music.getRingName().replace(".mp3", ".m4r");
      //			key = music.getUUID() + m4r;
      //			file = new File(Consts.NEW_DOWNLOAD_DIR+m4r);
      //			s3.putObject(new PutObjectRequest(Consts.AMAZON_M4R_BUCKET, key, file));      			//
      // upload ring
      //			s3.setObjectAcl(Consts.AMAZON_M4R_BUCKET, key, CannedAccessControlList.PublicRead);		//
      // set access

      // upload image
      key = music.getUUID() + music.getImageName();
      file = new File(Consts.NEW_DOWNLOAD_DIR + music.getImageName());
      s3.putObject(new PutObjectRequest(Consts.AMAZON_IMAGE_BUCKET, key, file)); // upload image
      s3.setObjectAcl(
          Consts.AMAZON_IMAGE_BUCKET, key, CannedAccessControlList.PublicRead); // set access

      return true;
    } catch (Exception e) {
      System.out.println("upload to S3 err");
      e.printStackTrace();
      return false;
    }
  }
コード例 #3
0
 public void deleteAllFiles(String folderName) {
   List<S3ObjectSummary> fileList =
       s3client.listObjects(bucketName, folderName).getObjectSummaries();
   for (S3ObjectSummary file : fileList) {
     s3client.deleteObject(bucketName, file.getKey());
   }
 }
コード例 #4
0
ファイル: S3Util.java プロジェクト: JialiDu/zork
  /**
   * get the maps
   *
   * @return List<String> list of name of maps
   */
  public List<String> getMapsList(String userName) {
    List<String> maps = new ArrayList<String>();
    ListObjectsRequest listObjectsRequest =
        new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix("maps/public/")
            .withDelimiter("/");
    ObjectListing objects = s3.listObjects(listObjectsRequest);

    for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
      if (!objectSummary.getKey().equals("maps/public")) {
        System.out.println("get map: " + objectSummary.getKey());
        maps.add(objectSummary.getKey());
      }
    }

    ListObjectsRequest listObjectsRequest2 =
        new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix("maps/" + userName + "/")
            .withDelimiter("/");
    ObjectListing objects2 = s3.listObjects(listObjectsRequest);

    for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
      if (!objectSummary.getKey().equals("maps/" + userName)) {
        System.out.println("get map: " + objectSummary.getKey());
        maps.add(objectSummary.getKey());
      }
    }

    return maps;
  }
コード例 #5
0
  private static void basicAbortMPU() throws IOException {
    System.out.println("basic abort MPU");
    String bucketName = "chttest";
    String fileName = "hello.txt";
    // String uploadID = "XHGTFV4F5XTEAC5O8N3LK12TIY3DSY7OFPXIWTHRMNTE7A3WB5M8N2U5AN"; //hi
    String uploadID = "LE5JS2K6C208JU7ZX1QD2TVRWXOWWF4VNG7LE7TFIX5SYNG4HLOGW9CLAD"; // hello

    AbortMultipartUploadRequest request =
        new AbortMultipartUploadRequest(bucketName, fileName, uploadID);

    AmazonS3 s3 =
        new AmazonS3Client(
            new PropertiesCredentials(
                putBucket.class.getResourceAsStream("AwsCredentials.properties")));
    try {
      s3.abortMultipartUpload(request);
      System.out.println();
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon S3, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with S3, "
              + "such as not being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
コード例 #6
0
ファイル: ImportS3.java プロジェクト: pragnesh/h2o
 @Override
 protected String parse(String input) throws IllegalArgumentException {
   AmazonS3 s3 = PersistS3.getClient();
   if (!s3.doesBucketExist(input))
     throw new IllegalArgumentException("S3 Bucket " + input + " not found!");
   return input;
 }
  private File retrieveS3File(String sqsdMessageBody) throws UnsupportedEncodingException {
    File localFile = null;

    if (!sqsdMessageBody.isEmpty()) {

      AmazonS3 s3 = new AmazonS3Client();

      List<S3EventNotificationRecord> records =
          S3EventNotification.parseJson(sqsdMessageBody).getRecords();

      S3EventNotificationRecord firstRecord = records.get(0);

      String bucketName = firstRecord.getS3().getBucket().getName();

      String objectRegion = firstRecord.getAwsRegion();
      Region s3Region = Region.getRegion(Regions.fromName(objectRegion));
      s3.setRegion(s3Region);

      // Object key may have spaces or unicode non-ASCII characters.
      String keyName = firstRecord.getS3().getObject().getKey().replace('+', ' ');
      keyName = URLDecoder.decode(keyName, "UTF-8");

      localFile = new File(keyName);

      System.out.println("Downloading file: " + objectRegion + "/" + bucketName + "/" + keyName);
      s3.getObject(new GetObjectRequest(bucketName, keyName), localFile);

      if (!localFile.canRead()) {
        localFile = null;
      }
    }
    return localFile;
  }
コード例 #8
0
  @Override
  public F.Promise<Result> getDownload(String key, String name) {
    GeneratePresignedUrlRequest generatePresignedUrlRequest =
        new GeneratePresignedUrlRequest(bucketName, key);
    ResponseHeaderOverrides responseHeaders = new ResponseHeaderOverrides();
    responseHeaders.setContentDisposition("attachment; filename=" + name);
    generatePresignedUrlRequest.setResponseHeaders(responseHeaders);

    AmazonS3 amazonS3 = new AmazonS3Client(credentials);

    try {
      URL url = amazonS3.generatePresignedUrl(generatePresignedUrlRequest);

      return F.Promise.pure(redirect(url.toString()));
    } catch (AmazonClientException ace) {
      Logger.error(
          "Caught an AmazonClientException, which "
              + "means the client encountered "
              + "an internal error while trying to "
              + "communicate with S3, "
              + "such as not being able to access the network."
              + " Error Message: "
              + ace.getMessage());
      return F.Promise.pure(internalServerError("Download failed"));
    }
  }
コード例 #9
0
  @Override
  public void saveFile(File file, String fileName) throws IOException {

    AmazonS3 s3Client = getAmzS3Client();

    s3Client.putObject(new PutObjectRequest(bucketName, fileName, file));

    file.delete();
  }
コード例 #10
0
  private AmazonS3 getAmzS3Client() {

    BasicAWSCredentials awsCreds = new BasicAWSCredentials(accessKeyId, secretAccessKey);
    AmazonS3 s3Client = new AmazonS3Client(awsCreds);
    Region region = Region.getRegion(Regions.EU_WEST_1);
    s3Client.setRegion(region);

    return s3Client;
  }
コード例 #11
0
ファイル: EC2Cloud.java プロジェクト: ryanaslett/ec2-plugin
 /**
  * Computes the presigned URL for the given S3 resource.
  *
  * @param path String like "/bucketName/folder/folder/abc.txt" that represents the resource to
  *     request.
  */
 public URL buildPresignedURL(String path) throws AmazonClientException {
   AWSCredentials credentials = awsCredentialsProvider.getCredentials();
   long expires = System.currentTimeMillis() + 60 * 60 * 1000;
   GeneratePresignedUrlRequest request =
       new GeneratePresignedUrlRequest(path, credentials.getAWSSecretKey());
   request.setExpiration(new Date(expires));
   AmazonS3 s3 = new AmazonS3Client(credentials);
   return s3.generatePresignedUrl(request);
 }
コード例 #12
0
  /*

  @author: Nitish

  */
  public static String UploadToS3() throws IOException {
    String existingBucketName = "s3-bucket-zerorpc";
    String keyName = "image.jpeg";
    String filePath = "C:\\Users\\Nitish\\Desktop\\IMG_2695.JPG";

    AmazonS3 s3Client = new AmazonS3Client(new BasicAWSCredentials("", ""));

    // Create a list of UploadPartResponse objects. You get one of these
    // for each part upload.
    List<PartETag> partETags = new ArrayList<PartETag>();

    // Step 1: Initialize.
    InitiateMultipartUploadRequest initRequest =
        new InitiateMultipartUploadRequest(existingBucketName, keyName);
    InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);

    File file = new File(filePath);
    long contentLength = file.length();
    long partSize = 5242880; // Set part size to 5 MB.

    try {
      // Step 2: Upload parts.
      long filePosition = 0;
      for (int i = 1; filePosition < contentLength; i++) {
        // Last part can be less than 5 MB. Adjust part size.
        partSize = Math.min(partSize, (contentLength - filePosition));

        // Create request to upload a part.
        UploadPartRequest uploadRequest =
            new UploadPartRequest()
                .withBucketName(existingBucketName)
                .withKey(keyName)
                .withUploadId(initResponse.getUploadId())
                .withPartNumber(i)
                .withFileOffset(filePosition)
                .withFile(file)
                .withPartSize(partSize);

        // Upload part and add response to our list.
        partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());

        filePosition += partSize;
      }

      // Step 3: Complete.
      CompleteMultipartUploadRequest compRequest =
          new CompleteMultipartUploadRequest(
              existingBucketName, keyName, initResponse.getUploadId(), partETags);

      s3Client.completeMultipartUpload(compRequest);

    } catch (Exception e) {
      s3Client.abortMultipartUpload(
          new AbortMultipartUploadRequest(existingBucketName, keyName, initResponse.getUploadId()));
    }
    return "Uploaded successfully";
  }
コード例 #13
0
ファイル: AwsBucketTest.java プロジェクト: jcabi/jcabi-s3
 /**
  * AwsBucket can throw a proper exception.
  *
  * @throws IOException If succeeds
  */
 @Test(expected = IOException.class)
 public void existsThrowsIOException() throws IOException {
   final Region region = Mockito.mock(Region.class);
   final AmazonS3 aws = Mockito.mock(AmazonS3.class);
   Mockito.when(region.aws()).thenReturn(aws);
   final String name = "throwing.bucket.com";
   Mockito.when(aws.doesBucketExist(name)).thenThrow(new AmazonServiceException("Test exception"));
   final Bucket bucket = new AwsBucket(region, name);
   bucket.exists();
 }
コード例 #14
0
ファイル: AwsBucketTest.java プロジェクト: jcabi/jcabi-s3
 /**
  * AwsBucket can correctly check the existence of the non-existing bucket.
  *
  * @throws IOException If fails
  */
 @Test
 public void existsNonExistingBucket() throws IOException {
   final Region region = Mockito.mock(Region.class);
   final AmazonS3 aws = Mockito.mock(AmazonS3.class);
   Mockito.when(region.aws()).thenReturn(aws);
   final String name = "non.existing.bucket.com";
   Mockito.when(aws.doesBucketExist(name)).thenReturn(false);
   final Bucket bucket = new AwsBucket(region, name);
   Assert.assertFalse(bucket.exists());
 }
コード例 #15
0
ファイル: S3Storage.java プロジェクト: carriercomm/cosbench
  @Override
  public void createContainer(String container, Config config) {
    super.createContainer(container, config);
    try {
      if (!client.doesBucketExist(container)) {

        client.createBucket(container);
      }
    } catch (Exception e) {
      throw new StorageException(e);
    }
  }
コード例 #16
0
  /** Data cleansing method */
  public void cleanseData(AmazonS3 client) throws Exception {

    AwsDataLoader loader = new AwsDataLoader();
    CSVReader reader = null;
    String prefix = loader.getS3Prefix(source);
    client.setEndpoint(S3_ENDPOINT);
    S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix));
    reader =
        new CSVReader(
            new BufferedReader(new InputStreamReader(object.getObjectContent())),
            CSVParser.DEFAULT_SEPARATOR,
            CSVParser.DEFAULT_QUOTE_CHARACTER,
            CSVParser.DEFAULT_ESCAPE_CHARACTER,
            HEADERS_LINE);
    ColumnPositionMappingStrategy<ProductLanguage> strat =
        new ColumnPositionMappingStrategy<ProductLanguage>();
    strat.setType(ProductLanguage.class);
    String[] columns =
        new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"};
    strat.setColumnMapping(columns);
    CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>();
    list = csv.parse(strat, reader);

    System.out.println("ProductLanguageCleanser input size: " + list.size());

    this.updateDataset(list);

    BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>();
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"');
    // strat.setColumnMapping(columns);
    log.info("List size: " + list.size());
    csvWriter.write(strat, writer, list);
    writer.flush();
    String dataset = os.toString();
    String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv";

    client.setEndpoint(S3_ENDPOINT);
    ObjectMetadata omd = new ObjectMetadata();

    try {
      byte[] content = dataset.getBytes(StandardCharsets.UTF_8);
      ByteArrayInputStream input = new ByteArrayInputStream(content);
      BufferedReader buffReader = new BufferedReader(new InputStreamReader(input));
      buffReader.readLine();
      InputStream inputObj = new ReaderInputStream(buffReader);
      // omd.setContentLength(IOUtils.toByteArray(input).length);
      client.putObject(BUCKET_NM, outPrefix, inputObj, omd);
      input.close();
    } catch (IOException e) {
      log.error("Axon data write to s3 failed: " + e.getMessage());
    }
  }
コード例 #17
0
  private void uploadToS3(File file, String bucketName, String key) {
    log.info("Uploading local WAR file {} to remote WAR file: {}", file.getName(), key);

    try {
      String s3key = URLEncoder.encode(key, "UTF-8");
      AmazonS3 s3 = new AmazonS3Client(getAwsCredentials());
      s3.putObject(bucketName, s3key, file);
    } catch (UnsupportedEncodingException ex) {
      log.error("Upload file to s3 failed.");
      log.error(ex.getMessage());
    }
  }
コード例 #18
0
  private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) {
    URI location = combinedObject.getLocation();
    log.info("starting multipart upload: %s", location);

    String bucket = getS3Bucket(location);
    String key = getS3ObjectKey(location);

    String uploadId =
        s3Service
            .initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key))
            .getUploadId();

    try {
      List<PartETag> parts = newArrayList();
      int partNumber = 1;
      for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
        CopyPartResult part =
            s3Service.copyPart(
                new CopyPartRequest()
                    .withUploadId(uploadId)
                    .withPartNumber(partNumber)
                    .withDestinationBucketName(bucket)
                    .withDestinationKey(key)
                    .withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation()))
                    .withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation())));
        parts.add(new PartETag(partNumber, part.getETag()));
        partNumber++;
      }

      String etag =
          s3Service
              .completeMultipartUpload(
                  new CompleteMultipartUploadRequest(bucket, key, uploadId, parts))
              .getETag();

      ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key);
      log.info("completed multipart upload: %s", location);

      if (!etag.equals(newObject.getETag())) {
        // this might happen in rare cases due to S3's eventual consistency
        throw new IllegalStateException("completed etag is different from combined object etag");
      }

      return updateStoredObject(location, newObject);
    } catch (AmazonClientException e) {
      try {
        s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
      } catch (AmazonClientException ignored) {
      }
      throw Throwables.propagate(e);
    }
  }
コード例 #19
0
 @Override
 protected JsonArray serve(String filter, int limit) {
   JsonArray array = new JsonArray();
   try {
     AmazonS3 s3 = PersistS3.getClient();
     filter = Strings.nullToEmpty(filter);
     for( Bucket b : s3.listBuckets() ) {
       if( b.getName().startsWith(filter) )
         array.add(new JsonPrimitive(b.getName()));
       if( array.size() == limit) break;
     }
   } catch( IllegalArgumentException _ ) { }
   return array;
 }
コード例 #20
0
ファイル: S3Storage.java プロジェクト: carriercomm/cosbench
 @Override
 public void deleteContainer(String container, Config config) {
   super.deleteContainer(container, config);
   try {
     if (client.doesBucketExist(container)) {
       client.deleteBucket(container);
     }
   } catch (AmazonS3Exception awse) {
     if (awse.getStatusCode() != HttpStatus.SC_NOT_FOUND) {
       throw new StorageException(awse);
     }
   } catch (Exception e) {
     throw new StorageException(e);
   }
 }
コード例 #21
0
ファイル: ceph.java プロジェクト: XiaoqingWang/MovieSite
 // 列出某个bucket的所有objects
 public static void listObjectInBucket(Bucket bucket) {
   List<Bucket> buckets = conn.listBuckets();
   ObjectListing objects = conn.listObjects(bucket.getName());
   do {
     for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
       System.out.println(
           objectSummary.getKey()
               + "\t"
               + objectSummary.getSize()
               + "\t"
               + StringUtils.fromDate(objectSummary.getLastModified()));
     }
     objects = conn.listNextBatchOfObjects(objects);
   } while (objects.isTruncated());
 }
コード例 #22
0
 public InputStream downloadRange(long start, long end, String file) {
   GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, file);
   rangeObjectRequest.setRange(start, end); // retrieve 1st 10 bytes.
   S3Object objectPortion = s3client.getObject(rangeObjectRequest);
   InputStream objectData = objectPortion.getObjectContent();
   return objectData;
 }
コード例 #23
0
 private ObjectMetadata getS3ObjectMetadata(final Path path) throws IOException {
   try {
     return retry()
         .maxAttempts(maxClientRetries)
         .exponentialBackoff(new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0)
         .stopOn(InterruptedException.class, UnrecoverableS3OperationException.class)
         .run(
             "getS3ObjectMetadata",
             () -> {
               try {
                 return s3.getObjectMetadata(uri.getHost(), keyFromPath(path));
               } catch (AmazonS3Exception e) {
                 if (e.getStatusCode() == SC_NOT_FOUND) {
                   return null;
                 } else if (e.getStatusCode() == SC_FORBIDDEN) {
                   throw new UnrecoverableS3OperationException(e);
                 }
                 throw Throwables.propagate(e);
               }
             });
   } catch (InterruptedException e) {
     Thread.currentThread().interrupt();
     throw Throwables.propagate(e);
   } catch (Exception e) {
     Throwables.propagateIfInstanceOf(e, IOException.class);
     throw Throwables.propagate(e);
   }
 }
コード例 #24
0
 private S3Object getS3Object(final Path path, final long start) throws IOException {
   try {
     return retry()
         .maxAttempts(maxClientRetry)
         .exponentialBackoff(
             new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0)
         .stopOn(InterruptedException.class, UnrecoverableS3OperationException.class)
         .run(
             "getS3Object",
             () -> {
               try {
                 return s3.getObject(
                     new GetObjectRequest(host, keyFromPath(path))
                         .withRange(start, Long.MAX_VALUE));
               } catch (AmazonServiceException e) {
                 if (e.getStatusCode() == SC_FORBIDDEN) {
                   throw new UnrecoverableS3OperationException(e);
                 }
                 throw Throwables.propagate(e);
               }
             });
   } catch (InterruptedException e) {
     Thread.currentThread().interrupt();
     throw Throwables.propagate(e);
   } catch (Exception e) {
     Throwables.propagateIfInstanceOf(e, IOException.class);
     throw Throwables.propagate(e);
   }
 }
コード例 #25
0
 public UrlList getUrlList(String url) {
   UrlList urlList = null;
   try {
     String key = Hash.hashKey(url);
     GetObjectRequest req = new GetObjectRequest(bucketName, key);
     S3Object s3Object = s3client.getObject(req);
     InputStream objectData = s3Object.getObjectContent();
     BufferedReader reader = new BufferedReader(new InputStreamReader(objectData));
     StringBuilder s3Content = new StringBuilder();
     String line;
     while ((line = reader.readLine()) != null) {
       s3Content.append(line + "\r\n");
     }
     reader.close();
     objectData.close();
     ObjectMapper mapper = new ObjectMapper();
     mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE);
     mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
     urlList = mapper.readValue(s3Content.toString(), UrlList.class);
   } catch (AmazonS3Exception ase) {
     System.out.println("S3UrlListDA : document does not exist");
     ase.printStackTrace();
   } catch (IOException e) {
     System.out.println("S3UrlListDA : IOException while fetching document from S3");
     e.printStackTrace();
   } catch (NoSuchAlgorithmException e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
   }
   return urlList;
 }
コード例 #26
0
ファイル: ceph.java プロジェクト: XiaoqingWang/MovieSite
 // 获取名为name的bucket
 public static Bucket getBucketByName(String name) {
   List<Bucket> buckets = conn.listBuckets();
   for (Bucket bucket : buckets) {
     if (bucket.getName().equalsIgnoreCase(name)) return bucket;
   }
   return null;
 }
コード例 #27
0
ファイル: ceph.java プロジェクト: XiaoqingWang/MovieSite
 // 列出所有buckets
 public static void listAllBuckets() {
   List<Bucket> buckets = conn.listBuckets();
   for (Bucket bucket : buckets) {
     System.out.println(
         bucket.getName() + "\t\t" + StringUtils.fromDate(bucket.getCreationDate()));
   }
 }
コード例 #28
0
ファイル: ceph.java プロジェクト: XiaoqingWang/MovieSite
 // 列出有prefix前缀的所有objects
 public static void listObjectInBucketWithPrefix(Bucket bucket, String prefix) {
   ListObjectsRequest listObjectsRequest =
       new ListObjectsRequest().withBucketName(bucket.getName()).withPrefix(prefix);
   ObjectListing objects = conn.listObjects(listObjectsRequest);
   do {
     for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
       System.out.println(
           objectSummary.getKey()
               + "\t"
               + objectSummary.getSize()
               + "\t"
               + StringUtils.fromDate(objectSummary.getLastModified()));
     }
     objects = conn.listNextBatchOfObjects(objects);
   } while (objects.isTruncated());
 }
コード例 #29
0
  private static void saveToS3(
      String bucket,
      String key,
      String imageType,
      AmazonS3 s3Client,
      String dstKey,
      BufferedImage resizedImage)
      throws IOException {
    // Re-encode image to target format
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    ImageIO.write(resizedImage, imageType, os);
    InputStream is = new ByteArrayInputStream(os.toByteArray());
    // Set Content-Length and Content-Type
    ObjectMetadata meta = new ObjectMetadata();
    meta.setContentLength(os.size());
    String contentType = imageTypeMime.get(imageType);
    if (contentType == null) {
      throw new RuntimeException(
          String.format("Unknown content type for image type %s", imageType));
    }
    meta.setContentType(contentType);

    // Uploading to S3 destination bucket
    System.out.println("Writing to: " + bucket + "/" + dstKey);
    s3Client.putObject(bucket, dstKey, is, meta);
    System.out.println(
        "Successfully resized " + bucket + "/" + key + " and uploaded to " + bucket + "/" + dstKey);
  }
コード例 #30
0
ファイル: S3NotebookRepo.java プロジェクト: conker84/zeppelin
  private Note getNote(String key) throws IOException {
    GsonBuilder gsonBuilder = new GsonBuilder();
    gsonBuilder.setPrettyPrinting();
    Gson gson =
        gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create();

    S3Object s3object;
    try {
      s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
    } catch (AmazonClientException ace) {
      throw new IOException("Unable to retrieve object from S3: " + ace, ace);
    }

    Note note;
    try (InputStream ins = s3object.getObjectContent()) {
      String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
      note = gson.fromJson(json, Note.class);
    }

    for (Paragraph p : note.getParagraphs()) {
      if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
        p.setStatus(Status.ABORT);
      }
    }

    return note;
  }