private static void createResources() { ClasspathPropertiesFileCredentialsProvider provider = new ClasspathPropertiesFileCredentialsProvider(); AmazonSQS sqs = new AmazonSQSClient(provider); sqs.setEndpoint(SQS_ENDPOINT); sqs.createQueue(new CreateQueueRequest().withQueueName(SQS_QUEUE)); AmazonS3 s3 = new AmazonS3Client(provider); if (!s3.doesBucketExist(BUCKET_NAME)) { s3.createBucket(new CreateBucketRequest(BUCKET_NAME)); } AmazonDynamoDBClient dynamo = new AmazonDynamoDBClient(provider); dynamo.setEndpoint(DYNAMO_ENDPOINT); if (!doesTableExist(dynamo, DYNAMO_TABLE_NAME)) { dynamo.createTable( new CreateTableRequest() .withTableName(DYNAMO_TABLE_NAME) .withProvisionedThroughput( new ProvisionedThroughput() .withReadCapacityUnits(50l) .withWriteCapacityUnits(50l)) .withKeySchema( new KeySchema() .withHashKeyElement( new KeySchemaElement() .withAttributeName("id") .withAttributeType(ScalarAttributeType.S)))); waitForTableToBecomeAvailable(dynamo, DYNAMO_TABLE_NAME); } }
public static boolean upload(MusicInfo music) { String key = ""; try { // upload mp3 ring key = music.getUUID() + music.getRingName(); File file = new File(Consts.NEW_DOWNLOAD_DIR + music.getRingName()); s3.putObject(new PutObjectRequest(Consts.AMAZON_RING_BUCKET, key, file)); // upload ring s3.setObjectAcl( Consts.AMAZON_RING_BUCKET, key, CannedAccessControlList.PublicRead); // set access // upload m4r ring // String m4r = music.getRingName().replace(".mp3", ".m4r"); // key = music.getUUID() + m4r; // file = new File(Consts.NEW_DOWNLOAD_DIR+m4r); // s3.putObject(new PutObjectRequest(Consts.AMAZON_M4R_BUCKET, key, file)); // // upload ring // s3.setObjectAcl(Consts.AMAZON_M4R_BUCKET, key, CannedAccessControlList.PublicRead); // // set access // upload image key = music.getUUID() + music.getImageName(); file = new File(Consts.NEW_DOWNLOAD_DIR + music.getImageName()); s3.putObject(new PutObjectRequest(Consts.AMAZON_IMAGE_BUCKET, key, file)); // upload image s3.setObjectAcl( Consts.AMAZON_IMAGE_BUCKET, key, CannedAccessControlList.PublicRead); // set access return true; } catch (Exception e) { System.out.println("upload to S3 err"); e.printStackTrace(); return false; } }
public void deleteAllFiles(String folderName) { List<S3ObjectSummary> fileList = s3client.listObjects(bucketName, folderName).getObjectSummaries(); for (S3ObjectSummary file : fileList) { s3client.deleteObject(bucketName, file.getKey()); } }
/** * get the maps * * @return List<String> list of name of maps */ public List<String> getMapsList(String userName) { List<String> maps = new ArrayList<String>(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix("maps/public/") .withDelimiter("/"); ObjectListing objects = s3.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { if (!objectSummary.getKey().equals("maps/public")) { System.out.println("get map: " + objectSummary.getKey()); maps.add(objectSummary.getKey()); } } ListObjectsRequest listObjectsRequest2 = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix("maps/" + userName + "/") .withDelimiter("/"); ObjectListing objects2 = s3.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { if (!objectSummary.getKey().equals("maps/" + userName)) { System.out.println("get map: " + objectSummary.getKey()); maps.add(objectSummary.getKey()); } } return maps; }
private static void basicAbortMPU() throws IOException { System.out.println("basic abort MPU"); String bucketName = "chttest"; String fileName = "hello.txt"; // String uploadID = "XHGTFV4F5XTEAC5O8N3LK12TIY3DSY7OFPXIWTHRMNTE7A3WB5M8N2U5AN"; //hi String uploadID = "LE5JS2K6C208JU7ZX1QD2TVRWXOWWF4VNG7LE7TFIX5SYNG4HLOGW9CLAD"; // hello AbortMultipartUploadRequest request = new AbortMultipartUploadRequest(bucketName, fileName, uploadID); AmazonS3 s3 = new AmazonS3Client( new PropertiesCredentials( putBucket.class.getResourceAsStream("AwsCredentials.properties"))); try { s3.abortMultipartUpload(request); System.out.println(); } catch (AmazonServiceException ase) { System.out.println( "Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println( "Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
@Override protected String parse(String input) throws IllegalArgumentException { AmazonS3 s3 = PersistS3.getClient(); if (!s3.doesBucketExist(input)) throw new IllegalArgumentException("S3 Bucket " + input + " not found!"); return input; }
private File retrieveS3File(String sqsdMessageBody) throws UnsupportedEncodingException { File localFile = null; if (!sqsdMessageBody.isEmpty()) { AmazonS3 s3 = new AmazonS3Client(); List<S3EventNotificationRecord> records = S3EventNotification.parseJson(sqsdMessageBody).getRecords(); S3EventNotificationRecord firstRecord = records.get(0); String bucketName = firstRecord.getS3().getBucket().getName(); String objectRegion = firstRecord.getAwsRegion(); Region s3Region = Region.getRegion(Regions.fromName(objectRegion)); s3.setRegion(s3Region); // Object key may have spaces or unicode non-ASCII characters. String keyName = firstRecord.getS3().getObject().getKey().replace('+', ' '); keyName = URLDecoder.decode(keyName, "UTF-8"); localFile = new File(keyName); System.out.println("Downloading file: " + objectRegion + "/" + bucketName + "/" + keyName); s3.getObject(new GetObjectRequest(bucketName, keyName), localFile); if (!localFile.canRead()) { localFile = null; } } return localFile; }
@Override public F.Promise<Result> getDownload(String key, String name) { GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key); ResponseHeaderOverrides responseHeaders = new ResponseHeaderOverrides(); responseHeaders.setContentDisposition("attachment; filename=" + name); generatePresignedUrlRequest.setResponseHeaders(responseHeaders); AmazonS3 amazonS3 = new AmazonS3Client(credentials); try { URL url = amazonS3.generatePresignedUrl(generatePresignedUrlRequest); return F.Promise.pure(redirect(url.toString())); } catch (AmazonClientException ace) { Logger.error( "Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network." + " Error Message: " + ace.getMessage()); return F.Promise.pure(internalServerError("Download failed")); } }
@Override public void saveFile(File file, String fileName) throws IOException { AmazonS3 s3Client = getAmzS3Client(); s3Client.putObject(new PutObjectRequest(bucketName, fileName, file)); file.delete(); }
private AmazonS3 getAmzS3Client() { BasicAWSCredentials awsCreds = new BasicAWSCredentials(accessKeyId, secretAccessKey); AmazonS3 s3Client = new AmazonS3Client(awsCreds); Region region = Region.getRegion(Regions.EU_WEST_1); s3Client.setRegion(region); return s3Client; }
/** * Computes the presigned URL for the given S3 resource. * * @param path String like "/bucketName/folder/folder/abc.txt" that represents the resource to * request. */ public URL buildPresignedURL(String path) throws AmazonClientException { AWSCredentials credentials = awsCredentialsProvider.getCredentials(); long expires = System.currentTimeMillis() + 60 * 60 * 1000; GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(path, credentials.getAWSSecretKey()); request.setExpiration(new Date(expires)); AmazonS3 s3 = new AmazonS3Client(credentials); return s3.generatePresignedUrl(request); }
/* @author: Nitish */ public static String UploadToS3() throws IOException { String existingBucketName = "s3-bucket-zerorpc"; String keyName = "image.jpeg"; String filePath = "C:\\Users\\Nitish\\Desktop\\IMG_2695.JPG"; AmazonS3 s3Client = new AmazonS3Client(new BasicAWSCredentials("", "")); // Create a list of UploadPartResponse objects. You get one of these // for each part upload. List<PartETag> partETags = new ArrayList<PartETag>(); // Step 1: Initialize. InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(existingBucketName, keyName); InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); File file = new File(filePath); long contentLength = file.length(); long partSize = 5242880; // Set part size to 5 MB. try { // Step 2: Upload parts. long filePosition = 0; for (int i = 1; filePosition < contentLength; i++) { // Last part can be less than 5 MB. Adjust part size. partSize = Math.min(partSize, (contentLength - filePosition)); // Create request to upload a part. UploadPartRequest uploadRequest = new UploadPartRequest() .withBucketName(existingBucketName) .withKey(keyName) .withUploadId(initResponse.getUploadId()) .withPartNumber(i) .withFileOffset(filePosition) .withFile(file) .withPartSize(partSize); // Upload part and add response to our list. partETags.add(s3Client.uploadPart(uploadRequest).getPartETag()); filePosition += partSize; } // Step 3: Complete. CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest( existingBucketName, keyName, initResponse.getUploadId(), partETags); s3Client.completeMultipartUpload(compRequest); } catch (Exception e) { s3Client.abortMultipartUpload( new AbortMultipartUploadRequest(existingBucketName, keyName, initResponse.getUploadId())); } return "Uploaded successfully"; }
/** * AwsBucket can throw a proper exception. * * @throws IOException If succeeds */ @Test(expected = IOException.class) public void existsThrowsIOException() throws IOException { final Region region = Mockito.mock(Region.class); final AmazonS3 aws = Mockito.mock(AmazonS3.class); Mockito.when(region.aws()).thenReturn(aws); final String name = "throwing.bucket.com"; Mockito.when(aws.doesBucketExist(name)).thenThrow(new AmazonServiceException("Test exception")); final Bucket bucket = new AwsBucket(region, name); bucket.exists(); }
/** * AwsBucket can correctly check the existence of the non-existing bucket. * * @throws IOException If fails */ @Test public void existsNonExistingBucket() throws IOException { final Region region = Mockito.mock(Region.class); final AmazonS3 aws = Mockito.mock(AmazonS3.class); Mockito.when(region.aws()).thenReturn(aws); final String name = "non.existing.bucket.com"; Mockito.when(aws.doesBucketExist(name)).thenReturn(false); final Bucket bucket = new AwsBucket(region, name); Assert.assertFalse(bucket.exists()); }
@Override public void createContainer(String container, Config config) { super.createContainer(container, config); try { if (!client.doesBucketExist(container)) { client.createBucket(container); } } catch (Exception e) { throw new StorageException(e); } }
/** Data cleansing method */ public void cleanseData(AmazonS3 client) throws Exception { AwsDataLoader loader = new AwsDataLoader(); CSVReader reader = null; String prefix = loader.getS3Prefix(source); client.setEndpoint(S3_ENDPOINT); S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix)); reader = new CSVReader( new BufferedReader(new InputStreamReader(object.getObjectContent())), CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, HEADERS_LINE); ColumnPositionMappingStrategy<ProductLanguage> strat = new ColumnPositionMappingStrategy<ProductLanguage>(); strat.setType(ProductLanguage.class); String[] columns = new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"}; strat.setColumnMapping(columns); CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>(); list = csv.parse(strat, reader); System.out.println("ProductLanguageCleanser input size: " + list.size()); this.updateDataset(list); BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>(); ByteArrayOutputStream os = new ByteArrayOutputStream(); CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"'); // strat.setColumnMapping(columns); log.info("List size: " + list.size()); csvWriter.write(strat, writer, list); writer.flush(); String dataset = os.toString(); String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv"; client.setEndpoint(S3_ENDPOINT); ObjectMetadata omd = new ObjectMetadata(); try { byte[] content = dataset.getBytes(StandardCharsets.UTF_8); ByteArrayInputStream input = new ByteArrayInputStream(content); BufferedReader buffReader = new BufferedReader(new InputStreamReader(input)); buffReader.readLine(); InputStream inputObj = new ReaderInputStream(buffReader); // omd.setContentLength(IOUtils.toByteArray(input).length); client.putObject(BUCKET_NM, outPrefix, inputObj, omd); input.close(); } catch (IOException e) { log.error("Axon data write to s3 failed: " + e.getMessage()); } }
private void uploadToS3(File file, String bucketName, String key) { log.info("Uploading local WAR file {} to remote WAR file: {}", file.getName(), key); try { String s3key = URLEncoder.encode(key, "UTF-8"); AmazonS3 s3 = new AmazonS3Client(getAwsCredentials()); s3.putObject(bucketName, s3key, file); } catch (UnsupportedEncodingException ex) { log.error("Upload file to s3 failed."); log.error(ex.getMessage()); } }
private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) { URI location = combinedObject.getLocation(); log.info("starting multipart upload: %s", location); String bucket = getS3Bucket(location); String key = getS3ObjectKey(location); String uploadId = s3Service .initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key)) .getUploadId(); try { List<PartETag> parts = newArrayList(); int partNumber = 1; for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) { CopyPartResult part = s3Service.copyPart( new CopyPartRequest() .withUploadId(uploadId) .withPartNumber(partNumber) .withDestinationBucketName(bucket) .withDestinationKey(key) .withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation())) .withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation()))); parts.add(new PartETag(partNumber, part.getETag())); partNumber++; } String etag = s3Service .completeMultipartUpload( new CompleteMultipartUploadRequest(bucket, key, uploadId, parts)) .getETag(); ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key); log.info("completed multipart upload: %s", location); if (!etag.equals(newObject.getETag())) { // this might happen in rare cases due to S3's eventual consistency throw new IllegalStateException("completed etag is different from combined object etag"); } return updateStoredObject(location, newObject); } catch (AmazonClientException e) { try { s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId)); } catch (AmazonClientException ignored) { } throw Throwables.propagate(e); } }
@Override protected JsonArray serve(String filter, int limit) { JsonArray array = new JsonArray(); try { AmazonS3 s3 = PersistS3.getClient(); filter = Strings.nullToEmpty(filter); for( Bucket b : s3.listBuckets() ) { if( b.getName().startsWith(filter) ) array.add(new JsonPrimitive(b.getName())); if( array.size() == limit) break; } } catch( IllegalArgumentException _ ) { } return array; }
@Override public void deleteContainer(String container, Config config) { super.deleteContainer(container, config); try { if (client.doesBucketExist(container)) { client.deleteBucket(container); } } catch (AmazonS3Exception awse) { if (awse.getStatusCode() != HttpStatus.SC_NOT_FOUND) { throw new StorageException(awse); } } catch (Exception e) { throw new StorageException(e); } }
// 列出某个bucket的所有objects public static void listObjectInBucket(Bucket bucket) { List<Bucket> buckets = conn.listBuckets(); ObjectListing objects = conn.listObjects(bucket.getName()); do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { System.out.println( objectSummary.getKey() + "\t" + objectSummary.getSize() + "\t" + StringUtils.fromDate(objectSummary.getLastModified())); } objects = conn.listNextBatchOfObjects(objects); } while (objects.isTruncated()); }
public InputStream downloadRange(long start, long end, String file) { GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, file); rangeObjectRequest.setRange(start, end); // retrieve 1st 10 bytes. S3Object objectPortion = s3client.getObject(rangeObjectRequest); InputStream objectData = objectPortion.getObjectContent(); return objectData; }
private ObjectMetadata getS3ObjectMetadata(final Path path) throws IOException { try { return retry() .maxAttempts(maxClientRetries) .exponentialBackoff(new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0) .stopOn(InterruptedException.class, UnrecoverableS3OperationException.class) .run( "getS3ObjectMetadata", () -> { try { return s3.getObjectMetadata(uri.getHost(), keyFromPath(path)); } catch (AmazonS3Exception e) { if (e.getStatusCode() == SC_NOT_FOUND) { return null; } else if (e.getStatusCode() == SC_FORBIDDEN) { throw new UnrecoverableS3OperationException(e); } throw Throwables.propagate(e); } }); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw Throwables.propagate(e); } catch (Exception e) { Throwables.propagateIfInstanceOf(e, IOException.class); throw Throwables.propagate(e); } }
private S3Object getS3Object(final Path path, final long start) throws IOException { try { return retry() .maxAttempts(maxClientRetry) .exponentialBackoff( new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0) .stopOn(InterruptedException.class, UnrecoverableS3OperationException.class) .run( "getS3Object", () -> { try { return s3.getObject( new GetObjectRequest(host, keyFromPath(path)) .withRange(start, Long.MAX_VALUE)); } catch (AmazonServiceException e) { if (e.getStatusCode() == SC_FORBIDDEN) { throw new UnrecoverableS3OperationException(e); } throw Throwables.propagate(e); } }); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw Throwables.propagate(e); } catch (Exception e) { Throwables.propagateIfInstanceOf(e, IOException.class); throw Throwables.propagate(e); } }
public UrlList getUrlList(String url) { UrlList urlList = null; try { String key = Hash.hashKey(url); GetObjectRequest req = new GetObjectRequest(bucketName, key); S3Object s3Object = s3client.getObject(req); InputStream objectData = s3Object.getObjectContent(); BufferedReader reader = new BufferedReader(new InputStreamReader(objectData)); StringBuilder s3Content = new StringBuilder(); String line; while ((line = reader.readLine()) != null) { s3Content.append(line + "\r\n"); } reader.close(); objectData.close(); ObjectMapper mapper = new ObjectMapper(); mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE); mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY); urlList = mapper.readValue(s3Content.toString(), UrlList.class); } catch (AmazonS3Exception ase) { System.out.println("S3UrlListDA : document does not exist"); ase.printStackTrace(); } catch (IOException e) { System.out.println("S3UrlListDA : IOException while fetching document from S3"); e.printStackTrace(); } catch (NoSuchAlgorithmException e) { // TODO Auto-generated catch block e.printStackTrace(); } return urlList; }
// 获取名为name的bucket public static Bucket getBucketByName(String name) { List<Bucket> buckets = conn.listBuckets(); for (Bucket bucket : buckets) { if (bucket.getName().equalsIgnoreCase(name)) return bucket; } return null; }
// 列出所有buckets public static void listAllBuckets() { List<Bucket> buckets = conn.listBuckets(); for (Bucket bucket : buckets) { System.out.println( bucket.getName() + "\t\t" + StringUtils.fromDate(bucket.getCreationDate())); } }
// 列出有prefix前缀的所有objects public static void listObjectInBucketWithPrefix(Bucket bucket, String prefix) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket.getName()).withPrefix(prefix); ObjectListing objects = conn.listObjects(listObjectsRequest); do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { System.out.println( objectSummary.getKey() + "\t" + objectSummary.getSize() + "\t" + StringUtils.fromDate(objectSummary.getLastModified())); } objects = conn.listNextBatchOfObjects(objects); } while (objects.isTruncated()); }
private static void saveToS3( String bucket, String key, String imageType, AmazonS3 s3Client, String dstKey, BufferedImage resizedImage) throws IOException { // Re-encode image to target format ByteArrayOutputStream os = new ByteArrayOutputStream(); ImageIO.write(resizedImage, imageType, os); InputStream is = new ByteArrayInputStream(os.toByteArray()); // Set Content-Length and Content-Type ObjectMetadata meta = new ObjectMetadata(); meta.setContentLength(os.size()); String contentType = imageTypeMime.get(imageType); if (contentType == null) { throw new RuntimeException( String.format("Unknown content type for image type %s", imageType)); } meta.setContentType(contentType); // Uploading to S3 destination bucket System.out.println("Writing to: " + bucket + "/" + dstKey); s3Client.putObject(bucket, dstKey, is, meta); System.out.println( "Successfully resized " + bucket + "/" + key + " and uploaded to " + bucket + "/" + dstKey); }
private Note getNote(String key) throws IOException { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create(); S3Object s3object; try { s3object = s3client.getObject(new GetObjectRequest(bucketName, key)); } catch (AmazonClientException ace) { throw new IOException("Unable to retrieve object from S3: " + ace, ace); } Note note; try (InputStream ins = s3object.getObjectContent()) { String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); note = gson.fromJson(json, Note.class); } for (Paragraph p : note.getParagraphs()) { if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { p.setStatus(Status.ABORT); } } return note; }