private File retrieveS3File(String sqsdMessageBody) throws UnsupportedEncodingException { File localFile = null; if (!sqsdMessageBody.isEmpty()) { AmazonS3 s3 = new AmazonS3Client(); List<S3EventNotificationRecord> records = S3EventNotification.parseJson(sqsdMessageBody).getRecords(); S3EventNotificationRecord firstRecord = records.get(0); String bucketName = firstRecord.getS3().getBucket().getName(); String objectRegion = firstRecord.getAwsRegion(); Region s3Region = Region.getRegion(Regions.fromName(objectRegion)); s3.setRegion(s3Region); // Object key may have spaces or unicode non-ASCII characters. String keyName = firstRecord.getS3().getObject().getKey().replace('+', ' '); keyName = URLDecoder.decode(keyName, "UTF-8"); localFile = new File(keyName); System.out.println("Downloading file: " + objectRegion + "/" + bucketName + "/" + keyName); s3.getObject(new GetObjectRequest(bucketName, keyName), localFile); if (!localFile.canRead()) { localFile = null; } } return localFile; }
public InputStream downloadRange(long start, long end, String file) { GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, file); rangeObjectRequest.setRange(start, end); // retrieve 1st 10 bytes. S3Object objectPortion = s3client.getObject(rangeObjectRequest); InputStream objectData = objectPortion.getObjectContent(); return objectData; }
public UrlList getUrlList(String url) { UrlList urlList = null; try { String key = Hash.hashKey(url); GetObjectRequest req = new GetObjectRequest(bucketName, key); S3Object s3Object = s3client.getObject(req); InputStream objectData = s3Object.getObjectContent(); BufferedReader reader = new BufferedReader(new InputStreamReader(objectData)); StringBuilder s3Content = new StringBuilder(); String line; while ((line = reader.readLine()) != null) { s3Content.append(line + "\r\n"); } reader.close(); objectData.close(); ObjectMapper mapper = new ObjectMapper(); mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE); mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY); urlList = mapper.readValue(s3Content.toString(), UrlList.class); } catch (AmazonS3Exception ase) { System.out.println("S3UrlListDA : document does not exist"); ase.printStackTrace(); } catch (IOException e) { System.out.println("S3UrlListDA : IOException while fetching document from S3"); e.printStackTrace(); } catch (NoSuchAlgorithmException e) { // TODO Auto-generated catch block e.printStackTrace(); } return urlList; }
private S3Object getS3Object(final Path path, final long start) throws IOException { try { return retry() .maxAttempts(maxClientRetry) .exponentialBackoff( new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0) .stopOn(InterruptedException.class, UnrecoverableS3OperationException.class) .run( "getS3Object", () -> { try { return s3.getObject( new GetObjectRequest(host, keyFromPath(path)) .withRange(start, Long.MAX_VALUE)); } catch (AmazonServiceException e) { if (e.getStatusCode() == SC_FORBIDDEN) { throw new UnrecoverableS3OperationException(e); } throw Throwables.propagate(e); } }); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw Throwables.propagate(e); } catch (Exception e) { Throwables.propagateIfInstanceOf(e, IOException.class); throw Throwables.propagate(e); } }
private Note getNote(String key) throws IOException { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create(); S3Object s3object; try { s3object = s3client.getObject(new GetObjectRequest(bucketName, key)); } catch (AmazonClientException ace) { throw new IOException("Unable to retrieve object from S3: " + ace, ace); } Note note; try (InputStream ins = s3object.getObjectContent()) { String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); note = gson.fromJson(json, Note.class); } for (Paragraph p : note.getParagraphs()) { if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { p.setStatus(Status.ABORT); } } return note; }
/** Data cleansing method */ public void cleanseData(AmazonS3 client) throws Exception { AwsDataLoader loader = new AwsDataLoader(); CSVReader reader = null; String prefix = loader.getS3Prefix(source); client.setEndpoint(S3_ENDPOINT); S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix)); reader = new CSVReader( new BufferedReader(new InputStreamReader(object.getObjectContent())), CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, HEADERS_LINE); ColumnPositionMappingStrategy<ProductLanguage> strat = new ColumnPositionMappingStrategy<ProductLanguage>(); strat.setType(ProductLanguage.class); String[] columns = new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"}; strat.setColumnMapping(columns); CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>(); list = csv.parse(strat, reader); System.out.println("ProductLanguageCleanser input size: " + list.size()); this.updateDataset(list); BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>(); ByteArrayOutputStream os = new ByteArrayOutputStream(); CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"'); // strat.setColumnMapping(columns); log.info("List size: " + list.size()); csvWriter.write(strat, writer, list); writer.flush(); String dataset = os.toString(); String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv"; client.setEndpoint(S3_ENDPOINT); ObjectMetadata omd = new ObjectMetadata(); try { byte[] content = dataset.getBytes(StandardCharsets.UTF_8); ByteArrayInputStream input = new ByteArrayInputStream(content); BufferedReader buffReader = new BufferedReader(new InputStreamReader(input)); buffReader.readLine(); InputStream inputObj = new ReaderInputStream(buffReader); // omd.setContentLength(IOUtils.toByteArray(input).length); client.putObject(BUCKET_NM, outPrefix, inputObj, omd); input.close(); } catch (IOException e) { log.error("Axon data write to s3 failed: " + e.getMessage()); } }
@Override public InputStream getObject(String container, String object, Config config) { super.getObject(container, object, config); InputStream stream; try { S3Object s3Obj = client.getObject(container, object); stream = s3Obj.getObjectContent(); } catch (Exception e) { throw new StorageException(e); } return stream; }
public static String scaleImage(String bucket, String key) throws IOException { if (key.startsWith(PREFIX)) { System.out.println("Target image is already scaled"); return "Nothing"; } Optional<String> optionalImageType = getImageType(key); if (!optionalImageType.isPresent()) { return ""; } String imageType = optionalImageType.get(); // Download the image from S3 into a stream AmazonS3 s3Client = new AmazonS3Client(); S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, key)); InputStream objectData = s3Object.getObjectContent(); // Read the source image BufferedImage srcImage = ImageIO.read(objectData); int srcHeight = srcImage.getHeight(); int srcWidth = srcImage.getWidth(); for (int scaleDimension : SCALE_DIMENSIONS) { // Infer the scaling factor to avoid stretching the image // unnaturally float scalingFactor = Math.min((float) scaleDimension / srcWidth, (float) scaleDimension / srcHeight); int width = (int) (scalingFactor * srcWidth); int height = (int) (scalingFactor * srcHeight); BufferedImage resizedImage = getHighQualityScaledInstance( srcImage, width, height, RenderingHints.VALUE_INTERPOLATION_BICUBIC); BufferedImage squaredImage = getSquaredImage(resizedImage, RenderingHints.VALUE_INTERPOLATION_BICUBIC); String dstKeyResized = PREFIX + "-" + scaleDimension + "-" + key; String dstKeyResizedSquared = PREFIX + "-" + scaleDimension + "-squared-" + key; saveToS3(bucket, key, imageType, s3Client, dstKeyResized, resizedImage); saveToS3(bucket, key, imageType, s3Client, dstKeyResizedSquared, squaredImage); } return "Ok"; }
public static void main(String[] args) { if (args.length < 2) { System.out.println("USAGE: localPath bucketname <bucketPrefix>"); System.exit(1); } String localPath = args[0]; String bucketName = args[1]; String bucketPrefix = ""; // add on extra slash if (!localPath.endsWith("/")) localPath += "/"; if (args.length == 3) bucketPrefix = args[2]; // check local dir, make if it does not exist File localDir = new File(localPath); if (!localDir.exists()) localDir.mkdirs(); if (!localDir.isDirectory()) { System.out.println("Local Dir is not a dir: " + localPath); System.exit(1); } long totalBytes = 0; long start = System.currentTimeMillis(); AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); ObjectListing listObjects = s3.listObjects(bucketName, bucketPrefix); do { for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) { S3Object object = s3.getObject(bucketName, objectSummary.getKey()); S3ObjectInputStream inputStream = object.getObjectContent(); if ("gzip".equals(object.getObjectMetadata().getContentEncoding())) { InputStream in = null; try { totalBytes += object.getObjectMetadata().getContentLength(); in = new GZIPInputStream(inputStream); // write this sucker out String path = localPath + object.getKey(); // have to take the gz off since this is not downloading compressed! if (path.endsWith(".gz")) path = path.substring(0, path.length() - 3); System.out.print("Writing file: " + path); File check = new File(path); File parentFile = check.getParentFile(); if (!parentFile.exists()) parentFile.mkdirs(); FileOutputStream out = new FileOutputStream(path); IOUtils.copy(in, out); System.out.println(" written."); } catch (IOException e) { System.out.println("crap"); e.printStackTrace(); throw new IllegalStateException("files are too hard", e); } finally { IOUtils.closeQuietly(in); } } else { System.out.println( "unhandled content encoding: " + object.getObjectMetadata().getContentEncoding()); } } listObjects = s3.listNextBatchOfObjects(listObjects); } while (listObjects.isTruncated()); long now = System.currentTimeMillis(); System.out.println( (totalBytes / 1000.0 / 1000.0) + " mb downloaded in " + ((now - start) / 1000) + " seconds."); }
public long getInputFileSize() { S3Object object = s3client.getObject(new GetObjectRequest(bucketName, inputFileName)); return object.getObjectMetadata().getContentLength(); }
public InputStream downloadFile(String fileName) { S3Object object = s3client.getObject(new GetObjectRequest(bucketName, fileName)); return object.getObjectContent(); }
public static void main(String[] args) throws IOException { /* * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println( "Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println( "Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
// 从bucket中下载object name,并放在"/home/zforCeph/download"文件夹下 public static void getObjectFromBucket(Bucket bucket, String path, String name) { conn.getObject(new GetObjectRequest(bucket.getName(), name), new File(path + name)); }
//////////////////////////////////////////////////////////////////////////// ///// private methods //////////////////////////////////////////////////////////////////////////// private InputStream loadEventFromS3(AmazonS3 s3Client, String bucketName, final String key) throws IOException { S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, key)); return s3Object.getObjectContent(); }
/** * get a single map * * @param mapName */ public void getMap(String mapName) { s3.getObject(new GetObjectRequest(bucketName, mapName), new File(mapName)); }