@Override protected void getResource( String resourceName, File destination, TransferProgress transferProgress) throws TransferFailedException, ResourceDoesNotExistException { InputStream in = null; OutputStream out = null; try { S3Object s3Object = this.amazonS3.getObject(this.bucketName, getKey(resourceName)); in = s3Object.getObjectContent(); out = new TransferProgressFileOutputStream(destination, transferProgress); IoUtils.copy(in, out); } catch (AmazonServiceException e) { throw new ResourceDoesNotExistException( String.format("'%s' does not exist", resourceName), e); } catch (FileNotFoundException e) { throw new TransferFailedException(String.format("Cannot write file to '%s'", destination), e); } catch (IOException e) { throw new TransferFailedException( String.format("Cannot read from '%s' and write to '%s'", resourceName, destination), e); } finally { IoUtils.closeQuietly(in, out); } }
@Override public PointSet load(String pointSetId) throws Exception { File cachedFile; if (!workOffline) { // get pointset metadata from S3 cachedFile = new File(POINT_DIR, pointSetId + ".json"); if (!cachedFile.exists()) { POINT_DIR.mkdirs(); S3Object obj = s3.getObject(pointsetBucket, pointSetId + ".json.gz"); ObjectMetadata objMet = obj.getObjectMetadata(); FileOutputStream fos = new FileOutputStream(cachedFile); GZIPInputStream gis = new GZIPInputStream(obj.getObjectContent()); try { ByteStreams.copy(gis, fos); } finally { fos.close(); gis.close(); } } } else cachedFile = new File(POINT_DIR, pointSetId + ".json"); // grab it from the cache return PointSet.fromGeoJson(cachedFile); }
public UrlList getUrlList(String url) { UrlList urlList = null; try { String key = Hash.hashKey(url); GetObjectRequest req = new GetObjectRequest(bucketName, key); S3Object s3Object = s3client.getObject(req); InputStream objectData = s3Object.getObjectContent(); BufferedReader reader = new BufferedReader(new InputStreamReader(objectData)); StringBuilder s3Content = new StringBuilder(); String line; while ((line = reader.readLine()) != null) { s3Content.append(line + "\r\n"); } reader.close(); objectData.close(); ObjectMapper mapper = new ObjectMapper(); mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE); mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY); urlList = mapper.readValue(s3Content.toString(), UrlList.class); } catch (AmazonS3Exception ase) { System.out.println("S3UrlListDA : document does not exist"); ase.printStackTrace(); } catch (IOException e) { System.out.println("S3UrlListDA : IOException while fetching document from S3"); e.printStackTrace(); } catch (NoSuchAlgorithmException e) { // TODO Auto-generated catch block e.printStackTrace(); } return urlList; }
/** * The method fetches the specified file from the amazon s3 bucket * * @throws IOException */ public void getObjectFromS3() throws IOException { /** * Initializing the GetObjectRequest of Amazon S3. It is used to read files stored in amazon s3 * bucket. It is initialized with the Aws_Bucket_Name, in which the file is stored, and the file * name which we want to read */ GetObjectRequest getObj = new GetObjectRequest("AWS_BUCKET_NAME", "fileName"); /** * Use the Amazon S3 client and the GetObjectRequest to fetch the files and hold it in the * S3Object container. */ S3Object s3FileObj = getS3Client().getObject(getObj); /** * creating a temp file in memory for writing the file content * * <p>The Amazon S3Object does not directly converts to a File, nor does it has any built-in * function to do so. Hence we need to use the IOUtils of common.io for writing the input Stream * to a file. We can do the same using the conventional manual style but IOUtils provide the * built-in function for it, thus lessening our work. */ File tempJsFile = File.createTempFile("temp", ".js"); FileOutputStream out = new FileOutputStream(tempJsFile); IOUtils.copy(s3FileObj.getObjectContent(), out); out.close(); }
private Note getNote(String key) throws IOException { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create(); S3Object s3object; try { s3object = s3client.getObject(new GetObjectRequest(bucketName, key)); } catch (AmazonClientException ace) { throw new IOException("Unable to retrieve object from S3: " + ace, ace); } Note note; try (InputStream ins = s3object.getObjectContent()) { String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); note = gson.fromJson(json, Note.class); } for (Paragraph p : note.getParagraphs()) { if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { p.setStatus(Status.ABORT); } } return note; }
public InputStream downloadRange(long start, long end, String file) { GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, file); rangeObjectRequest.setRange(start, end); // retrieve 1st 10 bytes. S3Object objectPortion = s3client.getObject(rangeObjectRequest); InputStream objectData = objectPortion.getObjectContent(); return objectData; }
/** * Downloads an S3Object, as returned from {@link * AmazonS3Client#getObject(com.amazonaws.services.s3.model.GetObjectRequest)}, to the specified * file. * * @param s3Object The S3Object containing a reference to an InputStream containing the object's * data. * @param destinationFile The file to store the object's data in. * @param performIntegrityCheck Boolean valuable to indicate whether do the integrity check or not */ public static void downloadObjectToFile( S3Object s3Object, File destinationFile, boolean performIntegrityCheck) { // attempt to create the parent if it doesn't exist File parentDirectory = destinationFile.getParentFile(); if (parentDirectory != null && !parentDirectory.exists()) { parentDirectory.mkdirs(); } OutputStream outputStream = null; try { outputStream = new BufferedOutputStream(new FileOutputStream(destinationFile)); byte[] buffer = new byte[1024 * 10]; int bytesRead; while ((bytesRead = s3Object.getObjectContent().read(buffer)) > -1) { outputStream.write(buffer, 0, bytesRead); } } catch (IOException e) { try { s3Object.getObjectContent().abort(); } catch (IOException abortException) { log.warn("Couldn't abort stream", e); } throw new AmazonClientException( "Unable to store object contents to disk: " + e.getMessage(), e); } finally { try { outputStream.close(); } catch (Exception e) { } try { s3Object.getObjectContent().close(); } catch (Exception e) { } } byte[] clientSideHash = null; byte[] serverSideHash = null; try { // Multipart Uploads don't have an MD5 calculated on the service side if (ServiceUtils.isMultipartUploadETag(s3Object.getObjectMetadata().getETag()) == false) { clientSideHash = Md5Utils.computeMD5Hash(new FileInputStream(destinationFile)); serverSideHash = BinaryUtils.fromHex(s3Object.getObjectMetadata().getETag()); } } catch (Exception e) { log.warn("Unable to calculate MD5 hash to validate download: " + e.getMessage(), e); } if (performIntegrityCheck && clientSideHash != null && serverSideHash != null && !Arrays.equals(clientSideHash, serverSideHash)) { throw new AmazonClientException( "Unable to verify integrity of data download. " + "Client calculated content hash didn't match hash calculated by Amazon S3. " + "The data stored in '" + destinationFile.getAbsolutePath() + "' may be corrupt."); } }
/** * Return the graph for the given unique identifier for graph builder inputs on S3. If this is the * same as the last graph built, just return the pre-built graph. If not, build the graph from the * inputs, fetching them from S3 to the local cache as needed. */ public synchronized Graph getGraph(String graphId) { LOG.info("Finding a graph for ID {}", graphId); if (graphId.equals(currGraphId)) { LOG.info("GraphID has not changed. Reusing the last graph that was built."); return currGraph; } // The location of the inputs that will be used to build this graph File graphDataDirectory = new File(GRAPH_CACHE_DIR, graphId); // If we don't have a local copy of the inputs, fetch graph data as a ZIP from S3 and unzip it if (!graphDataDirectory.exists() || graphDataDirectory.list().length == 0) { LOG.info("Downloading graph input files."); graphDataDirectory.mkdirs(); S3Object graphDataZipObject = s3.getObject(graphBucket, graphId + ".zip"); ZipInputStream zis = new ZipInputStream(graphDataZipObject.getObjectContent()); try { ZipEntry entry; while ((entry = zis.getNextEntry()) != null) { File entryDestination = new File(graphDataDirectory, entry.getName()); // Are both these mkdirs calls necessary? entryDestination.getParentFile().mkdirs(); if (entry.isDirectory()) entryDestination.mkdirs(); else { OutputStream entryFileOut = new FileOutputStream(entryDestination); IOUtils.copy(zis, entryFileOut); entryFileOut.close(); } } zis.close(); } catch (Exception e) { // TODO delete graph cache dir which is probably corrupted LOG.info("Error retrieving graph files", e); } } else { LOG.info("Graph input files were found locally. Using these files from the cache."); } // Now we have a local copy of these graph inputs. Make a graph out of them. CommandLineParameters params = new CommandLineParameters(); params.build = new File(GRAPH_CACHE_DIR, graphId); params.inMemory = true; GraphBuilder graphBuilder = GraphBuilder.forDirectory(params, params.build); graphBuilder.run(); Graph graph = graphBuilder.getGraph(); graph.routerId = graphId; graph.index(new DefaultStreetVertexIndexFactory()); graph.index.clusterStopsAsNeeded(); this.currGraphId = graphId; this.currGraph = graph; return graph; }
/** * Returns an updated object where the object content input stream contains the decrypted * contents. * * @param object The object whose contents are to be decrypted. * @param instruction The instruction that will be used to decrypt the object data. * @return The updated object where the object content input stream contains the decrypted * contents. * @deprecated no longer used and will be removed in the future */ @Deprecated public static S3Object decryptObjectUsingInstruction( S3Object object, EncryptionInstruction instruction) { S3ObjectInputStream objectContent = object.getObjectContent(); InputStream decryptedInputStream = new RepeatableCipherInputStream(objectContent, instruction.getCipherFactory()); object.setObjectContent( new S3ObjectInputStream(decryptedInputStream, objectContent.getHttpRequest())); return object; }
/** Data cleansing method */ public void cleanseData(AmazonS3 client) throws Exception { AwsDataLoader loader = new AwsDataLoader(); CSVReader reader = null; String prefix = loader.getS3Prefix(source); client.setEndpoint(S3_ENDPOINT); S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix)); reader = new CSVReader( new BufferedReader(new InputStreamReader(object.getObjectContent())), CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, HEADERS_LINE); ColumnPositionMappingStrategy<ProductLanguage> strat = new ColumnPositionMappingStrategy<ProductLanguage>(); strat.setType(ProductLanguage.class); String[] columns = new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"}; strat.setColumnMapping(columns); CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>(); list = csv.parse(strat, reader); System.out.println("ProductLanguageCleanser input size: " + list.size()); this.updateDataset(list); BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>(); ByteArrayOutputStream os = new ByteArrayOutputStream(); CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"'); // strat.setColumnMapping(columns); log.info("List size: " + list.size()); csvWriter.write(strat, writer, list); writer.flush(); String dataset = os.toString(); String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv"; client.setEndpoint(S3_ENDPOINT); ObjectMetadata omd = new ObjectMetadata(); try { byte[] content = dataset.getBytes(StandardCharsets.UTF_8); ByteArrayInputStream input = new ByteArrayInputStream(content); BufferedReader buffReader = new BufferedReader(new InputStreamReader(input)); buffReader.readLine(); InputStream inputObj = new ReaderInputStream(buffReader); // omd.setContentLength(IOUtils.toByteArray(input).length); client.putObject(BUCKET_NM, outPrefix, inputObj, omd); input.close(); } catch (IOException e) { log.error("Axon data write to s3 failed: " + e.getMessage()); } }
/** * Builds an instruction object from the contents of an instruction file. * * @param instructionFile A non-null instruction file retrieved from S3 that contains encryption * information * @param materialsProvider The non-null encryption materials provider to be used to encrypt and * decrypt data. * @param cryptoProvider The crypto provider whose encryption implementation will be used to * encrypt and decrypt data. Null is ok and uses the preferred provider from * Security.getProviders(). * @return A non-null instruction object containing encryption information * @deprecated no longer used and will be removed in the future */ @Deprecated public static EncryptionInstruction buildInstructionFromInstructionFile( S3Object instructionFile, EncryptionMaterialsProvider materialsProvider, Provider cryptoProvider) { JSONObject instructionJSON = parseJSONInstruction(instructionFile); try { // Get fields from instruction object String encryptedSymmetricKeyB64 = instructionJSON.getString(Headers.CRYPTO_KEY); String ivB64 = instructionJSON.getString(Headers.CRYPTO_IV); String materialsDescriptionString = instructionJSON.tryGetString(Headers.MATERIALS_DESCRIPTION); Map<String, String> materialsDescription = convertJSONToMap(materialsDescriptionString); // Decode from Base 64 to standard binary bytes byte[] encryptedSymmetricKey = Base64.decode(encryptedSymmetricKeyB64); byte[] iv = Base64.decode(ivB64); if (encryptedSymmetricKey == null || iv == null) { // If necessary encryption info was not found in the instruction file, throw an exception. throw new AmazonClientException( String.format( "Necessary encryption info not found in the instruction file '%s' in bucket '%s'", instructionFile.getKey(), instructionFile.getBucketName())); } EncryptionMaterials materials = retrieveOriginalMaterials(materialsDescription, materialsProvider); // If we're unable to retrieve the original encryption materials, we can't decrypt the object, // so // throw an exception. if (materials == null) { throw new AmazonClientException( String.format( "Unable to retrieve the encryption materials that originally " + "encrypted object corresponding to instruction file '%s' in bucket '%s'.", instructionFile.getKey(), instructionFile.getBucketName())); } // Decrypt the symmetric key and create the symmetric cipher SecretKey symmetricKey = getDecryptedSymmetricKey(encryptedSymmetricKey, materials, cryptoProvider); CipherFactory cipherFactory = new CipherFactory(symmetricKey, Cipher.DECRYPT_MODE, iv, cryptoProvider); return new EncryptionInstruction( materialsDescription, encryptedSymmetricKey, symmetricKey, cipherFactory); } catch (JSONException e) { throw new AmazonClientException( "Unable to parse retrieved instruction file : " + e.getMessage()); } }
private S3Object getConfigObject() throws Exception { try { S3Object object = s3Client.getObject(arguments.getBucket(), arguments.getKey()); if (object.getObjectMetadata().getContentLength() > 0) { return object; } } catch (AmazonS3Exception e) { if (!isNotFoundError(e)) { throw e; } } return null; }
@Override public InputStream getObject(String container, String object, Config config) { super.getObject(container, object, config); InputStream stream; try { S3Object s3Obj = client.getObject(container, object); stream = s3Obj.getObjectContent(); } catch (Exception e) { throw new StorageException(e); } return stream; }
/** * Read a file from s3 or from the filesystem * * @param filename * @return */ public static String readFile(String filename) { try { if (isS3File(filename)) { String[] parts = filename.split("/"); String[] keyArray = Arrays.copyOfRange(parts, 3, parts.length); String key = StringUtils.join(keyArray, "/"); S3Object object = getInstance().getObject(new GetObjectRequest(parts[2], key)); return IOUtils.toString(object.getObjectContent()); } else { return FileUtils.readFileToString(new File(filename)); } } catch (IOException e) { return null; } }
/** * Tests the S3 access. * * @throws Exception if test fails */ @Test public void s3Test() throws Exception { // Create a real object and mock its initClient method final S3NutDao dao = spy(new S3NutDao("/path", false, null, -1, "wuic", "login", "pwd", false)); // Build client mock final AmazonS3Client client = mock(AmazonS3Client.class); when(dao.initClient()).thenReturn(client); // List returned by client final ObjectListing list = mock(ObjectListing.class); final S3ObjectSummary summary = mock(S3ObjectSummary.class); when(summary.getKey()).thenReturn("[cloud].css"); final S3ObjectSummary summarBis = mock(S3ObjectSummary.class); when(summarBis.getKey()).thenReturn("cloud.css"); when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(list); when(list.getObjectSummaries()).thenReturn(Arrays.asList(summary, summarBis)); // Bytes returned by mocked S3 final byte[] array = ".cloud { text-align : justify;}".getBytes(); final S3Object object = mock(S3Object.class); when(object.getObjectContent()) .thenReturn(new S3ObjectInputStream(new ByteArrayInputStream(array), null)); when(client.getObject(anyString(), anyString())).thenReturn(object); // TODO : problem here : we specify '[cloud.css]' but getNuts() returns 'cloud.css' because // regex are always activated ! final NutsHeap nutsHeap = new NutsHeap(Arrays.asList("[cloud].css"), dao, "heap"); Assert.assertEquals(nutsHeap.getNuts().size(), 1); final Engine compressor = new CssYuiCompressorEngine(true, "UTF-8", -1); final Engine cacheEngine = new EhCacheEngine(false, null); final Engine aggregator = new CGTextAggregatorEngine(true); cacheEngine.setNext(compressor); compressor.setNext(aggregator); final List<Nut> group = cacheEngine.parse(new EngineRequest("", "", nutsHeap, new HashMap<NutType, Engine>())); Assert.assertFalse(group.isEmpty()); InputStream is; for (Nut res : group) { is = res.openStream(); Assert.assertTrue(IOUtils.readString(new InputStreamReader(is)).length() > 0); is.close(); } }
/** * Gets an object stored in S3 and downloads it into the specified file. This method includes the * one-time retry mechanism after integrity check failure on the downloaded file. It will also * return immediately after getting null valued S3Object (when getObject request does not meet the * specified constraints). * * @param file The file to store the object's data in. * @param safeS3DownloadTask The implementation of SafeS3DownloadTask interface which allows user * to get access to all the visible variables at the calling site of this method. */ public static S3Object retryableDownloadS3ObjectToFile( File file, RetryableS3DownloadTask retryableS3DownloadTask) { boolean hasRetried = false; boolean needRetry; S3Object s3Object; do { needRetry = false; s3Object = retryableS3DownloadTask.getS3ObjectStream(); if (s3Object == null) return null; try { ServiceUtils.downloadObjectToFile( s3Object, file, retryableS3DownloadTask.needIntegrityCheck()); } catch (AmazonClientException ace) { // Determine whether an immediate retry is needed according to the captured // AmazonClientException. // (There are three cases when downloadObjectToFile() throws AmazonClientException: // 1) SocketException or SSLProtocolException when writing to disk (e.g. when user aborts // the download) // 2) Other IOException when writing to disk // 3) MD5 hashes don't match // The current code will retry the download only when case 2) or 3) happens. if (ace.getCause() instanceof SocketException || ace.getCause() instanceof SSLProtocolException) { throw ace; } else { needRetry = true; if (hasRetried) throw ace; else { log.info( "Retry the download of object " + s3Object.getKey() + " (bucket " + s3Object.getBucketName() + ")", ace); hasRetried = true; } } } finally { try { s3Object.getObjectContent().abort(); } catch (IOException e) { } } } while (needRetry); return s3Object; }
@Override public void download(String key, File file) throws IOException { S3Object s3Object; try { s3Object = s3Client.getObject(Namespaces.get().getBucket(), key); } catch (AmazonClientException ace) { throw new IOException(ace); } file.delete(); InputStream in = s3Object.getObjectContent(); try { IOUtils.copyStreamToFile(in, file); } finally { in.close(); } }
/** * Converts and return the underlying S3 object as a json string. * * @throws AmazonClientException if failed in JSON conversion. */ String toJsonString() { try { return from(s3obj.getObjectContent()); } catch (Exception e) { throw new AmazonClientException("Error parsing JSON: " + e.getMessage()); } }
/** * Returns true if the specified S3Object contains encryption info in its metadata, false * otherwise. * * @param retrievedObject An S3Object * @return True if the specified S3Object contains encryption info in its metadata, false * otherwise. * @deprecated no longer used and will be removed in the future */ @Deprecated public static boolean isEncryptionInfoInMetadata(S3Object retrievedObject) { Map<String, String> metadata = retrievedObject.getObjectMetadata().getUserMetadata(); return metadata != null && metadata.containsKey(Headers.CRYPTO_IV) && metadata.containsKey(Headers.CRYPTO_KEY); }
/** * Returns true if this S3 object has the encryption information stored as user meta data; false * otherwise. */ final boolean hasEncryptionInfo() { ObjectMetadata metadata = s3obj.getObjectMetadata(); Map<String, String> userMeta = metadata.getUserMetadata(); return userMeta != null && userMeta.containsKey(Headers.CRYPTO_IV) && (userMeta.containsKey(Headers.CRYPTO_KEY_V2) || userMeta.containsKey(Headers.CRYPTO_KEY)); }
@Override public BufferedReader streamFrom(String key) throws IOException { S3Object s3Object; try { s3Object = s3Client.getObject(Namespaces.get().getBucket(), key); } catch (AmazonClientException ace) { throw new IOException(ace); } String[] contentTypeEncoding = guessContentTypeEncodingFromExtension(key); String encoding = contentTypeEncoding[1]; InputStream in = s3Object.getObjectContent(); try { if ("gzip".equals(encoding)) { in = new GZIPInputStream(in); } else if ("zip".equals(encoding)) { in = new ZipInputStream(in); } else if ("deflate".equals(encoding)) { in = new InflaterInputStream(in); } else if ("bzip2".equals(encoding)) { in = new BZip2CompressorInputStream(in); } else { ObjectMetadata metadata = s3Object.getObjectMetadata(); if (metadata != null) { String contentEncoding = metadata.getContentEncoding(); if ("gzip".equals(contentEncoding)) { in = new GZIPInputStream(in); } else if ("zip".equals(contentEncoding)) { in = new ZipInputStream(in); } else if ("deflate".equals(contentEncoding)) { in = new InflaterInputStream(in); } else if ("bzip2".equals(contentEncoding)) { in = new BZip2CompressorInputStream(in); } } } } catch (IOException ioe) { // Be extra sure this doesn't like in case of an error in.close(); throw ioe; } return new BufferedReader(new InputStreamReader(in, Charsets.UTF_8), 1 << 20); // ~1MB }
/** @see com.amazonaws.http.HttpResponseHandler#handle(com.amazonaws.http.HttpResponse) */ public AmazonWebServiceResponse<S3Object> handle(HttpResponse response) throws Exception { /* * TODO: It'd be nice to set the bucket name and key here, but the * information isn't easy to pull out of the response/request * currently. */ S3Object object = new S3Object(); AmazonWebServiceResponse<S3Object> awsResponse = parseResponseMetadata(response); if (response.getHeaders().get(Headers.REDIRECT_LOCATION) != null) { object.setRedirectLocation(response.getHeaders().get(Headers.REDIRECT_LOCATION)); } ObjectMetadata metadata = object.getObjectMetadata(); populateObjectMetadata(response, metadata); boolean hasServerSideCalculatedChecksum = !ServiceUtils.isMultipartUploadETag(metadata.getETag()); boolean responseContainsEntireObject = response.getHeaders().get("Content-Range") == null; if (hasServerSideCalculatedChecksum && responseContainsEntireObject) { byte[] expectedChecksum = BinaryUtils.fromHex(metadata.getETag()); object.setObjectContent( new S3ObjectInputStream( new ChecksumValidatingInputStream( response.getContent(), expectedChecksum, object.getBucketName() + "/" + object.getKey()), response.getHttpRequest())); } else { object.setObjectContent( new S3ObjectInputStream(response.getContent(), response.getHttpRequest())); } awsResponse.setResult(object); return awsResponse; }
public static String scaleImage(String bucket, String key) throws IOException { if (key.startsWith(PREFIX)) { System.out.println("Target image is already scaled"); return "Nothing"; } Optional<String> optionalImageType = getImageType(key); if (!optionalImageType.isPresent()) { return ""; } String imageType = optionalImageType.get(); // Download the image from S3 into a stream AmazonS3 s3Client = new AmazonS3Client(); S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, key)); InputStream objectData = s3Object.getObjectContent(); // Read the source image BufferedImage srcImage = ImageIO.read(objectData); int srcHeight = srcImage.getHeight(); int srcWidth = srcImage.getWidth(); for (int scaleDimension : SCALE_DIMENSIONS) { // Infer the scaling factor to avoid stretching the image // unnaturally float scalingFactor = Math.min((float) scaleDimension / srcWidth, (float) scaleDimension / srcHeight); int width = (int) (scalingFactor * srcWidth); int height = (int) (scalingFactor * srcHeight); BufferedImage resizedImage = getHighQualityScaledInstance( srcImage, width, height, RenderingHints.VALUE_INTERPOLATION_BICUBIC); BufferedImage squaredImage = getSquaredImage(resizedImage, RenderingHints.VALUE_INTERPOLATION_BICUBIC); String dstKeyResized = PREFIX + "-" + scaleDimension + "-" + key; String dstKeyResizedSquared = PREFIX + "-" + scaleDimension + "-squared-" + key; saveToS3(bucket, key, imageType, s3Client, dstKeyResized, resizedImage); saveToS3(bucket, key, imageType, s3Client, dstKeyResizedSquared, squaredImage); } return "Ok"; }
/** * Parses instruction data retrieved from S3 and returns a JSONObject representing the instruction */ @Deprecated private static JSONObject parseJSONInstruction(S3Object instructionObject) { try { String instructionString = convertStreamToString(instructionObject.getObjectContent()); return new JSONObject(instructionString); } catch (Exception e) { throw new AmazonClientException("Error parsing JSON instruction file: " + e.getMessage()); } }
@Override public void downloadBackup( Exhibitor exhibitor, BackupMetaData backup, File destination, Map<String, String> configValues) throws Exception { S3Object object = s3Client.getObject(configValues.get(CONFIG_BUCKET.getKey()), toKey(backup)); long startMs = System.currentTimeMillis(); RetryPolicy retryPolicy = makeRetryPolicy(configValues); int retryCount = 0; boolean done = false; while (!done) { Throttle throttle = makeThrottle(configValues); InputStream in = null; FileOutputStream out = null; try { out = new FileOutputStream(destination); in = object.getObjectContent(); FileChannel channel = out.getChannel(); CompressorIterator compressorIterator = compressor.decompress(in); for (; ; ) { ByteBuffer bytes = compressorIterator.next(); if (bytes == null) { break; } throttle.throttle(bytes.limit()); channel.write(bytes); } done = true; } catch (Exception e) { if (!retryPolicy.allowRetry(retryCount++, System.currentTimeMillis() - startMs)) { done = true; } } finally { Closeables.closeQuietly(in); Closeables.closeQuietly(out); } } }
/** * Adjusts the retrieved S3Object so that the object contents contain only the range of bytes * desired by the user. Since encrypted contents can only be retrieved in CIPHER_BLOCK_SIZE (16 * bytes) chunks, the S3Object potentially contains more bytes than desired, so this method * adjusts the contents range. * * @param object The S3Object retrieved from S3 that could possibly contain more bytes than * desired by the user. * @param range A two-element array of longs corresponding to the start and finish (inclusive) of * a desired range of bytes. * @return The S3Object with adjusted object contents containing only the range desired by the * user. If the range specified is invalid, then the S3Object is returned without any * modifications. * @deprecated no longer used and will be removed in the future */ @Deprecated public static S3Object adjustOutputToDesiredRange(S3Object object, long[] range) { if (range == null || range[0] > range[1]) { // Make no modifications if range is invalid. return object; } else { try { S3ObjectInputStream objectContent = object.getObjectContent(); InputStream adjustedRangeContents = new AdjustedRangeInputStream(objectContent, range[0], range[1]); object.setObjectContent( new S3ObjectInputStream(adjustedRangeContents, objectContent.getHttpRequest())); return object; } catch (IOException e) { throw new AmazonClientException( "Error adjusting output to desired byte range: " + e.getMessage()); } } }
@Override public LoadedInstanceConfig loadConfig() throws Exception { Date lastModified; Properties properties = new Properties(); S3Object object = getConfigObject(); if (object != null) { try { lastModified = object.getObjectMetadata().getLastModified(); properties.load(object.getObjectContent()); } finally { Closeables.closeQuietly(object.getObjectContent()); } } else { lastModified = new Date(0L); } PropertyBasedInstanceConfig config = new PropertyBasedInstanceConfig(properties, defaults); return new LoadedInstanceConfig(config, lastModified.getTime()); }
/** * Returns the original crypto scheme used for encryption, which may differ from the crypto scheme * used for decryption during, for example, a range-get operation. * * @param instructionFile the instruction file of the s3 object; or null if there is none. */ ContentCryptoScheme encryptionSchemeOf(Map<String, String> instructionFile) { if (instructionFile != null) { String cekAlgo = instructionFile.get(Headers.CRYPTO_CEK_ALGORITHM); return ContentCryptoScheme.fromCEKAlgo(cekAlgo); } ObjectMetadata meta = s3obj.getObjectMetadata(); Map<String, String> userMeta = meta.getUserMetadata(); String cekAlgo = userMeta.get(Headers.CRYPTO_CEK_ALGORITHM); return ContentCryptoScheme.fromCEKAlgo(cekAlgo); }
/** * Builds an instruction object from the object metadata. * * @param object A non-null object that contains encryption information in its headers * @param materialsProvider The non-null encryption materials provider to be used to encrypt and * decrypt data. * @param cryptoProvider The crypto provider whose encryption implementation will be used to * encrypt and decrypt data. Null is ok and uses the preferred provider from * Security.getProviders(). * @return A non-null instruction object containing encryption information * @throws AmazonClientException if encryption information is missing in the metadata, or the * encryption materials used to encrypt the object are not available via the materials * Accessor * @deprecated no longer used and will be removed in the future */ @Deprecated public static EncryptionInstruction buildInstructionFromObjectMetadata( S3Object object, EncryptionMaterialsProvider materialsProvider, Provider cryptoProvider) { ObjectMetadata metadata = object.getObjectMetadata(); // Get encryption info from metadata. byte[] encryptedSymmetricKeyBytes = getCryptoBytesFromMetadata(Headers.CRYPTO_KEY, metadata); byte[] initVectorBytes = getCryptoBytesFromMetadata(Headers.CRYPTO_IV, metadata); String materialsDescriptionString = getStringFromMetadata(Headers.MATERIALS_DESCRIPTION, metadata); Map<String, String> materialsDescription = convertJSONToMap(materialsDescriptionString); if (encryptedSymmetricKeyBytes == null || initVectorBytes == null) { // If necessary encryption info was not found in the instruction file, throw an exception. throw new AmazonClientException( String.format( "Necessary encryption info not found in the headers of file '%s' in bucket '%s'", object.getKey(), object.getBucketName())); } EncryptionMaterials materials = retrieveOriginalMaterials(materialsDescription, materialsProvider); // If we're unable to retrieve the original encryption materials, we can't decrypt the object, // so // throw an exception. if (materials == null) { throw new AmazonClientException( String.format( "Unable to retrieve the encryption materials that originally " + "encrypted file '%s' in bucket '%s'.", object.getKey(), object.getBucketName())); } // Decrypt the symmetric key and create the symmetric cipher SecretKey symmetricKey = getDecryptedSymmetricKey(encryptedSymmetricKeyBytes, materials, cryptoProvider); CipherFactory cipherFactory = new CipherFactory(symmetricKey, Cipher.DECRYPT_MODE, initVectorBytes, cryptoProvider); return new EncryptionInstruction( materialsDescription, encryptedSymmetricKeyBytes, symmetricKey, cipherFactory); }
/* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#getObject(com.amazonaws.services.s3.model.GetObjectRequest, java.io.File) */ @Override public ObjectMetadata getObject(GetObjectRequest getObjectRequest, File destinationFile) throws AmazonClientException, AmazonServiceException { assertParameterNotNull( destinationFile, "The destination file parameter must be specified when downloading an object directly to a file"); S3Object s3Object = getObject(getObjectRequest); // getObject can return null if constraints were specified but not met if (s3Object == null) return null; OutputStream outputStream = null; try { outputStream = new BufferedOutputStream(new FileOutputStream(destinationFile)); byte[] buffer = new byte[1024 * 10]; int bytesRead; while ((bytesRead = s3Object.getObjectContent().read(buffer)) > -1) { outputStream.write(buffer, 0, bytesRead); } } catch (IOException e) { throw new AmazonClientException( "Unable to store object contents to disk: " + e.getMessage(), e); } finally { try { outputStream.close(); } catch (Exception e) { } try { s3Object.getObjectContent().close(); } catch (Exception e) { } } /* * Unlike the standard Amazon S3 Client, the Amazon S3 Encryption Client does not do an MD5 check * here because the contents stored in S3 and the contents we just retrieved are different. In * S3, the stored contents are encrypted, and locally, the retrieved contents are decrypted. */ return s3Object.getObjectMetadata(); }