private File retrieveS3File(String sqsdMessageBody) throws UnsupportedEncodingException {
File localFile = null;
if (!sqsdMessageBody.isEmpty()) {
AmazonS3 s3 = new AmazonS3Client();
List<S3EventNotificationRecord> records =
S3EventNotification.parseJson(sqsdMessageBody).getRecords();
S3EventNotificationRecord firstRecord = records.get(0);
String bucketName = firstRecord.getS3().getBucket().getName();
String objectRegion = firstRecord.getAwsRegion();
Region s3Region = Region.getRegion(Regions.fromName(objectRegion));
s3.setRegion(s3Region);
// Object key may have spaces or unicode non-ASCII characters.
String keyName = firstRecord.getS3().getObject().getKey().replace('+', ' ');
keyName = URLDecoder.decode(keyName, "UTF-8");
localFile = new File(keyName);
System.out.println("Downloading file: " + objectRegion + "/" + bucketName + "/" + keyName);
s3.getObject(new GetObjectRequest(bucketName, keyName), localFile);
if (!localFile.canRead()) {
localFile = null;
}
}
return localFile;
}
public InputStream downloadRange(long start, long end, String file) {
GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, file);
rangeObjectRequest.setRange(start, end); // retrieve 1st 10 bytes.
S3Object objectPortion = s3client.getObject(rangeObjectRequest);
InputStream objectData = objectPortion.getObjectContent();
return objectData;
}
public UrlList getUrlList(String url) {
UrlList urlList = null;
try {
String key = Hash.hashKey(url);
GetObjectRequest req = new GetObjectRequest(bucketName, key);
S3Object s3Object = s3client.getObject(req);
InputStream objectData = s3Object.getObjectContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(objectData));
StringBuilder s3Content = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
s3Content.append(line + "\r\n");
}
reader.close();
objectData.close();
ObjectMapper mapper = new ObjectMapper();
mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE);
mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
urlList = mapper.readValue(s3Content.toString(), UrlList.class);
} catch (AmazonS3Exception ase) {
System.out.println("S3UrlListDA : document does not exist");
ase.printStackTrace();
} catch (IOException e) {
System.out.println("S3UrlListDA : IOException while fetching document from S3");
e.printStackTrace();
} catch (NoSuchAlgorithmException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return urlList;
}
private S3Object getS3Object(final Path path, final long start) throws IOException {
try {
return retry()
.maxAttempts(maxClientRetry)
.exponentialBackoff(
new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0)
.stopOn(InterruptedException.class, UnrecoverableS3OperationException.class)
.run(
"getS3Object",
() -> {
try {
return s3.getObject(
new GetObjectRequest(host, keyFromPath(path))
.withRange(start, Long.MAX_VALUE));
} catch (AmazonServiceException e) {
if (e.getStatusCode() == SC_FORBIDDEN) {
throw new UnrecoverableS3OperationException(e);
}
throw Throwables.propagate(e);
}
});
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw Throwables.propagate(e);
} catch (Exception e) {
Throwables.propagateIfInstanceOf(e, IOException.class);
throw Throwables.propagate(e);
}
}
private Note getNote(String key) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson =
gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create();
S3Object s3object;
try {
s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
} catch (AmazonClientException ace) {
throw new IOException("Unable to retrieve object from S3: " + ace, ace);
}
Note note;
try (InputStream ins = s3object.getObjectContent()) {
String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
note = gson.fromJson(json, Note.class);
}
for (Paragraph p : note.getParagraphs()) {
if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
p.setStatus(Status.ABORT);
}
}
return note;
}
/** Data cleansing method */
public void cleanseData(AmazonS3 client) throws Exception {
AwsDataLoader loader = new AwsDataLoader();
CSVReader reader = null;
String prefix = loader.getS3Prefix(source);
client.setEndpoint(S3_ENDPOINT);
S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix));
reader =
new CSVReader(
new BufferedReader(new InputStreamReader(object.getObjectContent())),
CSVParser.DEFAULT_SEPARATOR,
CSVParser.DEFAULT_QUOTE_CHARACTER,
CSVParser.DEFAULT_ESCAPE_CHARACTER,
HEADERS_LINE);
ColumnPositionMappingStrategy<ProductLanguage> strat =
new ColumnPositionMappingStrategy<ProductLanguage>();
strat.setType(ProductLanguage.class);
String[] columns =
new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"};
strat.setColumnMapping(columns);
CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>();
list = csv.parse(strat, reader);
System.out.println("ProductLanguageCleanser input size: " + list.size());
this.updateDataset(list);
BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>();
ByteArrayOutputStream os = new ByteArrayOutputStream();
CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"');
// strat.setColumnMapping(columns);
log.info("List size: " + list.size());
csvWriter.write(strat, writer, list);
writer.flush();
String dataset = os.toString();
String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv";
client.setEndpoint(S3_ENDPOINT);
ObjectMetadata omd = new ObjectMetadata();
try {
byte[] content = dataset.getBytes(StandardCharsets.UTF_8);
ByteArrayInputStream input = new ByteArrayInputStream(content);
BufferedReader buffReader = new BufferedReader(new InputStreamReader(input));
buffReader.readLine();
InputStream inputObj = new ReaderInputStream(buffReader);
// omd.setContentLength(IOUtils.toByteArray(input).length);
client.putObject(BUCKET_NM, outPrefix, inputObj, omd);
input.close();
} catch (IOException e) {
log.error("Axon data write to s3 failed: " + e.getMessage());
}
}
@Override
public InputStream getObject(String container, String object, Config config) {
super.getObject(container, object, config);
InputStream stream;
try {
S3Object s3Obj = client.getObject(container, object);
stream = s3Obj.getObjectContent();
} catch (Exception e) {
throw new StorageException(e);
}
return stream;
}
public static String scaleImage(String bucket, String key) throws IOException {
if (key.startsWith(PREFIX)) {
System.out.println("Target image is already scaled");
return "Nothing";
}
Optional<String> optionalImageType = getImageType(key);
if (!optionalImageType.isPresent()) {
return "";
}
String imageType = optionalImageType.get();
// Download the image from S3 into a stream
AmazonS3 s3Client = new AmazonS3Client();
S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, key));
InputStream objectData = s3Object.getObjectContent();
// Read the source image
BufferedImage srcImage = ImageIO.read(objectData);
int srcHeight = srcImage.getHeight();
int srcWidth = srcImage.getWidth();
for (int scaleDimension : SCALE_DIMENSIONS) {
// Infer the scaling factor to avoid stretching the image
// unnaturally
float scalingFactor =
Math.min((float) scaleDimension / srcWidth, (float) scaleDimension / srcHeight);
int width = (int) (scalingFactor * srcWidth);
int height = (int) (scalingFactor * srcHeight);
BufferedImage resizedImage =
getHighQualityScaledInstance(
srcImage, width, height, RenderingHints.VALUE_INTERPOLATION_BICUBIC);
BufferedImage squaredImage =
getSquaredImage(resizedImage, RenderingHints.VALUE_INTERPOLATION_BICUBIC);
String dstKeyResized = PREFIX + "-" + scaleDimension + "-" + key;
String dstKeyResizedSquared = PREFIX + "-" + scaleDimension + "-squared-" + key;
saveToS3(bucket, key, imageType, s3Client, dstKeyResized, resizedImage);
saveToS3(bucket, key, imageType, s3Client, dstKeyResizedSquared, squaredImage);
}
return "Ok";
}
public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: localPath bucketname <bucketPrefix>");
System.exit(1);
}
String localPath = args[0];
String bucketName = args[1];
String bucketPrefix = "";
// add on extra slash
if (!localPath.endsWith("/")) localPath += "/";
if (args.length == 3) bucketPrefix = args[2];
// check local dir, make if it does not exist
File localDir = new File(localPath);
if (!localDir.exists()) localDir.mkdirs();
if (!localDir.isDirectory()) {
System.out.println("Local Dir is not a dir: " + localPath);
System.exit(1);
}
long totalBytes = 0;
long start = System.currentTimeMillis();
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
ObjectListing listObjects = s3.listObjects(bucketName, bucketPrefix);
do {
for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) {
S3Object object = s3.getObject(bucketName, objectSummary.getKey());
S3ObjectInputStream inputStream = object.getObjectContent();
if ("gzip".equals(object.getObjectMetadata().getContentEncoding())) {
InputStream in = null;
try {
totalBytes += object.getObjectMetadata().getContentLength();
in = new GZIPInputStream(inputStream);
// write this sucker out
String path = localPath + object.getKey();
// have to take the gz off since this is not downloading compressed!
if (path.endsWith(".gz")) path = path.substring(0, path.length() - 3);
System.out.print("Writing file: " + path);
File check = new File(path);
File parentFile = check.getParentFile();
if (!parentFile.exists()) parentFile.mkdirs();
FileOutputStream out = new FileOutputStream(path);
IOUtils.copy(in, out);
System.out.println(" written.");
} catch (IOException e) {
System.out.println("crap");
e.printStackTrace();
throw new IllegalStateException("files are too hard", e);
} finally {
IOUtils.closeQuietly(in);
}
} else {
System.out.println(
"unhandled content encoding: " + object.getObjectMetadata().getContentEncoding());
}
}
listObjects = s3.listNextBatchOfObjects(listObjects);
} while (listObjects.isTruncated());
long now = System.currentTimeMillis();
System.out.println(
(totalBytes / 1000.0 / 1000.0)
+ " mb downloaded in "
+ ((now - start) / 1000)
+ " seconds.");
}
public long getInputFileSize() {
S3Object object = s3client.getObject(new GetObjectRequest(bucketName, inputFileName));
return object.getObjectMetadata().getContentLength();
}
public InputStream downloadFile(String fileName) {
S3Object object = s3client.getObject(new GetObjectRequest(bucketName, fileName));
return object.getObjectContent();
}
public static void main(String[] args) throws IOException {
/*
* This credentials provider implementation loads your AWS credentials
* from a properties file at the root of your classpath.
*
* Important: Be sure to fill in your AWS access credentials in the
* AwsCredentials.properties file before you try to run this
* sample.
* http://aws.amazon.com/security-credentials
*/
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
String key = "MyObjectKey";
System.out.println("===========================================");
System.out.println("Getting Started with Amazon S3");
System.out.println("===========================================\n");
try {
/*
* Create a new S3 bucket - Amazon S3 bucket names are globally unique,
* so once a bucket name has been taken by any user, you can't create
* another bucket with that same name.
*
* You can optionally specify a location for your bucket if you want to
* keep your data closer to your applications or users.
*/
System.out.println("Creating bucket " + bucketName + "\n");
s3.createBucket(bucketName);
/*
* List the buckets in your account
*/
System.out.println("Listing buckets");
for (Bucket bucket : s3.listBuckets()) {
System.out.println(" - " + bucket.getName());
}
System.out.println();
/*
* Upload an object to your bucket - You can easily upload a file to
* S3, or upload directly an InputStream if you know the length of
* the data in the stream. You can also specify your own metadata
* when uploading to S3, which allows you set a variety of options
* like content-type and content-encoding, plus additional metadata
* specific to your applications.
*/
System.out.println("Uploading a new object to S3 from a file\n");
s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));
/*
* Download an object - When you download an object, you get all of
* the object's metadata and a stream from which to read the contents.
* It's important to read the contents of the stream as quickly as
* possibly since the data is streamed directly from Amazon S3 and your
* network connection will remain open until you read all the data or
* close the input stream.
*
* GetObjectRequest also supports several other options, including
* conditional downloading of objects based on modification times,
* ETags, and selectively downloading a range of an object.
*/
System.out.println("Downloading an object");
S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
displayTextInputStream(object.getObjectContent());
/*
* List objects in your bucket by prefix - There are many options for
* listing the objects in your bucket. Keep in mind that buckets with
* many objects might truncate their results when listing their objects,
* so be sure to check if the returned object listing is truncated, and
* use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
* additional results.
*/
System.out.println("Listing objects");
ObjectListing objectListing =
s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
System.out.println(
" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")");
}
System.out.println();
/*
* Delete an object - Unless versioning has been turned on for your bucket,
* there is no way to undelete an object, so use caution when deleting objects.
*/
System.out.println("Deleting an object\n");
s3.deleteObject(bucketName, key);
/*
* Delete a bucket - A bucket must be completely empty before it can be
* deleted, so remember to delete any objects from your buckets before
* you try to delete them.
*/
System.out.println("Deleting bucket " + bucketName + "\n");
s3.deleteBucket(bucketName);
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
// 从bucket中下载object name，并放在"/home/zforCeph/download"文件夹下
public static void getObjectFromBucket(Bucket bucket, String path, String name) {
conn.getObject(new GetObjectRequest(bucket.getName(), name), new File(path + name));
}
////////////////////////////////////////////////////////////////////////////
///// private methods
////////////////////////////////////////////////////////////////////////////
private InputStream loadEventFromS3(AmazonS3 s3Client, String bucketName, final String key)
throws IOException {
S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, key));
return s3Object.getObjectContent();
}
/**
* get a single map
*
* @param mapName
*/
public void getMap(String mapName) {
s3.getObject(new GetObjectRequest(bucketName, mapName), new File(mapName));
}