public static boolean upload(MusicInfo music) {
String key = "";
try {
// upload mp3 ring
key = music.getUUID() + music.getRingName();
File file = new File(Consts.NEW_DOWNLOAD_DIR + music.getRingName());
s3.putObject(new PutObjectRequest(Consts.AMAZON_RING_BUCKET, key, file)); // upload ring
s3.setObjectAcl(
Consts.AMAZON_RING_BUCKET, key, CannedAccessControlList.PublicRead); // set access
// upload m4r ring
// String m4r = music.getRingName().replace(".mp3", ".m4r");
// key = music.getUUID() + m4r;
// file = new File(Consts.NEW_DOWNLOAD_DIR+m4r);
// s3.putObject(new PutObjectRequest(Consts.AMAZON_M4R_BUCKET, key, file)); //
// upload ring
// s3.setObjectAcl(Consts.AMAZON_M4R_BUCKET, key, CannedAccessControlList.PublicRead); //
// set access
// upload image
key = music.getUUID() + music.getImageName();
file = new File(Consts.NEW_DOWNLOAD_DIR + music.getImageName());
s3.putObject(new PutObjectRequest(Consts.AMAZON_IMAGE_BUCKET, key, file)); // upload image
s3.setObjectAcl(
Consts.AMAZON_IMAGE_BUCKET, key, CannedAccessControlList.PublicRead); // set access
return true;
} catch (Exception e) {
System.out.println("upload to S3 err");
e.printStackTrace();
return false;
}
}
private static void saveToS3(
String bucket,
String key,
String imageType,
AmazonS3 s3Client,
String dstKey,
BufferedImage resizedImage)
throws IOException {
// Re-encode image to target format
ByteArrayOutputStream os = new ByteArrayOutputStream();
ImageIO.write(resizedImage, imageType, os);
InputStream is = new ByteArrayInputStream(os.toByteArray());
// Set Content-Length and Content-Type
ObjectMetadata meta = new ObjectMetadata();
meta.setContentLength(os.size());
String contentType = imageTypeMime.get(imageType);
if (contentType == null) {
throw new RuntimeException(
String.format("Unknown content type for image type %s", imageType));
}
meta.setContentType(contentType);
// Uploading to S3 destination bucket
System.out.println("Writing to: " + bucket + "/" + dstKey);
s3Client.putObject(bucket, dstKey, is, meta);
System.out.println(
"Successfully resized " + bucket + "/" + key + " and uploaded to " + bucket + "/" + dstKey);
}
@Override
public void saveFile(File file, String fileName) throws IOException {
AmazonS3 s3Client = getAmzS3Client();
s3Client.putObject(new PutObjectRequest(bucketName, fileName, file));
file.delete();
}
/**
* Upload the input file to S3 bucket
*
* @param file : the file you want to upload
* @param name : File name
* @return if the upload is success, return true
*/
public boolean S3MapUpload(File file, String name, String userName, boolean ifPublic) {
System.out.println("Uploading a new object to S3 from a file\n");
if (ifPublic) {
PutObjectRequest por = new PutObjectRequest(bucketName, "maps/public/" + name, file);
por.setCannedAcl(CannedAccessControlList.PublicRead);
s3.putObject(por);
}
PutObjectRequest por = new PutObjectRequest(bucketName, "maps/" + userName + "/" + name, file);
por.setCannedAcl(CannedAccessControlList.PublicRead);
s3.putObject(por);
return true;
}
/** Data cleansing method */
public void cleanseData(AmazonS3 client) throws Exception {
AwsDataLoader loader = new AwsDataLoader();
CSVReader reader = null;
String prefix = loader.getS3Prefix(source);
client.setEndpoint(S3_ENDPOINT);
S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix));
reader =
new CSVReader(
new BufferedReader(new InputStreamReader(object.getObjectContent())),
CSVParser.DEFAULT_SEPARATOR,
CSVParser.DEFAULT_QUOTE_CHARACTER,
CSVParser.DEFAULT_ESCAPE_CHARACTER,
HEADERS_LINE);
ColumnPositionMappingStrategy<ProductLanguage> strat =
new ColumnPositionMappingStrategy<ProductLanguage>();
strat.setType(ProductLanguage.class);
String[] columns =
new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"};
strat.setColumnMapping(columns);
CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>();
list = csv.parse(strat, reader);
System.out.println("ProductLanguageCleanser input size: " + list.size());
this.updateDataset(list);
BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>();
ByteArrayOutputStream os = new ByteArrayOutputStream();
CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"');
// strat.setColumnMapping(columns);
log.info("List size: " + list.size());
csvWriter.write(strat, writer, list);
writer.flush();
String dataset = os.toString();
String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv";
client.setEndpoint(S3_ENDPOINT);
ObjectMetadata omd = new ObjectMetadata();
try {
byte[] content = dataset.getBytes(StandardCharsets.UTF_8);
ByteArrayInputStream input = new ByteArrayInputStream(content);
BufferedReader buffReader = new BufferedReader(new InputStreamReader(input));
buffReader.readLine();
InputStream inputObj = new ReaderInputStream(buffReader);
// omd.setContentLength(IOUtils.toByteArray(input).length);
client.putObject(BUCKET_NM, outPrefix, inputObj, omd);
input.close();
} catch (IOException e) {
log.error("Axon data write to s3 failed: " + e.getMessage());
}
}
private void uploadToS3(File file, String bucketName, String key) {
log.info("Uploading local WAR file {} to remote WAR file: {}", file.getName(), key);
try {
String s3key = URLEncoder.encode(key, "UTF-8");
AmazonS3 s3 = new AmazonS3Client(getAwsCredentials());
s3.putObject(bucketName, s3key, file);
} catch (UnsupportedEncodingException ex) {
log.error("Upload file to s3 failed.");
log.error(ex.getMessage());
}
}
@Override
public void createObject(
String container, String object, InputStream data, long length, Config config) {
super.createObject(container, object, data, length, config);
try {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(length);
metadata.setContentType("application/octet-stream");
client.putObject(container, object, data, metadata);
} catch (Exception e) {
throw new StorageException(e);
}
}
// Makes a putObjectRequest to make file public, and sends request.
private static boolean putObject(AmazonS3 s3, PutObjectRequest req) {
// Set file as public.
req.withCannedAcl(CannedAccessControlList.PublicRead);
// Send upload request.
try {
s3.putObject(req);
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
return false;
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
return false;
}
return true;
}
@Override
public StoredObject putObject(URI location, File source) {
try {
log.info("starting upload: %s", location);
PutObjectResult result =
s3Service.putObject(getS3Bucket(location), getS3ObjectKey(location), source);
ObjectMetadata metadata =
s3Service.getObjectMetadata(getS3Bucket(location), getS3ObjectKey(location));
if (!result.getETag().equals(metadata.getETag())) {
// this might happen in rare cases due to S3's eventual consistency
throw new IllegalStateException("uploaded etag is different from retrieved object etag");
}
log.info("completed upload: %s", location);
return updateStoredObject(location, metadata);
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
@Override
public void run() {
String fileName = null;
try {
// Init Amazon S3
AmazonS3 s3 =
new AmazonS3Client(
new BasicAWSCredentials(
"AKIAI42VQL6SUU5S27SA", "/P3g0/U/hnKgWKRMsRXtG1w+gM+H0dwtLPnuvOBG"));
String bucketName = "diaosi-mapreduce";
// Download and Extract
fileName = s3Path.substring(s3Path.indexOf("enwiki-20121001-"));
fileName = fileName.substring(0, fileName.indexOf("bz2")) + ".xml";
FileOutputStream fout = new FileOutputStream(fileName);
URL url = new URL(s3Path);
URLConnection conn = url.openConnection();
conn.setConnectTimeout(20000);
conn.setReadTimeout(20000);
InputStream is = conn.getInputStream();
is.read();
is.read();
CBZip2InputStream fin = new CBZip2InputStream(is);
while (true) {
int len = fin.read(buf, 0, buf.length);
System.out.println(fileName + ": " + len);
if (len == -1) break;
fout.write(buf, 0, len);
}
fout.flush();
fout.close();
fin.close();
// Upload to S3
File file = new File(fileName);
String key = fileName;
s3.putObject(new PutObjectRequest(bucketName, key, file));
} catch (Exception e) {
e.printStackTrace();
}
}
public void storeMessageInS3(String message) {
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
String bucketName = "mailAppTestBucket_1";
String key = "message_" + UUID.randomUUID();
// System.out.println("Uploading a new object to S3 from a file\n");
try {
s3.putObject(new PutObjectRequest(bucketName, key, createMessageFile(message)));
} catch (AmazonServiceException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (AmazonClientException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void putUrlList(UrlList urlList) {
try {
// hash url to get key
String s3Key = Hash.hashKey(urlList.getParentUrl());
// put document into s3
ObjectMapper mapper = new ObjectMapper();
mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE);
mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
ObjectWriter ow = mapper.writer().withDefaultPrettyPrinter();
String json = ow.writeValueAsString(urlList);
ByteArrayInputStream inputStream = new ByteArrayInputStream(json.getBytes());
ObjectMetadata omd = new ObjectMetadata();
omd.setContentLength(json.getBytes().length);
PutObjectRequest request = new PutObjectRequest(bucketName, s3Key, inputStream, omd);
s3client.putObject(request);
// No need to put into dynamo as it should already be there
/*
ObjectMapper mapper = new ObjectMapper();
mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE);
mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
ObjectWriter ow = mapper.writer().withDefaultPrettyPrinter();
String json = ow.writeValueAsString(doc);
ObjectMetadata omd = new ObjectMetadata();
omd.setContentType("text/html");
omd.setContentLength(json.length());
ByteArrayInputStream inputStream = new ByteArrayInputStream(
json.getBytes());
PutObjectRequest request = new PutObjectRequest(bucketName, s3Key,
inputStream, omd);
s3client.putObject(request);
dynamo.putItem("documentUrl", doc.getDocumentId(), "s3Key", s3Key);*/
} catch (JsonProcessingException e) {
System.out.println("S3UrlListDA : json processing excecption exception");
e.printStackTrace();
} catch (NoSuchAlgorithmException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Override
public void save(Note note, AuthenticationInfo subject) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
String json = gson.toJson(note);
String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";
File file = File.createTempFile("note", "json");
try {
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
writer.write(json);
writer.close();
s3client.putObject(new PutObjectRequest(bucketName, key, file));
} catch (AmazonClientException ace) {
throw new IOException("Unable to store note in S3: " + ace, ace);
} finally {
FileUtils.deleteQuietly(file);
}
}
@Override
public void upload(
final String key,
final String md5,
final String contentType,
final InputStream body,
final long contentLength)
throws Exception {
final ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(contentLength);
metadata.setContentType(contentType);
if (md5.equalsIgnoreCase(etags.get(key))) {
System.out.println("skipping:" + key);
return;
}
System.out.println("uploading:" + key);
s3.putObject(
new PutObjectRequest(bucket, key, body, metadata)
.withCannedAcl(CannedAccessControlList.PublicRead));
}
// 将文件f放到bucket中，名为name； 权限为公共读写。
public static void putObjectInBucket(Bucket bucket, String name, File f) {
conn.putObject(bucket.getName(), name, f);
conn.setObjectAcl(bucket.getName(), name, CannedAccessControlList.PublicReadWrite);
}
private void jButton1ActionPerformed(
java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jButton1ActionPerformed
// TODO add your handling code here: upload file
JFileChooser jfc = new JFileChooser();
jfc.setAcceptAllFileFilterUsed(false);
jfc.setFileFilter(new FileNameExtensionFilter("Text Files", "txt"));
int returnVal = jfc.showOpenDialog(this);
if (returnVal == JFileChooser.APPROVE_OPTION) {
File file = jfc.getSelectedFile();
try {
filepath = file.getCanonicalPath();
FileReader fr = new FileReader(filepath);
BufferedReader br = new BufferedReader(fr);
String line;
int total_no_of_words = 0;
while ((line = br.readLine()) != null) {
String[] words_arr = line.split(" ");
total_no_of_words += words_arr.length;
}
jLabel4.setText("word count :" + total_no_of_words);
System.out.println("Total no of words = " + total_no_of_words);
if (total_no_of_words != 0) {
File userFile = new File(filepath);
keyName = userFile.getName();
String myAccessKeyID = "AKIAI7PATIX6LHJA3A3Q";
String mySecretKey = "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";
AWSCredentials myCredentials = new BasicAWSCredentials(myAccessKeyID, mySecretKey);
ClientConfiguration clientConfig = new ClientConfiguration();
clientConfig.setProtocol(Protocol.HTTP);
AmazonS3 conn = new AmazonS3Client(myCredentials, clientConfig);
try {
File file1 = new File(filepath);
file_size = file1.length();
if (file_size < 100000) {
conn.putObject(new PutObjectRequest(bucketName, keyName, file1));
jLabel2.setText("Uploading complete !");
} else {
jLabel2.setText("Uploading failed size greater than 100kb...");
}
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response"
+ " for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
} /*catch (InterruptedException ex) {
Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
}*/
} else {
jLabel2.setText("Uploading failed file empty !");
}
} catch (IOException ex) {
Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
}
}
if (returnVal == JFileChooser.CANCEL_OPTION) {
System.exit(0);
}
} // GEN-LAST:event_jButton1ActionPerformed
public void uploadFile(String s, String file) throws UnsupportedEncodingException {
InputStream inStream = new ByteArrayInputStream(s.getBytes("UTF-8"));
s3client.deleteObject(bucketName, MasterAddrFileName);
s3client.putObject(new PutObjectRequest(bucketName, file, inStream, new ObjectMetadata()));
}
public static void main(String[] args) throws IOException {
/*
* This credentials provider implementation loads your AWS credentials
* from a properties file at the root of your classpath.
*
* Important: Be sure to fill in your AWS access credentials in the
* AwsCredentials.properties file before you try to run this
* sample.
* http://aws.amazon.com/security-credentials
*/
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
String key = "MyObjectKey";
System.out.println("===========================================");
System.out.println("Getting Started with Amazon S3");
System.out.println("===========================================\n");
try {
/*
* Create a new S3 bucket - Amazon S3 bucket names are globally unique,
* so once a bucket name has been taken by any user, you can't create
* another bucket with that same name.
*
* You can optionally specify a location for your bucket if you want to
* keep your data closer to your applications or users.
*/
System.out.println("Creating bucket " + bucketName + "\n");
s3.createBucket(bucketName);
/*
* List the buckets in your account
*/
System.out.println("Listing buckets");
for (Bucket bucket : s3.listBuckets()) {
System.out.println(" - " + bucket.getName());
}
System.out.println();
/*
* Upload an object to your bucket - You can easily upload a file to
* S3, or upload directly an InputStream if you know the length of
* the data in the stream. You can also specify your own metadata
* when uploading to S3, which allows you set a variety of options
* like content-type and content-encoding, plus additional metadata
* specific to your applications.
*/
System.out.println("Uploading a new object to S3 from a file\n");
s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));
/*
* Download an object - When you download an object, you get all of
* the object's metadata and a stream from which to read the contents.
* It's important to read the contents of the stream as quickly as
* possibly since the data is streamed directly from Amazon S3 and your
* network connection will remain open until you read all the data or
* close the input stream.
*
* GetObjectRequest also supports several other options, including
* conditional downloading of objects based on modification times,
* ETags, and selectively downloading a range of an object.
*/
System.out.println("Downloading an object");
S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
displayTextInputStream(object.getObjectContent());
/*
* List objects in your bucket by prefix - There are many options for
* listing the objects in your bucket. Keep in mind that buckets with
* many objects might truncate their results when listing their objects,
* so be sure to check if the returned object listing is truncated, and
* use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
* additional results.
*/
System.out.println("Listing objects");
ObjectListing objectListing =
s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
System.out.println(
" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")");
}
System.out.println();
/*
* Delete an object - Unless versioning has been turned on for your bucket,
* there is no way to undelete an object, so use caution when deleting objects.
*/
System.out.println("Deleting an object\n");
s3.deleteObject(bucketName, key);
/*
* Delete a bucket - A bucket must be completely empty before it can be
* deleted, so remember to delete any objects from your buckets before
* you try to delete them.
*/
System.out.println("Deleting bucket " + bucketName + "\n");
s3.deleteBucket(bucketName);
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
public void storeFile(final String fileName, final File file) {
s3client.putObject(new PutObjectRequest(S3_BUCKET, fileName, file));
}
