private static void basicAbortMPU() throws IOException {
System.out.println("basic abort MPU");
String bucketName = "chttest";
String fileName = "hello.txt";
// String uploadID = "XHGTFV4F5XTEAC5O8N3LK12TIY3DSY7OFPXIWTHRMNTE7A3WB5M8N2U5AN"; //hi
String uploadID = "LE5JS2K6C208JU7ZX1QD2TVRWXOWWF4VNG7LE7TFIX5SYNG4HLOGW9CLAD"; // hello
AbortMultipartUploadRequest request =
new AbortMultipartUploadRequest(bucketName, fileName, uploadID);
AmazonS3 s3 =
new AmazonS3Client(
new PropertiesCredentials(
putBucket.class.getResourceAsStream("AwsCredentials.properties")));
try {
s3.abortMultipartUpload(request);
System.out.println();
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
@Override
public int read(
String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
logger.debug("readkey: " + key + " from table: " + table);
GetItemRequest req = new GetItemRequest(table, createPrimaryKey(key));
req.setAttributesToGet(fields);
req.setConsistentRead(consistentRead);
GetItemResult res = null;
try {
res = dynamoDB.getItem(req);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
if (null != res.getItem()) {
result.putAll(extractResult(res.getItem()));
logger.debug("Result: " + res.toString());
}
return OK;
}
private static void createKey(String keyName, AmazonEC2 ec2) {
try {
List<KeyPairInfo> keyPairList = ec2.describeKeyPairs().getKeyPairs();
for (KeyPairInfo keyPair : keyPairList) {
if (keyName.equalsIgnoreCase(keyPair.getKeyName())) {
System.out.println("Using key " + keyName);
return;
}
}
System.out.println("Creating key " + keyName + "in local directory");
CreateKeyPairRequest newKeyRequest = new CreateKeyPairRequest();
newKeyRequest.setKeyName(keyName);
CreateKeyPairResult keyresult = ec2.createKeyPair(newKeyRequest);
KeyPair keyPair = new KeyPair();
keyPair = keyresult.getKeyPair();
String privateKey = keyPair.getKeyMaterial();
writeKeytoFile(keyName, privateKey);
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
}
}
public static void createBucket(AmazonS3Client s3, String bucketName, String zone) {
try {
List<Bucket> bucketList = s3.listBuckets();
for (Bucket bucket : bucketList) {
// System.out.println(bucket.getName());
if (bucketName.equalsIgnoreCase(bucket.getName())) {
System.out.println("Using bucket " + bucketName);
return;
}
}
System.out.println("Created s3 bucket " + bucketName);
s3.createBucket(bucketName);
return;
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
}
}
public LinkedList<String> listNames(String prefix, String bucketName, String[] canonicalIDs)
throws StorageCloudException {
LinkedList<String> find = new LinkedList<String>();
try {
ObjectListing objectListing = null;
if (bucketName == null)
objectListing =
conn.listObjects(
new ListObjectsRequest().withBucketName(defaultBucketName).withPrefix(prefix));
else {
bucketName = bucketName.concat(location);
objectListing =
conn.listObjects(
new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix));
}
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
find.add(objectSummary.getKey());
}
} catch (AmazonServiceException e1) {
throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
} catch (AmazonClientException e2) {
throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
}
return find;
}
public String[] setAcl(String bucketNameToShare, String[] canonicalId, String permission)
throws StorageCloudException {
boolean withRead = false;
if (bucketNameToShare != null) {
bucketNameToShare = bucketNameToShare.concat(location);
if (!conn.doesBucketExist(bucketNameToShare)) {
conn.createBucket(bucketNameToShare, region);
}
} else {
return null;
}
// set acl
AccessControlList acl = conn.getBucketAcl(bucketNameToShare);
for (int i = 0; i < canonicalId.length; i++) {
if (permission.equals("rw")) {
CanonicalGrantee grantee = new CanonicalGrantee(canonicalId[i]);
acl.grantPermission(grantee, Permission.Read);
acl.grantPermission(grantee, Permission.Write);
withRead = true;
} else if (permission.equals("r")) {
acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
withRead = true;
} else if (permission.equals("w")) {
acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Write);
}
}
try {
if (withRead) {
ObjectListing objectListing = conn.listObjects(bucketNameToShare);
AccessControlList aclKeys = null;
for (S3ObjectSummary elem : objectListing.getObjectSummaries()) {
aclKeys = conn.getObjectAcl(bucketNameToShare, elem.getKey());
for (int i = 0; i < canonicalId.length; i++) {
aclKeys.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
}
conn.setObjectAcl(bucketNameToShare, elem.getKey(), aclKeys);
}
}
// confirm if acl well
conn.setBucketAcl(bucketNameToShare, acl);
AccessControlList newAcl = conn.getBucketAcl(bucketNameToShare);
Set<Grant> grants = newAcl.getGrants();
boolean flag = false;
for (Grant grant : grants) {
if (grant.getGrantee().getIdentifier().equals(canonicalId[0])) {
flag = true;
}
}
if (!flag) {
throw new ServiceSiteException("AWSS3Exception:: ACL");
}
} catch (AmazonServiceException e1) {
throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
} catch (AmazonClientException e2) {
throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
}
return canonicalId;
}
private int parseClockSkewOffset(
org.apache.http.HttpResponse response, AmazonServiceException exception) {
Date deviceDate = new Date();
Date serverDate = null;
String serverDateStr = null;
Header[] responseDateHeader = response.getHeaders("Date");
try {
if (responseDateHeader.length == 0) {
// SQS doesn't return Date header
serverDateStr = getServerDateFromException(exception.getMessage());
serverDate = DateUtils.parseCompressedISO8601Date(serverDateStr);
} else {
serverDateStr = responseDateHeader[0].getValue();
serverDate = DateUtils.parseRFC822Date(serverDateStr);
}
} catch (RuntimeException e) {
log.warn("Unable to parse clock skew offset from response: " + serverDateStr, e);
return 0;
}
long diff = deviceDate.getTime() - serverDate.getTime();
return (int) (diff / 1000);
}
public void storeMessageInQueue(String message) {
// System.out.println("Storing Message...");
// Send a message
if (driectDbConnection) {
DbConnector db = null;
try {
db = dbf.createDBConnectorInstance();
} catch (MailAppDBException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
MailAppMessage mailMsg =
new MailAppMessage(message, "" + System.currentTimeMillis() + "_" + UUID.randomUUID());
try {
// printDebug("Calling DB-Instance to store message "+mailMsg.getTimeuuid()+" length:
// "+mailMsg.getSize()+"\n"+mailMsg.getWholeMessageWITHOUT2LINES());
db.storeMessage(mailMsg);
} catch (MailAppDBException mae) {
// MailStoringWorkerStarter.stopMailStoringWorker(MailStoringWorkerStarter.getWorkerList(), this);
// stop=true;
mae.printStackTrace();
}
} else {
try {
sqs.sendMessage(new SendMessageRequest(mailQueueUrl, message));
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon SQS, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with SQS, such as not "
+ "being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
printDebug("Message stored to " + mailQueueUrl);
// connect to db ( Cassandra ) and store message
// OR write Message to SQS to be consumed by Storage Worker ->
// better? more scalable, db independent
}
}
public static void sendMessage(AmazonSQS sqs, String sqsUrl, String info) {
try {
sqs.sendMessage(new SendMessageRequest(sqsUrl, info));
logger.info("Message sent to queue: " + info);
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
}
}
private void logException(AmazonServiceException ase) {
logger.error(
"AmazonServiceException: error={}, statuscode={}, "
+ "awserrcode={}, errtype={}, reqid={}",
ase.getMessage(),
ase.getStatusCode(),
ase.getErrorCode(),
ase.getErrorType(),
ase.getRequestId());
}
/**
* Retrieves an instruction file from S3. If no instruction file is found, returns null.
*
* @param getObjectRequest A GET request for an object in S3. The parameters from this request
* will be used to retrieve the corresponding instruction file.
* @return An instruction file, or null if no instruction file was found.
*/
private S3Object getInstructionFile(GetObjectRequest getObjectRequest) {
try {
GetObjectRequest instructionFileRequest =
EncryptionUtils.createInstructionGetRequest(getObjectRequest);
return super.getObject(instructionFileRequest);
} catch (AmazonServiceException e) {
// If no instruction file is found, log a debug message, and return null.
log.debug("Unable to retrieve instruction file : " + e.getMessage());
return null;
}
}
public static void deleteTaskMessage(Message msg, String sqsUrl, AmazonSQS sqs) {
String handle = msg.getReceiptHandle();
try {
sqs.deleteMessage(new DeleteMessageRequest(sqsUrl, handle));
logger.info("Message deleted: " + msg.getBody());
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
}
}
public boolean uploadToS3(String fileName, String text) throws IOException {
try {
String folderName = this.p.getProperty("KEY_NAME");
String bucketName = this.p.getProperty("AWS_BUCKET_NAME");
logger.info(LogKey.MESSAGE, "Uploading a new object to S3 from a file");
byte[] contentAsBytes = text.getBytes("UTF-8");
ByteArrayInputStream contentsAsStream = new ByteArrayInputStream(contentAsBytes);
ObjectMetadata md = new ObjectMetadata();
md.setContentLength(contentAsBytes.length);
if (this.s3client == null) {
logger.debug(LogKey.MESSAGE, "reuse S3 client !!");
this.s3client = this.getS3Client();
}
logger.info(LogKey.BUCKET_NAME, bucketName);
logger.info(LogKey.KEY_NAME, folderName);
logger.info(LogKey.FILE_NAME, fileName);
this.s3client.putObject(
new PutObjectRequest(
bucketName, folderName + File.separator + fileName, contentsAsStream, md));
return true;
} catch (AmazonServiceException ase) {
logger.error(LogKey.EXCEPTION, ase);
logger.error(
LogKey.EXCEPTION,
"Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response"
+ " for some reason.");
logger.error(LogKey.EXCEPTION, "Error Message: " + ase.getMessage());
logger.error(LogKey.EXCEPTION, "HTTP Status Code: " + ase.getStatusCode());
logger.error(LogKey.EXCEPTION, "AWS Error Code: " + ase.getErrorCode());
logger.error(LogKey.EXCEPTION, "Error Type: " + ase.getErrorType());
logger.error(LogKey.EXCEPTION, "Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
logger.error(
LogKey.EXCEPTION,
"Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network.");
logger.error(LogKey.EXCEPTION, "Error Message: " + ace.getMessage());
}
return false;
}
// Appends a S3 http:// and bucket address to file path.
public static String generateS3FileAddress(AmazonS3 s3, String path) {
String address;
try {
address = s3.getBucketLocation(bucket);
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
return null;
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
return null;
}
return "https://s3-" + address + ".amazonaws.com/" + bucket + "/" + path;
}
@Action(
value = "/manage/submitEC2Key",
results = {
@Result(name = "input", location = "/manage/view_ec2_keys.jsp"),
@Result(name = "success", location = "/manage/viewEC2Keys.action", type = "redirect")
})
public String submitEC2Key() {
String retVal = SUCCESS;
try {
// get AWS credentials from DB
AWSCred awsCred = AWSCredDB.getAWSCred();
// set AWS credentials for service
BasicAWSCredentials awsCredentials =
new BasicAWSCredentials(awsCred.getAccessKey(), awsCred.getSecretKey());
// create service
AmazonEC2 service = new AmazonEC2Client(awsCredentials);
service.setEndpoint(ec2Key.getEc2Region());
// create key pair request
CreateKeyPairRequest createKeyPairRequest = new CreateKeyPairRequest();
createKeyPairRequest.withKeyName(ec2Key.getKeyNm());
// call service
CreateKeyPairResult createKeyPairResult = service.createKeyPair(createKeyPairRequest);
// get key pair result
KeyPair keyPair = createKeyPairResult.getKeyPair();
// set private key
String privateKey = keyPair.getKeyMaterial();
ec2Key.setPrivateKey(privateKey);
// add to db
Long keyId = EC2KeyDB.saveEC2Key(ec2Key);
// store private key
SSHUtil.storePrivateKey(keyId.toString(), ec2Key.getPrivateKey().trim());
} catch (AmazonServiceException ex) {
addActionError(ex.getMessage());
retVal = INPUT;
}
return retVal;
}
@Action(
value = "/manage/importEC2Key",
results = {
@Result(name = "input", location = "/manage/view_ec2_keys.jsp"),
@Result(name = "success", location = "/manage/viewEC2Keys.action", type = "redirect")
})
public String importEC2Key() {
String retVal = SUCCESS;
try {
// get AWS credentials from DB
AWSCred awsCred = AWSCredDB.getAWSCred();
// set AWS credentials for service
BasicAWSCredentials awsCredentials =
new BasicAWSCredentials(awsCred.getAccessKey(), awsCred.getSecretKey());
// create service
AmazonEC2 service = new AmazonEC2Client(awsCredentials);
service.setEndpoint(ec2Key.getEc2Region());
// describe key pair request
DescribeKeyPairsRequest describeKeyPairsRequest = new DescribeKeyPairsRequest();
describeKeyPairsRequest.setKeyNames(Arrays.asList(ec2Key.getKeyNm()));
// call service
DescribeKeyPairsResult describeKeyPairsResult =
service.describeKeyPairs(describeKeyPairsRequest);
if (describeKeyPairsResult != null && describeKeyPairsResult.getKeyPairs().size() > 0) {
// add to db
Long keyId = EC2KeyDB.saveEC2Key(ec2Key);
SSHUtil.storePrivateKey(keyId.toString(), ec2Key.getPrivateKey().trim());
} else {
addActionError("Imported key does not exist on AWS");
retVal = INPUT;
}
} catch (AmazonServiceException ex) {
addActionError(ex.getMessage());
retVal = INPUT;
}
return retVal;
}
@Override
public int delete(String table, String key) {
logger.debug("deletekey: " + key + " from table: " + table);
DeleteItemRequest req = new DeleteItemRequest(table, createPrimaryKey(key));
DeleteItemResult res = null;
try {
res = dynamoDB.deleteItem(req);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
return OK;
}
// Makes a putObjectRequest to make file public, and sends request.
private static boolean putObject(AmazonS3 s3, PutObjectRequest req) {
// Set file as public.
req.withCannedAcl(CannedAccessControlList.PublicRead);
// Send upload request.
try {
s3.putObject(req);
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
return false;
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
return false;
}
return true;
}
public static List<Message> getMessages(ReceiveMessageRequest req, AmazonSQS sqs) {
logger.fine("Getting SQS messages.");
List<Message> msgs;
try {
msgs = sqs.receiveMessage(req).getMessages();
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
return null;
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
return null;
}
return msgs;
}
@Inject
public AmazonS3Storage(Configuration configuration) {
bucketName = configuration.getString("storage.s3.bucket");
String accessKey = configuration.getString("storage.s3.accesskey");
String secretKey = configuration.getString("storage.s3.secretkey");
credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 amazonS3 = new AmazonS3Client(credentials);
try {
if (!(amazonS3.doesBucketExist(bucketName))) {
amazonS3.createBucket(new CreateBucketRequest(bucketName));
}
String bucketLocation = amazonS3.getBucketLocation(new GetBucketLocationRequest(bucketName));
Logger.info("Amazon S3 bucket created at " + bucketLocation);
} catch (AmazonServiceException ase) {
Logger.error(
"Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response "
+ "for some reason."
+ " Error Message: "
+ ase.getMessage()
+ " HTTP Status Code: "
+ ase.getStatusCode()
+ " AWS Error Code: "
+ ase.getErrorCode()
+ " Error Type: "
+ ase.getErrorType()
+ " Request ID: "
+ ase.getRequestId());
} catch (AmazonClientException ace) {
Logger.error(
"Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network."
+ " Error Message: "
+ ace.getMessage());
}
}
public static String xs3_generate_url(String xs3_objname, String content_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
java.util.Date expiration = new java.util.Date();
long milliSeconds = expiration.getTime();
milliSeconds += 1000 * 60 * 5;
expiration.setTime(milliSeconds);
GeneratePresignedUrlRequest xs3_genurl_req =
new GeneratePresignedUrlRequest(xs3_bucketname, xs3_objname);
xs3_genurl_req.setMethod(HttpMethod.PUT);
xs3_genurl_req.setExpiration(expiration);
xs3_genurl_req.setContentType(content_type);
xs3_genurl_req.addRequestParameter("x-amz-acl", "public-read");
URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
System.out.println(url.toString());
return url.toString();
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
@Override
public int insert(String table, String key, HashMap<String, ByteIterator> values) {
logger.debug("insertkey: " + primaryKeyName + "-" + key + " from table: " + table);
Map<String, AttributeValue> attributes = createAttributes(values);
// adding primary key
attributes.put(primaryKeyName, new AttributeValue(key));
PutItemRequest putItemRequest = new PutItemRequest(table, attributes);
PutItemResult res = null;
try {
res = dynamoDB.putItem(putItemRequest);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
return OK;
}
public static String xs3_init_multi_upload(String xs3_objname, int file_size, String file_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
InitiateMultipartUploadRequest xs3_multi_req =
new InitiateMultipartUploadRequest(xs3_bucketname, xs3_objname);
xs3_multi_req.setCannedACL(CannedAccessControlList.PublicRead);
ObjectMetadata xs3_meta = new ObjectMetadata();
xs3_meta.setContentType(file_type);
xs3_multi_req.setObjectMetadata(xs3_meta);
InitiateMultipartUploadResult xs3_multi_res =
xs3_client.initiateMultipartUpload(xs3_multi_req);
String xs3_multi_uploadid = xs3_multi_res.getUploadId();
String json_urls = gen_part_url(xs3_multi_uploadid, file_size, xs3_objname, file_type);
return json_urls;
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
public static void xs3_coplete_multi_upload(String xs3_objname, String uploadId) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
List<PartETag> rest_parts = listPartsXml(xs3_objname, uploadId);
if (null == rest_parts) {
return;
}
for (PartETag item : rest_parts) {
System.out.println(item.getETag() + " -> " + item.getPartNumber());
}
CompleteMultipartUploadRequest comp_req =
new CompleteMultipartUploadRequest(xs3_bucketname, xs3_objname, uploadId, rest_parts);
CompleteMultipartUploadResult comp_result = xs3_client.completeMultipartUpload(comp_req);
System.out.println(comp_result.getETag());
System.out.println(comp_result.getKey());
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
}
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
try {
String myQueueUrl = "https://queue.amazonaws.com/034307772076/MyQueue";
AmazonSQS sqs =
new AmazonSQSClient(
new PropertiesCredentials(
SimpleQueueServiceSample.class.getResourceAsStream("AwsCredentials.properties")));
System.out.println("Sending message 1.");
sqs.sendMessage(new SendMessageRequest(myQueueUrl, "Test message."));
System.out.println("Receiving messages:");
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl);
List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages();
for (Message message : messages) {
System.out.print(message.getMessageId());
System.out.println(" : " + message.getBody());
}
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon SQS, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with SQS, such as not "
+ "being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
@Override
public int update(String table, String key, HashMap<String, ByteIterator> values) {
logger.debug("updatekey: " + key + " from table: " + table);
Map<String, AttributeValueUpdate> attributes =
new HashMap<String, AttributeValueUpdate>(values.size());
for (Entry<String, ByteIterator> val : values.entrySet()) {
AttributeValue v = new AttributeValue(val.getValue().toString());
attributes.put(val.getKey(), new AttributeValueUpdate().withValue(v).withAction("PUT"));
}
UpdateItemRequest req = new UpdateItemRequest(table, createPrimaryKey(key), attributes);
try {
dynamoDB.updateItem(req);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
return OK;
}
/** download the content of the file 'id' */
public byte[] downloadData(String bucketName, String fileId, String[] canonicalIDs)
throws StorageCloudException {
try {
S3Object object = null;
if (bucketName == null) {
object = conn.getObject(new GetObjectRequest(defaultBucketName, fileId));
} else {
bucketName = bucketName.concat(location);
object = conn.getObject(new GetObjectRequest(bucketName, fileId));
}
byte[] array = getBytesFromInputStream(object.getObjectContent());
object.getObjectContent().close();
return array;
} catch (AmazonServiceException e1) {
throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
} catch (AmazonClientException e2) {
throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
} catch (IOException e3) {
e3.printStackTrace();
throw new StorageCloudException("AWSS3Exception::" + e3.getMessage());
}
}
/** writes the value 'data' in the file 'id' */
public String uploadData(String bucketName, byte[] data, String fileId, String[] canonicalIDs)
throws StorageCloudException {
try {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(data.length);
ByteArrayInputStream in = new ByteArrayInputStream(data);
if (bucketName != null) {
bucketName = bucketName.concat(location);
if (!conn.doesBucketExist(bucketName)) {
conn.createBucket(bucketName, region);
}
if (canonicalIDs != null) {
AccessControlList acl = new AccessControlList();
for (int i = 0; i < canonicalIDs.length; i++) {
acl.grantPermission(new CanonicalGrantee(canonicalIDs[i]), Permission.Read);
}
conn.putObject(
new PutObjectRequest(bucketName, fileId, in, metadata).withAccessControlList(acl));
} else {
conn.putObject(new PutObjectRequest(bucketName, fileId, in, metadata));
}
} else {
conn.putObject(new PutObjectRequest(defaultBucketName, fileId, in, metadata));
}
in.close();
return fileId;
} catch (AmazonServiceException e1) {
throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
} catch (AmazonClientException e2) {
throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
} catch (IOException e3) {
e3.printStackTrace();
throw new StorageCloudException("AWSS3Exception::" + e3.getMessage());
}
}
public static void main(String[] args) throws Exception {
System.out.println("===========================================");
System.out.println("Welcome to the AWS Java SDK!");
System.out.println("===========================================");
init();
try {
/*
* The Amazon EC2 client allows you to easily launch and configure
* computing capacity in AWS datacenters.
*
* In this sample, we use the EC2 client to list the availability zones
* in a region, and then list the instances running in those zones.
*/
DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
List<AvailabilityZone> availabilityZones = availabilityZonesResult.getAvailabilityZones();
System.out.println("You have access to " + availabilityZones.size() + " availability zones:");
for (AvailabilityZone zone : availabilityZones) {
System.out.println(" - " + zone.getZoneName() + " (" + zone.getRegionName() + ")");
}
DescribeInstancesResult describeInstancesResult = ec2.describeInstances();
Set<Instance> instances = new HashSet<Instance>();
for (Reservation reservation : describeInstancesResult.getReservations()) {
instances.addAll(reservation.getInstances());
}
System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");
/*
* The Amazon S3 client allows you to manage and configure buckets
* and to upload and download data.
*
* In this sample, we use the S3 client to list all the buckets in
* your account, and then iterate over the object metadata for all
* objects in one bucket to calculate the total object count and
* space usage for that one bucket. Note that this sample only
* retrieves the object's metadata and doesn't actually download the
* object's content.
*
* In addition to the low-level Amazon S3 client in the SDK, there
* is also a high-level TransferManager API that provides
* asynchronous management of uploads and downloads with an easy to
* use API:
* http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/transfer/TransferManager.html
*/
List<Bucket> buckets = s3.listBuckets();
System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s).");
if (buckets.size() > 0) {
Bucket bucket = buckets.get(0);
long totalSize = 0;
long totalItems = 0;
/*
* The S3Objects and S3Versions classes provide convenient APIs
* for iterating over the contents of your buckets, without
* having to manually deal with response pagination.
*/
for (S3ObjectSummary objectSummary : S3Objects.inBucket(s3, bucket.getName())) {
totalSize += objectSummary.getSize();
totalItems++;
}
System.out.println(
"The bucket '"
+ bucket.getName()
+ "' contains "
+ totalItems
+ " objects "
+ "with a total size of "
+ totalSize
+ " bytes.");
}
} catch (AmazonServiceException ase) {
/*
* AmazonServiceExceptions represent an error response from an AWS
* services, i.e. your request made it to AWS, but the AWS service
* either found it invalid or encountered an error trying to execute
* it.
*/
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
/*
* AmazonClientExceptions represent an error that occurred inside
* the client on the local host, either while trying to send the
* request to AWS or interpret the response. For example, if no
* network connection is available, the client won't be able to
* connect to AWS to execute a request and will throw an
* AmazonClientException.
*/
System.out.println("Error Message: " + ace.getMessage());
}
}
public static String gen_part_url(
String uploadId, int file_size, String file_name, String file_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
final int xs3_part_size = 1024 * 1024 * 5;
int xs3_part_count = (int) Math.ceil((double) (file_size) / (double) xs3_part_size);
JSONArray jsonArray = new JSONArray();
JSONObject jsonObject_1 = new JSONObject();
jsonObject_1.put("total_num", xs3_part_count);
jsonObject_1.put("upload_id", uploadId);
JSONArray jsonArray_sub = new JSONArray();
for (int part_no = 0; part_no < xs3_part_count; part_no++) {
long xs3_offset_bytes = xs3_part_size * part_no;
long part_size =
xs3_part_size < (file_size - xs3_offset_bytes)
? xs3_part_size
: (file_size - xs3_offset_bytes);
java.util.Date expiration = new java.util.Date();
long milliSeconds = expiration.getTime();
milliSeconds += 1000 * 60 * 5;
expiration.setTime(milliSeconds);
GeneratePresignedUrlRequest xs3_genurl_req =
new GeneratePresignedUrlRequest(xs3_bucketname, file_name);
xs3_genurl_req.setMethod(HttpMethod.PUT);
xs3_genurl_req.setExpiration(expiration);
xs3_genurl_req.setContentType(file_type);
xs3_genurl_req.addRequestParameter("uploadId", uploadId);
xs3_genurl_req.addRequestParameter("partNumber", String.valueOf(part_no + 1));
URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
System.out.println(url.toString());
JSONObject jsonObject = new JSONObject();
jsonObject.put("part_idx", part_no);
jsonObject.put("part_url", url.toString());
jsonObject.put("upload_len", part_size);
jsonObject.put("part_begin", xs3_offset_bytes);
jsonObject.put("part_end", xs3_offset_bytes + part_size);
jsonArray_sub.add(jsonObject);
}
jsonObject_1.put("multi_list", jsonArray_sub);
jsonArray.add(jsonObject_1);
return jsonArray.toString();
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
