private void updateItemVersionedInternal(
String tableName,
Key key,
Map<String, AttributeValueUpdate> attributes,
String expectedVersion,
PersistentEntity persistentEntity,
int attempt)
throws DataAccessException {
UpdateItemRequest request =
new UpdateItemRequest(tableName, key, attributes)
.withExpected(getOptimisticVersionCondition(expectedVersion));
try {
ddb.updateItem(request);
} catch (AmazonServiceException e) {
if (DynamoDBUtil.AWS_ERR_CODE_CONDITIONAL_CHECK_FAILED.equals(e.getErrorCode())) {
throw new OptimisticLockingException(persistentEntity, key);
} else if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
throw new IllegalArgumentException("no such table: " + tableName, e);
} else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
// retry after a small pause
DynamoDBUtil.sleepBeforeRetry(attempt);
attempt++;
updateItemVersionedInternal(
tableName, key, attributes, expectedVersion, persistentEntity, attempt);
} else {
throw new DataStoreOperationException(
"problem with table: " + tableName + ", key: " + key + ", attributes: " + attributes,
e);
}
}
}
private static void createKey(String keyName, AmazonEC2 ec2) {
try {
List<KeyPairInfo> keyPairList = ec2.describeKeyPairs().getKeyPairs();
for (KeyPairInfo keyPair : keyPairList) {
if (keyName.equalsIgnoreCase(keyPair.getKeyName())) {
System.out.println("Using key " + keyName);
return;
}
}
System.out.println("Creating key " + keyName + "in local directory");
CreateKeyPairRequest newKeyRequest = new CreateKeyPairRequest();
newKeyRequest.setKeyName(keyName);
CreateKeyPairResult keyresult = ec2.createKeyPair(newKeyRequest);
KeyPair keyPair = new KeyPair();
keyPair = keyresult.getKeyPair();
String privateKey = keyPair.getKeyMaterial();
writeKeytoFile(keyName, privateKey);
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
}
}
private static void waitForTableToBecomeAvailable(AmazonDynamoDB dynamo, String tableName) {
System.out.println("Waiting for " + tableName + " to become ACTIVE...");
long startTime = System.currentTimeMillis();
long endTime = startTime + (10 * 60 * 1000);
while (System.currentTimeMillis() < endTime) {
try {
Thread.sleep(1000 * 20);
} catch (Exception e) {
}
try {
DescribeTableRequest request = new DescribeTableRequest().withTableName(tableName);
TableDescription table = dynamo.describeTable(request).getTable();
if (table == null) continue;
String tableStatus = table.getTableStatus();
System.out.println(" - current state: " + tableStatus);
if (tableStatus.equals(TableStatus.ACTIVE.toString())) return;
} catch (AmazonServiceException ase) {
if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) throw ase;
}
}
throw new RuntimeException("Table " + tableName + " never went active");
}
private Map<String, AttributeValue> getConsistentInternal(
String tableName, Key key, int attempt) {
GetItemRequest request = new GetItemRequest(tableName, key);
request.setConsistentRead(true);
try {
GetItemResult result = ddb.getItem(request);
Map<String, AttributeValue> attributes = result.getItem();
if (attributes == null || attributes.isEmpty()) {
return null;
}
return attributes;
} catch (AmazonServiceException e) {
if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
throw new IllegalArgumentException("no such table: " + tableName, e);
} else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
// retry after a small pause
DynamoDBUtil.sleepBeforeRetry(attempt);
attempt++;
return getConsistentInternal(tableName, key, attempt);
} else {
throw new DataStoreOperationException(
"problem with table: " + tableName + ", key: " + key, e);
}
}
}
public static void createBucket(AmazonS3Client s3, String bucketName, String zone) {
try {
List<Bucket> bucketList = s3.listBuckets();
for (Bucket bucket : bucketList) {
// System.out.println(bucket.getName());
if (bucketName.equalsIgnoreCase(bucket.getName())) {
System.out.println("Using bucket " + bucketName);
return;
}
}
System.out.println("Created s3 bucket " + bucketName);
s3.createBucket(bucketName);
return;
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
}
}
private static void basicAbortMPU() throws IOException {
System.out.println("basic abort MPU");
String bucketName = "chttest";
String fileName = "hello.txt";
// String uploadID = "XHGTFV4F5XTEAC5O8N3LK12TIY3DSY7OFPXIWTHRMNTE7A3WB5M8N2U5AN"; //hi
String uploadID = "LE5JS2K6C208JU7ZX1QD2TVRWXOWWF4VNG7LE7TFIX5SYNG4HLOGW9CLAD"; // hello
AbortMultipartUploadRequest request =
new AbortMultipartUploadRequest(bucketName, fileName, uploadID);
AmazonS3 s3 =
new AmazonS3Client(
new PropertiesCredentials(
putBucket.class.getResourceAsStream("AwsCredentials.properties")));
try {
s3.abortMultipartUpload(request);
System.out.println();
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
private boolean isServiceSignUpException(Exception e) {
if (e instanceof AmazonServiceException) {
AmazonServiceException ase = (AmazonServiceException) e;
return "OptInRequired".equalsIgnoreCase(ase.getErrorCode());
}
return false;
}
public boolean wipeCredentialsOnAuthError(AmazonServiceException ex) {
if (
// STS
// http://docs.amazonwebservices.com/STS/latest/APIReference/CommonErrors.html
ex.getErrorCode().equals("IncompleteSignature")
|| ex.getErrorCode().equals("InternalFailure")
|| ex.getErrorCode().equals("InvalidClientTokenId")
|| ex.getErrorCode().equals("OptInRequired")
|| ex.getErrorCode().equals("RequestExpired")
|| ex.getErrorCode().equals("ServiceUnavailable")
// DynamoDB
// http://docs.amazonwebservices.com/amazondynamodb/latest/developerguide/ErrorHandling.html#APIErrorTypes
|| ex.getErrorCode().equals("AccessDeniedException")
|| ex.getErrorCode().equals("IncompleteSignatureException")
|| ex.getErrorCode().equals("MissingAuthenticationTokenException")
|| ex.getErrorCode().equals("ValidationException")
|| ex.getErrorCode().equals("InternalFailure")
|| ex.getErrorCode().equals("InternalServerError")) {
clearCredentials();
return true;
}
return false;
}
public void storeMessageInQueue(String message) {
// System.out.println("Storing Message...");
// Send a message
if (driectDbConnection) {
DbConnector db = null;
try {
db = dbf.createDBConnectorInstance();
} catch (MailAppDBException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
MailAppMessage mailMsg =
new MailAppMessage(message, "" + System.currentTimeMillis() + "_" + UUID.randomUUID());
try {
// printDebug("Calling DB-Instance to store message "+mailMsg.getTimeuuid()+" length:
// "+mailMsg.getSize()+"\n"+mailMsg.getWholeMessageWITHOUT2LINES());
db.storeMessage(mailMsg);
} catch (MailAppDBException mae) {
// MailStoringWorkerStarter.stopMailStoringWorker(MailStoringWorkerStarter.getWorkerList(), this);
// stop=true;
mae.printStackTrace();
}
} else {
try {
sqs.sendMessage(new SendMessageRequest(mailQueueUrl, message));
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon SQS, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with SQS, such as not "
+ "being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
printDebug("Message stored to " + mailQueueUrl);
// connect to db ( Cassandra ) and store message
// OR write Message to SQS to be consumed by Storage Worker ->
// better? more scalable, db independent
}
}
private void logException(AmazonServiceException ase) {
logger.error(
"AmazonServiceException: error={}, statuscode={}, "
+ "awserrcode={}, errtype={}, reqid={}",
ase.getMessage(),
ase.getStatusCode(),
ase.getErrorCode(),
ase.getErrorType(),
ase.getRequestId());
}
@Then("^I expect the response error code to be \"(.+?)\"$")
public void then_I_expect_response_error_code(String expected) {
assertNotNull(exception);
assertTrue(exception instanceof AmazonServiceException);
String actual = exception.getErrorCode();
assertEquals(
"Error code doesn't match. Expected : " + expected + ". Actual :" + actual,
expected,
actual);
}
private static boolean doesTableExist(AmazonDynamoDB dynamo, String tableName) {
try {
TableDescription table =
dynamo.describeTable(new DescribeTableRequest().withTableName(tableName)).getTable();
return "ACTIVE".equals(table.getTableStatus());
} catch (AmazonServiceException ase) {
if (ase.getErrorCode().equals("ResourceNotFoundException")) return false;
throw ase;
}
}
/** {@inheritDoc} */
@Override
public void terminateInstance(String instanceId) {
try {
ec2Client().terminateInstances(new TerminateInstancesRequest(Arrays.asList(instanceId)));
} catch (AmazonServiceException e) {
if (e.getErrorCode().equals("InvalidInstanceID.NotFound")) {
throw new NotFoundException("AWS instance " + instanceId + " not found", e);
}
throw e;
}
}
public boolean uploadToS3(String fileName, String text) throws IOException {
try {
String folderName = this.p.getProperty("KEY_NAME");
String bucketName = this.p.getProperty("AWS_BUCKET_NAME");
logger.info(LogKey.MESSAGE, "Uploading a new object to S3 from a file");
byte[] contentAsBytes = text.getBytes("UTF-8");
ByteArrayInputStream contentsAsStream = new ByteArrayInputStream(contentAsBytes);
ObjectMetadata md = new ObjectMetadata();
md.setContentLength(contentAsBytes.length);
if (this.s3client == null) {
logger.debug(LogKey.MESSAGE, "reuse S3 client !!");
this.s3client = this.getS3Client();
}
logger.info(LogKey.BUCKET_NAME, bucketName);
logger.info(LogKey.KEY_NAME, folderName);
logger.info(LogKey.FILE_NAME, fileName);
this.s3client.putObject(
new PutObjectRequest(
bucketName, folderName + File.separator + fileName, contentsAsStream, md));
return true;
} catch (AmazonServiceException ase) {
logger.error(LogKey.EXCEPTION, ase);
logger.error(
LogKey.EXCEPTION,
"Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response"
+ " for some reason.");
logger.error(LogKey.EXCEPTION, "Error Message: " + ase.getMessage());
logger.error(LogKey.EXCEPTION, "HTTP Status Code: " + ase.getStatusCode());
logger.error(LogKey.EXCEPTION, "AWS Error Code: " + ase.getErrorCode());
logger.error(LogKey.EXCEPTION, "Error Type: " + ase.getErrorType());
logger.error(LogKey.EXCEPTION, "Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
logger.error(
LogKey.EXCEPTION,
"Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network.");
logger.error(LogKey.EXCEPTION, "Error Message: " + ace.getMessage());
}
return false;
}
protected Void doInBackground(Void... voids) {
try {
SimpleDB.createItem(domainName.getSelectedItem().toString(), itemName.getText().toString());
} catch (AmazonServiceException e) {
if ("InvalidClientTokenId".equals(e.getErrorCode())) {
putRefreshError();
} else {
setStackAndPost(e);
}
} catch (Throwable e) {
setStackAndPost(e);
}
return null;
}
/** Verifies the RetryCondition has collected the expected context information. */
public static void verifyExpectedContextData(
ContextDataCollection contextDataCollection,
AmazonWebServiceRequest failedRequest,
AmazonClientException expectedException,
int expectedRetries) {
Assert.assertEquals(expectedRetries, contextDataCollection.failedRequests.size());
Assert.assertEquals(expectedRetries, contextDataCollection.exceptions.size());
Assert.assertEquals(expectedRetries, contextDataCollection.retriesAttemptedValues.size());
if (expectedRetries > 0) {
// It should keep getting the same original request instance
for (AmazonWebServiceRequest seenRequest : contextDataCollection.failedRequests) {
Assert.assertTrue(seenRequest == failedRequest);
}
// Verify the exceptions
if (expectedException instanceof AmazonServiceException) {
// It should get service exceptions with the expected error and
// status code
AmazonServiceException ase = (AmazonServiceException) expectedException;
for (AmazonClientException seenException : contextDataCollection.exceptions) {
Assert.assertTrue(seenException instanceof AmazonServiceException);
Assert.assertEquals(
ase.getErrorCode(), ((AmazonServiceException) seenException).getErrorCode());
Assert.assertEquals(
ase.getStatusCode(), ((AmazonServiceException) seenException).getStatusCode());
}
} else {
// Client exceptions should have the same expected cause (the
// same
// throwable instance from the mock HttpClient).
Throwable expectedCause = expectedException.getCause();
for (AmazonClientException seenException : contextDataCollection.exceptions) {
Assert.assertTrue(expectedCause == seenException.getCause());
}
}
// It should get "retriesAttempted" values starting from 0
int expectedRetriesAttempted = 0;
for (int seenRetriesAttempted : contextDataCollection.retriesAttemptedValues) {
Assert.assertEquals(expectedRetriesAttempted++, seenRetriesAttempted);
}
}
}
private void deleteItemInternal(String tableName, Key key, int attempt) {
DeleteItemRequest request = new DeleteItemRequest(tableName, key);
try {
ddb.deleteItem(request);
} catch (AmazonServiceException e) {
if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
throw new IllegalArgumentException("no such table: " + tableName, e);
} else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
// retry after a small pause
DynamoDBUtil.sleepBeforeRetry(attempt);
attempt++;
deleteItemInternal(tableName, key, attempt);
} else {
throw new DataStoreOperationException(
"problem with table: " + tableName + ", key: " + key, e);
}
}
}
/**
* Update the tags stored in EC2 with the specified information. Re-try 5 times if instances isn't
* up by catchErrorCode - e.g. InvalidSpotInstanceRequestID.NotFound or
* InvalidInstanceRequestID.NotFound
*
* @param ec2
* @param instTags
* @param catchErrorCode
* @param params
* @throws InterruptedException
*/
private void updateRemoteTags(
AmazonEC2 ec2, Collection<Tag> instTags, String catchErrorCode, String... params)
throws InterruptedException {
for (int i = 0; i < 5; i++) {
try {
CreateTagsRequest tagRequest = new CreateTagsRequest();
tagRequest.withResources(params).setTags(instTags);
ec2.createTags(tagRequest);
break;
} catch (AmazonServiceException e) {
if (e.getErrorCode().equals(catchErrorCode)) {
Thread.sleep(5000);
continue;
}
LOGGER.log(Level.SEVERE, e.getErrorMessage(), e);
}
}
}
@Inject
public AmazonS3Storage(Configuration configuration) {
bucketName = configuration.getString("storage.s3.bucket");
String accessKey = configuration.getString("storage.s3.accesskey");
String secretKey = configuration.getString("storage.s3.secretkey");
credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 amazonS3 = new AmazonS3Client(credentials);
try {
if (!(amazonS3.doesBucketExist(bucketName))) {
amazonS3.createBucket(new CreateBucketRequest(bucketName));
}
String bucketLocation = amazonS3.getBucketLocation(new GetBucketLocationRequest(bucketName));
Logger.info("Amazon S3 bucket created at " + bucketLocation);
} catch (AmazonServiceException ase) {
Logger.error(
"Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response "
+ "for some reason."
+ " Error Message: "
+ ase.getMessage()
+ " HTTP Status Code: "
+ ase.getStatusCode()
+ " AWS Error Code: "
+ ase.getErrorCode()
+ " Error Type: "
+ ase.getErrorType()
+ " Request ID: "
+ ase.getRequestId());
} catch (AmazonClientException ace) {
Logger.error(
"Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network."
+ " Error Message: "
+ ace.getMessage());
}
}
public static String xs3_generate_url(String xs3_objname, String content_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
java.util.Date expiration = new java.util.Date();
long milliSeconds = expiration.getTime();
milliSeconds += 1000 * 60 * 5;
expiration.setTime(milliSeconds);
GeneratePresignedUrlRequest xs3_genurl_req =
new GeneratePresignedUrlRequest(xs3_bucketname, xs3_objname);
xs3_genurl_req.setMethod(HttpMethod.PUT);
xs3_genurl_req.setExpiration(expiration);
xs3_genurl_req.setContentType(content_type);
xs3_genurl_req.addRequestParameter("x-amz-acl", "public-read");
URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
System.out.println(url.toString());
return url.toString();
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
private List<Map<String, AttributeValue>> scanInternal(
String tableName, Map<String, Condition> filter, int max, int attempt) {
LinkedList<Map<String, AttributeValue>> items = new LinkedList<Map<String, AttributeValue>>();
try {
ScanRequest request = new ScanRequest(tableName).withScanFilter(filter);
ScanResult result = ddb.scan(request);
items.addAll(result.getItems());
// keep repeating until we get through all matched items
Key lastKeyEvaluated = null;
do {
lastKeyEvaluated = result.getLastEvaluatedKey();
if (lastKeyEvaluated != null) {
request =
new ScanRequest(tableName)
.withScanFilter(filter)
.withExclusiveStartKey(lastKeyEvaluated);
result = ddb.scan(request);
items.addAll(result.getItems());
}
} while (lastKeyEvaluated != null && items.size() < max);
// truncate if needed
while (items.size() > max) {
items.removeLast();
}
return items;
} catch (AmazonServiceException e) {
if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
throw new IllegalArgumentException("no such table: " + tableName, e);
} else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
// retry after a small pause
DynamoDBUtil.sleepBeforeRetry(attempt);
attempt++;
return scanInternal(tableName, filter, max, attempt);
} else {
throw new DataStoreOperationException(
"problem with table: " + tableName + ", filter: " + filter, e);
}
}
}
private void putItemInternal(
String tableName, Map<String, AttributeValue> attributes, int attempt)
throws DataAccessException {
try {
PutItemRequest request = new PutItemRequest(tableName, attributes);
ddb.putItem(request);
} catch (AmazonServiceException e) {
if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
throw new IllegalArgumentException("no such table: " + tableName, e);
} else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
// retry after a small pause
DynamoDBUtil.sleepBeforeRetry(attempt);
attempt++;
putItemInternal(tableName, attributes, attempt);
} else {
throw new DataStoreOperationException(
"problem with table: " + tableName + ", attributes: " + attributes, e);
}
}
}
public static String xs3_init_multi_upload(String xs3_objname, int file_size, String file_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
InitiateMultipartUploadRequest xs3_multi_req =
new InitiateMultipartUploadRequest(xs3_bucketname, xs3_objname);
xs3_multi_req.setCannedACL(CannedAccessControlList.PublicRead);
ObjectMetadata xs3_meta = new ObjectMetadata();
xs3_meta.setContentType(file_type);
xs3_multi_req.setObjectMetadata(xs3_meta);
InitiateMultipartUploadResult xs3_multi_res =
xs3_client.initiateMultipartUpload(xs3_multi_req);
String xs3_multi_uploadid = xs3_multi_res.getUploadId();
String json_urls = gen_part_url(xs3_multi_uploadid, file_size, xs3_objname, file_type);
return json_urls;
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
public static void xs3_coplete_multi_upload(String xs3_objname, String uploadId) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
List<PartETag> rest_parts = listPartsXml(xs3_objname, uploadId);
if (null == rest_parts) {
return;
}
for (PartETag item : rest_parts) {
System.out.println(item.getETag() + " -> " + item.getPartNumber());
}
CompleteMultipartUploadRequest comp_req =
new CompleteMultipartUploadRequest(xs3_bucketname, xs3_objname, uploadId, rest_parts);
CompleteMultipartUploadResult comp_result = xs3_client.completeMultipartUpload(comp_req);
System.out.println(comp_result.getETag());
System.out.println(comp_result.getKey());
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
}
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
try {
String myQueueUrl = "https://queue.amazonaws.com/034307772076/MyQueue";
AmazonSQS sqs =
new AmazonSQSClient(
new PropertiesCredentials(
SimpleQueueServiceSample.class.getResourceAsStream("AwsCredentials.properties")));
System.out.println("Sending message 1.");
sqs.sendMessage(new SendMessageRequest(myQueueUrl, "Test message."));
System.out.println("Receiving messages:");
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl);
List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages();
for (Message message : messages) {
System.out.print(message.getMessageId());
System.out.println(" : " + message.getBody());
}
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon SQS, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with SQS, such as not "
+ "being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
public static void main(String[] args) throws IOException {
/*
* This credentials provider implementation loads your AWS credentials
* from a properties file at the root of your classpath.
*
* Important: Be sure to fill in your AWS access credentials in the
* AwsCredentials.properties file before you try to run this
* sample.
* http://aws.amazon.com/security-credentials
*/
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
String key = "MyObjectKey";
System.out.println("===========================================");
System.out.println("Getting Started with Amazon S3");
System.out.println("===========================================\n");
try {
/*
* Create a new S3 bucket - Amazon S3 bucket names are globally unique,
* so once a bucket name has been taken by any user, you can't create
* another bucket with that same name.
*
* You can optionally specify a location for your bucket if you want to
* keep your data closer to your applications or users.
*/
System.out.println("Creating bucket " + bucketName + "\n");
s3.createBucket(bucketName);
/*
* List the buckets in your account
*/
System.out.println("Listing buckets");
for (Bucket bucket : s3.listBuckets()) {
System.out.println(" - " + bucket.getName());
}
System.out.println();
/*
* Upload an object to your bucket - You can easily upload a file to
* S3, or upload directly an InputStream if you know the length of
* the data in the stream. You can also specify your own metadata
* when uploading to S3, which allows you set a variety of options
* like content-type and content-encoding, plus additional metadata
* specific to your applications.
*/
System.out.println("Uploading a new object to S3 from a file\n");
s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));
/*
* Download an object - When you download an object, you get all of
* the object's metadata and a stream from which to read the contents.
* It's important to read the contents of the stream as quickly as
* possibly since the data is streamed directly from Amazon S3 and your
* network connection will remain open until you read all the data or
* close the input stream.
*
* GetObjectRequest also supports several other options, including
* conditional downloading of objects based on modification times,
* ETags, and selectively downloading a range of an object.
*/
System.out.println("Downloading an object");
S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
displayTextInputStream(object.getObjectContent());
/*
* List objects in your bucket by prefix - There are many options for
* listing the objects in your bucket. Keep in mind that buckets with
* many objects might truncate their results when listing their objects,
* so be sure to check if the returned object listing is truncated, and
* use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
* additional results.
*/
System.out.println("Listing objects");
ObjectListing objectListing =
s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
System.out.println(
" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")");
}
System.out.println();
/*
* Delete an object - Unless versioning has been turned on for your bucket,
* there is no way to undelete an object, so use caution when deleting objects.
*/
System.out.println("Deleting an object\n");
s3.deleteObject(bucketName, key);
/*
* Delete a bucket - A bucket must be completely empty before it can be
* deleted, so remember to delete any objects from your buckets before
* you try to delete them.
*/
System.out.println("Deleting bucket " + bucketName + "\n");
s3.deleteBucket(bucketName);
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
private void jButton1ActionPerformed(
java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jButton1ActionPerformed
// TODO add your handling code here: upload file
JFileChooser jfc = new JFileChooser();
jfc.setAcceptAllFileFilterUsed(false);
jfc.setFileFilter(new FileNameExtensionFilter("Text Files", "txt"));
int returnVal = jfc.showOpenDialog(this);
if (returnVal == JFileChooser.APPROVE_OPTION) {
File file = jfc.getSelectedFile();
try {
filepath = file.getCanonicalPath();
FileReader fr = new FileReader(filepath);
BufferedReader br = new BufferedReader(fr);
String line;
int total_no_of_words = 0;
while ((line = br.readLine()) != null) {
String[] words_arr = line.split(" ");
total_no_of_words += words_arr.length;
}
jLabel4.setText("word count :" + total_no_of_words);
System.out.println("Total no of words = " + total_no_of_words);
if (total_no_of_words != 0) {
File userFile = new File(filepath);
keyName = userFile.getName();
String myAccessKeyID = "AKIAI7PATIX6LHJA3A3Q";
String mySecretKey = "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";
AWSCredentials myCredentials = new BasicAWSCredentials(myAccessKeyID, mySecretKey);
ClientConfiguration clientConfig = new ClientConfiguration();
clientConfig.setProtocol(Protocol.HTTP);
AmazonS3 conn = new AmazonS3Client(myCredentials, clientConfig);
try {
File file1 = new File(filepath);
file_size = file1.length();
if (file_size < 100000) {
conn.putObject(new PutObjectRequest(bucketName, keyName, file1));
jLabel2.setText("Uploading complete !");
} else {
jLabel2.setText("Uploading failed size greater than 100kb...");
}
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response"
+ " for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
} /*catch (InterruptedException ex) {
Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
}*/
} else {
jLabel2.setText("Uploading failed file empty !");
}
} catch (IOException ex) {
Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
}
}
if (returnVal == JFileChooser.CANCEL_OPTION) {
System.exit(0);
}
} // GEN-LAST:event_jButton1ActionPerformed
public static void main(String[] args) throws Exception {
System.out.println("===========================================");
System.out.println("Welcome to the AWS Java SDK!");
System.out.println("===========================================");
init();
try {
/*
* The Amazon EC2 client allows you to easily launch and configure
* computing capacity in AWS datacenters.
*
* In this sample, we use the EC2 client to list the availability zones
* in a region, and then list the instances running in those zones.
*/
DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
List<AvailabilityZone> availabilityZones = availabilityZonesResult.getAvailabilityZones();
System.out.println("You have access to " + availabilityZones.size() + " availability zones:");
for (AvailabilityZone zone : availabilityZones) {
System.out.println(" - " + zone.getZoneName() + " (" + zone.getRegionName() + ")");
}
DescribeInstancesResult describeInstancesResult = ec2.describeInstances();
Set<Instance> instances = new HashSet<Instance>();
for (Reservation reservation : describeInstancesResult.getReservations()) {
instances.addAll(reservation.getInstances());
}
System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");
/*
* The Amazon S3 client allows you to manage and configure buckets
* and to upload and download data.
*
* In this sample, we use the S3 client to list all the buckets in
* your account, and then iterate over the object metadata for all
* objects in one bucket to calculate the total object count and
* space usage for that one bucket. Note that this sample only
* retrieves the object's metadata and doesn't actually download the
* object's content.
*
* In addition to the low-level Amazon S3 client in the SDK, there
* is also a high-level TransferManager API that provides
* asynchronous management of uploads and downloads with an easy to
* use API:
* http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/transfer/TransferManager.html
*/
List<Bucket> buckets = s3.listBuckets();
System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s).");
if (buckets.size() > 0) {
Bucket bucket = buckets.get(0);
long totalSize = 0;
long totalItems = 0;
/*
* The S3Objects and S3Versions classes provide convenient APIs
* for iterating over the contents of your buckets, without
* having to manually deal with response pagination.
*/
for (S3ObjectSummary objectSummary : S3Objects.inBucket(s3, bucket.getName())) {
totalSize += objectSummary.getSize();
totalItems++;
}
System.out.println(
"The bucket '"
+ bucket.getName()
+ "' contains "
+ totalItems
+ " objects "
+ "with a total size of "
+ totalSize
+ " bytes.");
}
} catch (AmazonServiceException ase) {
/*
* AmazonServiceExceptions represent an error response from an AWS
* services, i.e. your request made it to AWS, but the AWS service
* either found it invalid or encountered an error trying to execute
* it.
*/
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
/*
* AmazonClientExceptions represent an error that occurred inside
* the client on the local host, either while trying to send the
* request to AWS or interpret the response. For example, if no
* network connection is available, the client won't be able to
* connect to AWS to execute a request and will throw an
* AmazonClientException.
*/
System.out.println("Error Message: " + ace.getMessage());
}
}
/**
* Returns true if the specified exception is a throttling error.
*
* @param ase The exception to test.
* @return True if the exception resulted from a throttling error message from a service,
* otherwise false.
*/
private boolean isThrottlingException(AmazonServiceException ase) {
if (ase == null) return false;
return "Throttling".equals(ase.getErrorCode())
|| "ThrottlingException".equals(ase.getErrorCode())
|| "ProvisionedThroughputExceededException".equals(ase.getErrorCode());
}
public static String gen_part_url(
String uploadId, int file_size, String file_name, String file_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
final int xs3_part_size = 1024 * 1024 * 5;
int xs3_part_count = (int) Math.ceil((double) (file_size) / (double) xs3_part_size);
JSONArray jsonArray = new JSONArray();
JSONObject jsonObject_1 = new JSONObject();
jsonObject_1.put("total_num", xs3_part_count);
jsonObject_1.put("upload_id", uploadId);
JSONArray jsonArray_sub = new JSONArray();
for (int part_no = 0; part_no < xs3_part_count; part_no++) {
long xs3_offset_bytes = xs3_part_size * part_no;
long part_size =
xs3_part_size < (file_size - xs3_offset_bytes)
? xs3_part_size
: (file_size - xs3_offset_bytes);
java.util.Date expiration = new java.util.Date();
long milliSeconds = expiration.getTime();
milliSeconds += 1000 * 60 * 5;
expiration.setTime(milliSeconds);
GeneratePresignedUrlRequest xs3_genurl_req =
new GeneratePresignedUrlRequest(xs3_bucketname, file_name);
xs3_genurl_req.setMethod(HttpMethod.PUT);
xs3_genurl_req.setExpiration(expiration);
xs3_genurl_req.setContentType(file_type);
xs3_genurl_req.addRequestParameter("uploadId", uploadId);
xs3_genurl_req.addRequestParameter("partNumber", String.valueOf(part_no + 1));
URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
System.out.println(url.toString());
JSONObject jsonObject = new JSONObject();
jsonObject.put("part_idx", part_no);
jsonObject.put("part_url", url.toString());
jsonObject.put("upload_len", part_size);
jsonObject.put("part_begin", xs3_offset_bytes);
jsonObject.put("part_end", xs3_offset_bytes + part_size);
jsonArray_sub.add(jsonObject);
}
jsonObject_1.put("multi_list", jsonArray_sub);
jsonArray.add(jsonObject_1);
return jsonArray.toString();
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
