private static void createKey(String keyName, AmazonEC2 ec2) {
try {
List<KeyPairInfo> keyPairList = ec2.describeKeyPairs().getKeyPairs();
for (KeyPairInfo keyPair : keyPairList) {
if (keyName.equalsIgnoreCase(keyPair.getKeyName())) {
System.out.println("Using key " + keyName);
return;
}
}
System.out.println("Creating key " + keyName + "in local directory");
CreateKeyPairRequest newKeyRequest = new CreateKeyPairRequest();
newKeyRequest.setKeyName(keyName);
CreateKeyPairResult keyresult = ec2.createKeyPair(newKeyRequest);
KeyPair keyPair = new KeyPair();
keyPair = keyresult.getKeyPair();
String privateKey = keyPair.getKeyMaterial();
writeKeytoFile(keyName, privateKey);
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
}
}
public static void createBucket(AmazonS3Client s3, String bucketName, String zone) {
try {
List<Bucket> bucketList = s3.listBuckets();
for (Bucket bucket : bucketList) {
// System.out.println(bucket.getName());
if (bucketName.equalsIgnoreCase(bucket.getName())) {
System.out.println("Using bucket " + bucketName);
return;
}
}
System.out.println("Created s3 bucket " + bucketName);
s3.createBucket(bucketName);
return;
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
}
}
private static void basicAbortMPU() throws IOException {
System.out.println("basic abort MPU");
String bucketName = "chttest";
String fileName = "hello.txt";
// String uploadID = "XHGTFV4F5XTEAC5O8N3LK12TIY3DSY7OFPXIWTHRMNTE7A3WB5M8N2U5AN"; //hi
String uploadID = "LE5JS2K6C208JU7ZX1QD2TVRWXOWWF4VNG7LE7TFIX5SYNG4HLOGW9CLAD"; // hello
AbortMultipartUploadRequest request =
new AbortMultipartUploadRequest(bucketName, fileName, uploadID);
AmazonS3 s3 =
new AmazonS3Client(
new PropertiesCredentials(
putBucket.class.getResourceAsStream("AwsCredentials.properties")));
try {
s3.abortMultipartUpload(request);
System.out.println();
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
public void storeMessageInQueue(String message) {
// System.out.println("Storing Message...");
// Send a message
if (driectDbConnection) {
DbConnector db = null;
try {
db = dbf.createDBConnectorInstance();
} catch (MailAppDBException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
MailAppMessage mailMsg =
new MailAppMessage(message, "" + System.currentTimeMillis() + "_" + UUID.randomUUID());
try {
// printDebug("Calling DB-Instance to store message "+mailMsg.getTimeuuid()+" length:
// "+mailMsg.getSize()+"\n"+mailMsg.getWholeMessageWITHOUT2LINES());
db.storeMessage(mailMsg);
} catch (MailAppDBException mae) {
// MailStoringWorkerStarter.stopMailStoringWorker(MailStoringWorkerStarter.getWorkerList(), this);
// stop=true;
mae.printStackTrace();
}
} else {
try {
sqs.sendMessage(new SendMessageRequest(mailQueueUrl, message));
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon SQS, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with SQS, such as not "
+ "being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
printDebug("Message stored to " + mailQueueUrl);
// connect to db ( Cassandra ) and store message
// OR write Message to SQS to be consumed by Storage Worker ->
// better? more scalable, db independent
}
}
private void logException(AmazonServiceException ase) {
logger.error(
"AmazonServiceException: error={}, statuscode={}, "
+ "awserrcode={}, errtype={}, reqid={}",
ase.getMessage(),
ase.getStatusCode(),
ase.getErrorCode(),
ase.getErrorType(),
ase.getRequestId());
}
public boolean uploadToS3(String fileName, String text) throws IOException {
try {
String folderName = this.p.getProperty("KEY_NAME");
String bucketName = this.p.getProperty("AWS_BUCKET_NAME");
logger.info(LogKey.MESSAGE, "Uploading a new object to S3 from a file");
byte[] contentAsBytes = text.getBytes("UTF-8");
ByteArrayInputStream contentsAsStream = new ByteArrayInputStream(contentAsBytes);
ObjectMetadata md = new ObjectMetadata();
md.setContentLength(contentAsBytes.length);
if (this.s3client == null) {
logger.debug(LogKey.MESSAGE, "reuse S3 client !!");
this.s3client = this.getS3Client();
}
logger.info(LogKey.BUCKET_NAME, bucketName);
logger.info(LogKey.KEY_NAME, folderName);
logger.info(LogKey.FILE_NAME, fileName);
this.s3client.putObject(
new PutObjectRequest(
bucketName, folderName + File.separator + fileName, contentsAsStream, md));
return true;
} catch (AmazonServiceException ase) {
logger.error(LogKey.EXCEPTION, ase);
logger.error(
LogKey.EXCEPTION,
"Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response"
+ " for some reason.");
logger.error(LogKey.EXCEPTION, "Error Message: " + ase.getMessage());
logger.error(LogKey.EXCEPTION, "HTTP Status Code: " + ase.getStatusCode());
logger.error(LogKey.EXCEPTION, "AWS Error Code: " + ase.getErrorCode());
logger.error(LogKey.EXCEPTION, "Error Type: " + ase.getErrorType());
logger.error(LogKey.EXCEPTION, "Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
logger.error(
LogKey.EXCEPTION,
"Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network.");
logger.error(LogKey.EXCEPTION, "Error Message: " + ace.getMessage());
}
return false;
}
@Override
public void delete(final TranscodingJob transcodingJob) {
final ImmutableMap.Builder<String, AttributeValue> keyBuilder = ImmutableMap.builder();
keyBuilder.put("transcode-job-id", new AttributeValue(transcodingJob.getId()));
try {
this.dynamoDB.deleteItem(TABLE_NAME, keyBuilder.build());
} catch (final AmazonServiceException e) {
logger.error(
"AmazonServiceException while storing {}. RID: {}", transcodingJob, e.getRequestId(), e);
} catch (final AmazonClientException e) {
logger.error("AmazonClientException while storing {}.", transcodingJob, e);
}
}
@Override
public void store(final TranscodingJob transcodingJob) {
final ImmutableMap.Builder<String, AttributeValue> itemBuilder = ImmutableMap.builder();
itemBuilder.put("transcode-job-id", new AttributeValue(transcodingJob.getId()));
itemBuilder.put("inputMovie", new AttributeValue(transcodingJob.getInputPath().toString()));
itemBuilder.put("outputMovie", new AttributeValue(transcodingJob.getOutputPath().toString()));
try {
this.dynamoDB.putItem(TABLE_NAME, itemBuilder.build());
} catch (final AmazonServiceException e) {
logger.error(
"AmazonServiceException while storing {}. RID: {}", transcodingJob, e.getRequestId(), e);
} catch (final AmazonClientException e) {
logger.error("AmazonClientException while storing {}.", transcodingJob, e);
}
}
@Inject
public AmazonS3Storage(Configuration configuration) {
bucketName = configuration.getString("storage.s3.bucket");
String accessKey = configuration.getString("storage.s3.accesskey");
String secretKey = configuration.getString("storage.s3.secretkey");
credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 amazonS3 = new AmazonS3Client(credentials);
try {
if (!(amazonS3.doesBucketExist(bucketName))) {
amazonS3.createBucket(new CreateBucketRequest(bucketName));
}
String bucketLocation = amazonS3.getBucketLocation(new GetBucketLocationRequest(bucketName));
Logger.info("Amazon S3 bucket created at " + bucketLocation);
} catch (AmazonServiceException ase) {
Logger.error(
"Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response "
+ "for some reason."
+ " Error Message: "
+ ase.getMessage()
+ " HTTP Status Code: "
+ ase.getStatusCode()
+ " AWS Error Code: "
+ ase.getErrorCode()
+ " Error Type: "
+ ase.getErrorType()
+ " Request ID: "
+ ase.getRequestId());
} catch (AmazonClientException ace) {
Logger.error(
"Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network."
+ " Error Message: "
+ ace.getMessage());
}
}
@Override
public TranscodingJob get(final String jobId) {
final ImmutableMap.Builder<String, AttributeValue> keyBuilder = ImmutableMap.builder();
keyBuilder.put("transcode-job-id", new AttributeValue(jobId));
try {
final GetItemResult result = this.dynamoDB.getItem(TABLE_NAME, keyBuilder.build());
return new TranscodingJob(
result.getItem().get("transcode-job-id").getS(),
Paths.get(result.getItem().get("inputMovie").getS()),
Paths.get(result.getItem().get("outputMovie").getS()));
} catch (final AmazonServiceException e) {
logger.error("AmazonServiceException while getting {}. RID: {}", jobId, e.getRequestId(), e);
throw e;
} catch (final AmazonClientException e) {
logger.error("AmazonClientException while getting {}.", jobId, e);
throw e;
}
}
public static String xs3_generate_url(String xs3_objname, String content_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
java.util.Date expiration = new java.util.Date();
long milliSeconds = expiration.getTime();
milliSeconds += 1000 * 60 * 5;
expiration.setTime(milliSeconds);
GeneratePresignedUrlRequest xs3_genurl_req =
new GeneratePresignedUrlRequest(xs3_bucketname, xs3_objname);
xs3_genurl_req.setMethod(HttpMethod.PUT);
xs3_genurl_req.setExpiration(expiration);
xs3_genurl_req.setContentType(content_type);
xs3_genurl_req.addRequestParameter("x-amz-acl", "public-read");
URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
System.out.println(url.toString());
return url.toString();
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
public static String xs3_init_multi_upload(String xs3_objname, int file_size, String file_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
InitiateMultipartUploadRequest xs3_multi_req =
new InitiateMultipartUploadRequest(xs3_bucketname, xs3_objname);
xs3_multi_req.setCannedACL(CannedAccessControlList.PublicRead);
ObjectMetadata xs3_meta = new ObjectMetadata();
xs3_meta.setContentType(file_type);
xs3_multi_req.setObjectMetadata(xs3_meta);
InitiateMultipartUploadResult xs3_multi_res =
xs3_client.initiateMultipartUpload(xs3_multi_req);
String xs3_multi_uploadid = xs3_multi_res.getUploadId();
String json_urls = gen_part_url(xs3_multi_uploadid, file_size, xs3_objname, file_type);
return json_urls;
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
public static void xs3_coplete_multi_upload(String xs3_objname, String uploadId) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
List<PartETag> rest_parts = listPartsXml(xs3_objname, uploadId);
if (null == rest_parts) {
return;
}
for (PartETag item : rest_parts) {
System.out.println(item.getETag() + " -> " + item.getPartNumber());
}
CompleteMultipartUploadRequest comp_req =
new CompleteMultipartUploadRequest(xs3_bucketname, xs3_objname, uploadId, rest_parts);
CompleteMultipartUploadResult comp_result = xs3_client.completeMultipartUpload(comp_req);
System.out.println(comp_result.getETag());
System.out.println(comp_result.getKey());
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
}
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
try {
String myQueueUrl = "https://queue.amazonaws.com/034307772076/MyQueue";
AmazonSQS sqs =
new AmazonSQSClient(
new PropertiesCredentials(
SimpleQueueServiceSample.class.getResourceAsStream("AwsCredentials.properties")));
System.out.println("Sending message 1.");
sqs.sendMessage(new SendMessageRequest(myQueueUrl, "Test message."));
System.out.println("Receiving messages:");
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl);
List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages();
for (Message message : messages) {
System.out.print(message.getMessageId());
System.out.println(" : " + message.getBody());
}
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon SQS, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with SQS, such as not "
+ "being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
/** @param args */
public static void main(String[] args) {
// ============================================================================================//
// =============================== Submitting a Request
// =======================================//
// ============================================================================================//
// Retrieves the credentials from an AWSCredentials.properties file.
AWSCredentials credentials = null;
try {
credentials =
new PropertiesCredentials(
InlineTaggingCodeSampleApp.class.getResourceAsStream("AwsCredentials.properties"));
} catch (IOException e1) {
System.out.println("Credentials were not properly entered into AwsCredentials.properties.");
System.out.println(e1.getMessage());
System.exit(-1);
}
// Create the AmazonEC2Client object so we can call various APIs.
AmazonEC2 ec2 = new AmazonEC2Client(credentials);
// Initializes a Spot Instance Request
RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest();
// Request 1 x t1.micro instance with a bid price of $0.03.
requestRequest.setSpotPrice("0.03");
requestRequest.setInstanceCount(Integer.valueOf(1));
// Setup the specifications of the launch. This includes the instance type (e.g. t1.micro)
// and the latest Amazon Linux AMI id available. Note, you should always use the latest
// Amazon Linux AMI id or another of your choosing.
LaunchSpecification launchSpecification = new LaunchSpecification();
launchSpecification.setImageId("ami-8c1fece5");
launchSpecification.setInstanceType("t1.micro");
// Add the security group to the request.
ArrayList<String> securityGroups = new ArrayList<String>();
securityGroups.add("GettingStartedGroup");
launchSpecification.setSecurityGroups(securityGroups);
// Add the launch specifications to the request.
requestRequest.setLaunchSpecification(launchSpecification);
// ============================================================================================//
// =========================== Getting the Request ID from the Request
// ========================//
// ============================================================================================//
// Call the RequestSpotInstance API.
RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest);
List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();
// Setup an arraylist to collect all of the request ids we want to watch hit the running
// state.
ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();
// Add all of the request ids to the hashset, so we can determine when they hit the
// active state.
for (SpotInstanceRequest requestResponse : requestResponses) {
System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId());
spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId());
}
// ============================================================================================//
// ====================================== Tag the Spot Requests
// ===============================//
// ============================================================================================//
// Create the list of tags we want to create
ArrayList<Tag> requestTags = new ArrayList<Tag>();
requestTags.add(new Tag("keyname1", "value1"));
// Create a tag request for requests.
CreateTagsRequest createTagsRequest_requests = new CreateTagsRequest();
createTagsRequest_requests.setResources(spotInstanceRequestIds);
createTagsRequest_requests.setTags(requestTags);
// Try to tag the Spot request submitted.
try {
ec2.createTags(createTagsRequest_requests);
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.out.println("Error terminating instances");
System.out.println("Caught Exception: " + e.getMessage());
System.out.println("Reponse Status Code: " + e.getStatusCode());
System.out.println("Error Code: " + e.getErrorCode());
System.out.println("Request ID: " + e.getRequestId());
}
// ============================================================================================//
// =========================== Determining the State of the Spot Request
// ======================//
// ============================================================================================//
// Create a variable that will track whether there are any requests still in the open state.
boolean anyOpen;
// Initialize variables.
ArrayList<String> instanceIds = new ArrayList<String>();
do {
// Create the describeRequest with tall of the request id to monitor (e.g. that we started).
DescribeSpotInstanceRequestsRequest describeRequest =
new DescribeSpotInstanceRequestsRequest();
describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);
// Initialize the anyOpen variable to false – which assumes there are no requests open unless
// we find one that is still open.
anyOpen = false;
try {
// Retrieve all of the requests we want to monitor.
DescribeSpotInstanceRequestsResult describeResult =
ec2.describeSpotInstanceRequests(describeRequest);
List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();
// Look through each request and determine if they are all in the active state.
for (SpotInstanceRequest describeResponse : describeResponses) {
// If the state is open, it hasn't changed since we attempted to request it.
// There is the potential for it to transition almost immediately to closed or
// cancelled so we compare against open instead of active.
if (describeResponse.getState().equals("open")) {
anyOpen = true;
break;
}
// Add the instance id to the list we will eventually terminate.
instanceIds.add(describeResponse.getInstanceId());
}
} catch (AmazonServiceException e) {
// If we have an exception, ensure we don't break out of the loop.
// This prevents the scenario where there was blip on the wire.
anyOpen = true;
}
try {
// Sleep for 60 seconds.
Thread.sleep(60 * 1000);
} catch (Exception e) {
// Do nothing because it woke up early.
}
} while (anyOpen);
// ============================================================================================//
// ====================================== Tag the Spot Instances
// ===============================//
// ============================================================================================//
// Create the list of tags we want to create
ArrayList<Tag> instanceTags = new ArrayList<Tag>();
instanceTags.add(new Tag("keyname1", "value1"));
// Create a tag request for instances.
CreateTagsRequest createTagsRequest_instances = new CreateTagsRequest();
createTagsRequest_instances.setResources(instanceIds);
createTagsRequest_instances.setTags(instanceTags);
// Try to tag the Spot instance started.
try {
ec2.createTags(createTagsRequest_instances);
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.out.println("Error terminating instances");
System.out.println("Caught Exception: " + e.getMessage());
System.out.println("Reponse Status Code: " + e.getStatusCode());
System.out.println("Error Code: " + e.getErrorCode());
System.out.println("Request ID: " + e.getRequestId());
}
// ============================================================================================//
// ====================================== Canceling the Request ==============================//
// ============================================================================================//
try {
// Cancel requests.
CancelSpotInstanceRequestsRequest cancelRequest =
new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds);
ec2.cancelSpotInstanceRequests(cancelRequest);
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.out.println("Error cancelling instances");
System.out.println("Caught Exception: " + e.getMessage());
System.out.println("Reponse Status Code: " + e.getStatusCode());
System.out.println("Error Code: " + e.getErrorCode());
System.out.println("Request ID: " + e.getRequestId());
}
// ============================================================================================//
// =================================== Terminating any Instances
// ==============================//
// ============================================================================================//
try {
// Terminate instances.
TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds);
ec2.terminateInstances(terminateRequest);
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.out.println("Error terminating instances");
System.out.println("Caught Exception: " + e.getMessage());
System.out.println("Reponse Status Code: " + e.getStatusCode());
System.out.println("Error Code: " + e.getErrorCode());
System.out.println("Request ID: " + e.getRequestId());
}
}
public static void start() {
AWSCredentials credentials = null;
try {
credentials =
new PropertiesCredentials(
CreateInitialVMs.class.getResourceAsStream("AwsCredentials.properties"));
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
/**
* *******************************************
*
* <p>#1 Create Amazon Client object
*
* <p>*******************************************
*/
System.out.println("#1 Create Amazon Client object");
ec2 = new AmazonEC2Client(credentials);
try {
/**
* *******************************************
*
* <p>#6 Create an Instance
*
* <p>*******************************************
*/
System.out.println("#5 Create an Instance");
String imageId = "ami-76f0061f"; // Basic 32-bit Amazon Linux AMI
int minInstanceCount = 1;
int maxInstanceCount = 2; // create 2 instances
String keyPairName = "testElastic";
String securityGroupName = "Prachi";
ArrayList<String> securityGroup = new ArrayList<String>();
securityGroup.add(securityGroupName);
RunInstancesRequest rir =
new RunInstancesRequest(imageId, minInstanceCount, maxInstanceCount);
rir.setKeyName(keyPairName);
rir.setSecurityGroups(securityGroup);
Placement p = new Placement();
p.setAvailabilityZone("us-east-1a");
rir.setPlacement(p);
RunInstancesResult result = ec2.runInstances(rir);
// get instanceId from the result
List<Instance> resultInstance = result.getReservation().getInstances();
String createdInstanceId = null;
for (Instance ins : resultInstance) {
createdInstanceId = ins.getInstanceId();
System.out.println("State of instance " + ins.getInstanceId() + " : ");
while (true) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.print(ins.getState().getName() + "\n");
ins = updatedInstance(ins);
// if(ins.getPublicIpAddress()!= null){
// if(ins.getState().getName().equals("running")){
if (ins.getPublicIpAddress() != null && ins.getState().getName().equals("running")) {
break;
}
}
System.out.println("State: " + ins.getState().getName());
System.out.println("New instance has been created: " + ins.getInstanceId());
System.out.println("Instance Key: " + ins.getKeyName());
System.out.println("Public DNS Name: " + ins.getPublicDnsName());
System.out.println("Public DNS IP Address: " + ins.getPublicIpAddress());
System.out.println("Instance ID: " + ins.getInstanceId());
VMNode vmn = new VMNode();
vmn.setInstanceId(ins.getInstanceId());
/**
* ******************************************* Allocate elastic IP addresses.
* *******************************************
*/
System.out.println("Public IP before association: " + ins.getPublicIpAddress());
ins = allocateElasticIP(ins);
vmn.setElasticIp(ins.getPublicIpAddress());
System.out.println("Public IP after association: " + ins.getPublicIpAddress());
/**
* ******************************************* Create a volume
* *******************************************
*/
String createdVolumeId = createVolume();
/**
* ******************************************* Attach the volume to the instance
* *******************************************
*/
attachVolume(createdInstanceId, createdVolumeId);
vmn.setVolumeId(createdVolumeId);
/**
* ******************************************* Detach the volume from the instance
* *******************************************
*/
detachVolume(createdInstanceId, createdVolumeId);
/**
* ********************************* Create an AMI from an instance
* *******************************
*/
String createdImageId = createAmi(createdInstanceId);
vmn.setAmiId(createdImageId);
nodes.add(vmn);
/** ********************************* Stop the instance ******************************* */
// stopInstance(ins.getInstanceId());
// Diassociate Elastic IP from instance
disassociateElasticIp(ins.getPublicIpAddress());
}
System.out.println("Listing Nodes: ");
for (VMNode n : nodes) {
System.out.println("Instance ID: " + n.getInstanceid());
System.out.println("Elastic IP: " + n.getElasticIp());
System.out.println("Volume ID: " + n.getVolumeId());
System.out.println("AMI ID: " + n.getAmiId());
System.out.println();
}
/**
* *******************************************
*
* <p>#10 shutdown client object
*
* <p>*******************************************
*/
ec2.shutdown();
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
}
}
public static String gen_part_url(
String uploadId, int file_size, String file_name, String file_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
final int xs3_part_size = 1024 * 1024 * 5;
int xs3_part_count = (int) Math.ceil((double) (file_size) / (double) xs3_part_size);
JSONArray jsonArray = new JSONArray();
JSONObject jsonObject_1 = new JSONObject();
jsonObject_1.put("total_num", xs3_part_count);
jsonObject_1.put("upload_id", uploadId);
JSONArray jsonArray_sub = new JSONArray();
for (int part_no = 0; part_no < xs3_part_count; part_no++) {
long xs3_offset_bytes = xs3_part_size * part_no;
long part_size =
xs3_part_size < (file_size - xs3_offset_bytes)
? xs3_part_size
: (file_size - xs3_offset_bytes);
java.util.Date expiration = new java.util.Date();
long milliSeconds = expiration.getTime();
milliSeconds += 1000 * 60 * 5;
expiration.setTime(milliSeconds);
GeneratePresignedUrlRequest xs3_genurl_req =
new GeneratePresignedUrlRequest(xs3_bucketname, file_name);
xs3_genurl_req.setMethod(HttpMethod.PUT);
xs3_genurl_req.setExpiration(expiration);
xs3_genurl_req.setContentType(file_type);
xs3_genurl_req.addRequestParameter("uploadId", uploadId);
xs3_genurl_req.addRequestParameter("partNumber", String.valueOf(part_no + 1));
URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
System.out.println(url.toString());
JSONObject jsonObject = new JSONObject();
jsonObject.put("part_idx", part_no);
jsonObject.put("part_url", url.toString());
jsonObject.put("upload_len", part_size);
jsonObject.put("part_begin", xs3_offset_bytes);
jsonObject.put("part_end", xs3_offset_bytes + part_size);
jsonArray_sub.add(jsonObject);
}
jsonObject_1.put("multi_list", jsonArray_sub);
jsonArray.add(jsonObject_1);
return jsonArray.toString();
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
/**
* Internal method to execute the HTTP method given.
*
* @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler)
* @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler,
* ExecutionContext)
*/
private <T> Response<T> executeHelper(
Request<?> request,
HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
HttpResponseHandler<AmazonServiceException> errorResponseHandler,
ExecutionContext executionContext)
throws AmazonClientException, AmazonServiceException {
/*
* Depending on which response handler we end up choosing to handle the
* HTTP response, it might require us to leave the underlying HTTP
* connection open, depending on whether or not it reads the complete
* HTTP response stream from the HTTP connection, or if delays reading
* any of the content until after a response is returned to the caller.
*/
boolean leaveHttpConnectionOpen = false;
AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
/* add the service endpoint to the logs. You can infer service name from service endpoint */
awsRequestMetrics.addProperty(Field.ServiceName, request.getServiceName());
awsRequestMetrics.addProperty(Field.ServiceEndpoint, request.getEndpoint());
// Apply whatever request options we know how to handle, such as user-agent.
setUserAgent(request);
int requestCount = 0;
URI redirectedURI = null;
HttpEntity entity = null;
AmazonClientException retriedException = null;
// Make a copy of the original request params and headers so that we can
// permute it in this loop and start over with the original every time.
Map<String, String> originalParameters = new LinkedHashMap<String, String>();
originalParameters.putAll(request.getParameters());
Map<String, String> originalHeaders = new HashMap<String, String>();
originalHeaders.putAll(request.getHeaders());
final AWSCredentials credentials = executionContext.getCredentials();
Signer signer = null;
while (true) {
++requestCount;
awsRequestMetrics.setCounter(Field.RequestCount, requestCount);
if (requestCount > 1) { // retry
request.setParameters(originalParameters);
request.setHeaders(originalHeaders);
}
HttpRequestBase httpRequest = null;
org.apache.http.HttpResponse apacheResponse = null;
try {
// Sign the request if a signer was provided
if (signer == null) signer = executionContext.getSignerByURI(request.getEndpoint());
if (signer != null && credentials != null) {
awsRequestMetrics.startEvent(Field.RequestSigningTime);
try {
signer.sign(request, credentials);
} finally {
awsRequestMetrics.endEvent(Field.RequestSigningTime);
}
}
if (requestLog.isDebugEnabled()) {
requestLog.debug("Sending Request: " + request.toString());
}
httpRequest = httpRequestFactory.createHttpRequest(request, config, executionContext);
if (httpRequest instanceof HttpEntityEnclosingRequest) {
entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
}
if (redirectedURI != null) {
httpRequest.setURI(redirectedURI);
}
if (requestCount > 1) { // retry
awsRequestMetrics.startEvent(Field.RetryPauseTime);
try {
pauseBeforeNextRetry(
request.getOriginalRequest(),
retriedException,
requestCount,
config.getRetryPolicy());
} finally {
awsRequestMetrics.endEvent(Field.RetryPauseTime);
}
}
if (entity != null) {
InputStream content = entity.getContent();
if (requestCount > 1) { // retry
if (content.markSupported()) {
content.reset();
content.mark(-1);
}
} else {
if (content.markSupported()) {
content.mark(-1);
}
}
}
captureConnectionPoolMetrics(httpClient.getConnectionManager(), awsRequestMetrics);
HttpContext httpContext = new BasicHttpContext();
httpContext.setAttribute(AWSRequestMetrics.class.getSimpleName(), awsRequestMetrics);
retriedException = null;
awsRequestMetrics.startEvent(Field.HttpRequestTime);
try {
apacheResponse = httpClient.execute(httpRequest, httpContext);
} finally {
awsRequestMetrics.endEvent(Field.HttpRequestTime);
}
if (isRequestSuccessful(apacheResponse)) {
awsRequestMetrics.addProperty(
Field.StatusCode, apacheResponse.getStatusLine().getStatusCode());
/*
* If we get back any 2xx status code, then we know we should
* treat the service call as successful.
*/
leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen();
HttpResponse httpResponse = createResponse(httpRequest, request, apacheResponse);
T response =
handleResponse(
request,
responseHandler,
httpRequest,
httpResponse,
apacheResponse,
executionContext);
return new Response<T>(response, httpResponse);
} else if (isTemporaryRedirect(apacheResponse)) {
/*
* S3 sends 307 Temporary Redirects if you try to delete an
* EU bucket from the US endpoint. If we get a 307, we'll
* point the HTTP method to the redirected location, and let
* the next retry deliver the request to the right location.
*/
Header[] locationHeaders = apacheResponse.getHeaders("location");
String redirectedLocation = locationHeaders[0].getValue();
log.debug("Redirecting to: " + redirectedLocation);
redirectedURI = URI.create(redirectedLocation);
httpRequest.setURI(redirectedURI);
awsRequestMetrics.addProperty(
Field.StatusCode, apacheResponse.getStatusLine().getStatusCode());
awsRequestMetrics.addProperty(Field.RedirectLocation, redirectedLocation);
awsRequestMetrics.addProperty(Field.AWSRequestID, null);
} else {
leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen();
AmazonServiceException ase =
handleErrorResponse(request, errorResponseHandler, httpRequest, apacheResponse);
awsRequestMetrics.addProperty(Field.AWSRequestID, ase.getRequestId());
awsRequestMetrics.addProperty(Field.AWSErrorCode, ase.getErrorCode());
awsRequestMetrics.addProperty(Field.StatusCode, ase.getStatusCode());
if (!shouldRetry(
request.getOriginalRequest(),
httpRequest,
ase,
requestCount,
config.getRetryPolicy())) {
throw ase;
}
// Cache the retryable exception
retriedException = ase;
/*
* Checking for clock skew error again because we don't want to set the
* global time offset for every service exception.
*/
if (RetryUtils.isClockSkewError(ase)) {
int timeOffset = parseClockSkewOffset(apacheResponse, ase);
SDKGlobalConfiguration.setGlobalTimeOffset(timeOffset);
}
resetRequestAfterError(request, ase);
}
} catch (IOException ioe) {
if (log.isInfoEnabled()) {
log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
}
awsRequestMetrics.incrementCounter(Field.Exception);
awsRequestMetrics.addProperty(Field.Exception, ioe);
awsRequestMetrics.addProperty(Field.AWSRequestID, null);
AmazonClientException ace =
new AmazonClientException("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
if (!shouldRetry(
request.getOriginalRequest(),
httpRequest,
ace,
requestCount,
config.getRetryPolicy())) {
throw ace;
}
// Cache the retryable exception
retriedException = ace;
resetRequestAfterError(request, ioe);
} catch (RuntimeException e) {
throw handleUnexpectedFailure(e, awsRequestMetrics);
} catch (Error e) {
throw handleUnexpectedFailure(e, awsRequestMetrics);
} finally {
/*
* Some response handlers need to manually manage the HTTP
* connection and will take care of releasing the connection on
* their own, but if this response handler doesn't need the
* connection left open, we go ahead and release the it to free
* up resources.
*/
if (!leaveHttpConnectionOpen) {
try {
if (apacheResponse != null
&& apacheResponse.getEntity() != null
&& apacheResponse.getEntity().getContent() != null) {
apacheResponse.getEntity().getContent().close();
}
} catch (IOException e) {
log.warn("Cannot close the response content.", e);
}
}
}
} /* end while (true) */
}
/**
* Internal method to execute the HTTP method given.
*
* @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler)
* @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler,
* ExecutionContext)
*/
private <T extends Object> T executeHelper(
Request<?> request,
HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
HttpResponseHandler<AmazonServiceException> errorResponseHandler,
ExecutionContext executionContext)
throws AmazonClientException, AmazonServiceException {
/*
* Depending on which response handler we end up choosing to handle the
* HTTP response, it might require us to leave the underlying HTTP
* connection open, depending on whether or not it reads the complete
* HTTP response stream from the HTTP connection, or if delays reading
* any of the content until after a response is returned to the caller.
*/
boolean leaveHttpConnectionOpen = false;
AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
/* add the service endpoint to the logs. You can infer service name from service endpoint */
awsRequestMetrics.addProperty(Field.ServiceName.name(), request.getServiceName());
awsRequestMetrics.addProperty(Field.ServiceEndpoint.name(), request.getEndpoint());
// Apply whatever request options we know how to handle, such as user-agent.
applyRequestData(request);
int retryCount = 0;
URI redirectedURI = null;
HttpEntity entity = null;
AmazonServiceException exception = null;
// Make a copy of the original request params and headers so that we can
// permute it in this loop and start over with the original every time.
Map<String, String> originalParameters = new HashMap<String, String>();
originalParameters.putAll(request.getParameters());
Map<String, String> originalHeaders = new HashMap<String, String>();
originalHeaders.putAll(request.getHeaders());
while (true) {
awsRequestMetrics.setCounter(Field.AttemptCount.name(), retryCount + 1);
if (retryCount > 0) {
request.setParameters(originalParameters);
request.setHeaders(originalHeaders);
}
HttpRequestBase httpRequest = null;
org.apache.http.HttpResponse response = null;
try {
// Sign the request if a signer was provided
if (executionContext.getSigner() != null && executionContext.getCredentials() != null) {
awsRequestMetrics.startEvent(Field.RequestSigningTime.name());
executionContext.getSigner().sign(request, executionContext.getCredentials());
awsRequestMetrics.endEvent(Field.RequestSigningTime.name());
}
if (requestLog.isDebugEnabled()) {
requestLog.debug("Sending Request: " + request.toString());
}
httpRequest =
httpRequestFactory.createHttpRequest(request, config, entity, executionContext);
if (httpRequest instanceof HttpEntityEnclosingRequest) {
entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
}
if (redirectedURI != null) {
httpRequest.setURI(redirectedURI);
}
if (retryCount > 0) {
awsRequestMetrics.startEvent(Field.RetryPauseTime.name());
pauseExponentially(retryCount, exception, executionContext.getCustomBackoffStrategy());
awsRequestMetrics.endEvent(Field.RetryPauseTime.name());
}
if (entity != null) {
InputStream content = entity.getContent();
if (retryCount > 0) {
if (content.markSupported()) {
content.reset();
content.mark(-1);
}
} else {
if (content.markSupported()) {
content.mark(-1);
}
}
}
exception = null;
awsRequestMetrics.startEvent(Field.HttpRequestTime.name());
response = httpClient.execute(httpRequest);
awsRequestMetrics.endEvent(Field.HttpRequestTime.name());
if (isRequestSuccessful(response)) {
awsRequestMetrics.addProperty(
Field.StatusCode.name(), response.getStatusLine().getStatusCode());
/*
* If we get back any 2xx status code, then we know we should
* treat the service call as successful.
*/
leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen();
return handleResponse(request, responseHandler, httpRequest, response, executionContext);
} else if (isTemporaryRedirect(response)) {
/*
* S3 sends 307 Temporary Redirects if you try to delete an
* EU bucket from the US endpoint. If we get a 307, we'll
* point the HTTP method to the redirected location, and let
* the next retry deliver the request to the right location.
*/
Header[] locationHeaders = response.getHeaders("location");
String redirectedLocation = locationHeaders[0].getValue();
log.debug("Redirecting to: " + redirectedLocation);
redirectedURI = URI.create(redirectedLocation);
httpRequest.setURI(redirectedURI);
awsRequestMetrics.addProperty(
Field.StatusCode.name(), response.getStatusLine().getStatusCode());
awsRequestMetrics.addProperty(Field.RedirectLocation.name(), redirectedLocation);
awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null);
} else {
leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen();
exception = handleErrorResponse(request, errorResponseHandler, httpRequest, response);
awsRequestMetrics.addProperty(Field.AWSRequestID.name(), exception.getRequestId());
awsRequestMetrics.addProperty(Field.AWSErrorCode.name(), exception.getErrorCode());
awsRequestMetrics.addProperty(Field.StatusCode.name(), exception.getStatusCode());
if (!shouldRetry(httpRequest, exception, retryCount)) {
throw exception;
}
resetRequestAfterError(request, exception);
}
} catch (IOException ioe) {
log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
awsRequestMetrics.addProperty(Field.Exception.name(), ioe.toString());
awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null);
if (!shouldRetry(httpRequest, ioe, retryCount)) {
throw new AmazonClientException(
"Unable to execute HTTP request: " + ioe.getMessage(), ioe);
}
resetRequestAfterError(request, ioe);
} finally {
retryCount++;
/*
* Some response handlers need to manually manage the HTTP
* connection and will take care of releasing the connection on
* their own, but if this response handler doesn't need the
* connection left open, we go ahead and release the it to free
* up resources.
*/
if (!leaveHttpConnectionOpen) {
try {
response.getEntity().getContent().close();
} catch (Throwable t) {
}
}
}
} /* end while (true) */
}
public static void main(String[] args) throws Exception {
System.out.println("===========================================");
System.out.println("Welcome to the AWS Java SDK!");
System.out.println("===========================================");
init();
try {
/*
* The Amazon EC2 client allows you to easily launch and configure
* computing capacity in AWS datacenters.
*
* In this sample, we use the EC2 client to list the availability zones
* in a region, and then list the instances running in those zones.
*/
DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
List<AvailabilityZone> availabilityZones = availabilityZonesResult.getAvailabilityZones();
System.out.println("You have access to " + availabilityZones.size() + " availability zones:");
for (AvailabilityZone zone : availabilityZones) {
System.out.println(" - " + zone.getZoneName() + " (" + zone.getRegionName() + ")");
}
DescribeInstancesResult describeInstancesResult = ec2.describeInstances();
Set<Instance> instances = new HashSet<Instance>();
for (Reservation reservation : describeInstancesResult.getReservations()) {
instances.addAll(reservation.getInstances());
}
System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");
/*
* The Amazon S3 client allows you to manage and configure buckets
* and to upload and download data.
*
* In this sample, we use the S3 client to list all the buckets in
* your account, and then iterate over the object metadata for all
* objects in one bucket to calculate the total object count and
* space usage for that one bucket. Note that this sample only
* retrieves the object's metadata and doesn't actually download the
* object's content.
*
* In addition to the low-level Amazon S3 client in the SDK, there
* is also a high-level TransferManager API that provides
* asynchronous management of uploads and downloads with an easy to
* use API:
* http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/transfer/TransferManager.html
*/
List<Bucket> buckets = s3.listBuckets();
System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s).");
if (buckets.size() > 0) {
Bucket bucket = buckets.get(0);
long totalSize = 0;
long totalItems = 0;
/*
* The S3Objects and S3Versions classes provide convenient APIs
* for iterating over the contents of your buckets, without
* having to manually deal with response pagination.
*/
for (S3ObjectSummary objectSummary : S3Objects.inBucket(s3, bucket.getName())) {
totalSize += objectSummary.getSize();
totalItems++;
}
System.out.println(
"The bucket '"
+ bucket.getName()
+ "' contains "
+ totalItems
+ " objects "
+ "with a total size of "
+ totalSize
+ " bytes.");
}
} catch (AmazonServiceException ase) {
/*
* AmazonServiceExceptions represent an error response from an AWS
* services, i.e. your request made it to AWS, but the AWS service
* either found it invalid or encountered an error trying to execute
* it.
*/
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
/*
* AmazonClientExceptions represent an error that occurred inside
* the client on the local host, either while trying to send the
* request to AWS or interpret the response. For example, if no
* network connection is available, the client won't be able to
* connect to AWS to execute a request and will throw an
* AmazonClientException.
*/
System.out.println("Error Message: " + ace.getMessage());
}
}
public void run() {
logger.debug("started GridManager");
System.out.println("started GridManager");
AmazonSQSClient sqsClient = new AmazonSQSClient(papa.getCredentials());
AmazonEC2Client ec2Client = new AmazonEC2Client(papa.getCredentials());
// Get current imageID from the metadata table
String imageID = Constants.imageID; // get the default imageID from Constants
try {
Connection conn = PooledConnectionFactory.INSTANCE.getCumulusConnection();
Statement stmt = conn.createStatement();
String imageQuery = "SELECT mvalue FROM cumulus.metadata WHERE mkey='imageID';";
ResultSet results = stmt.executeQuery(imageQuery);
results.next(); // move the cursor into the results
imageID = results.getString(1);
} catch (SQLException e) {
logger.warn(e);
}
if (imageID == null
|| imageID.equals("")) { // if we get an empty imageID revert to the hardcoded one
imageID = Constants.imageID;
}
logger.debug(imageID);
System.out.println(imageID);
while (true) { // loop from startup to shutdown
// check the number of messages in the queue.
GetQueueAttributesRequest sqsRequest =
new GetQueueAttributesRequest(papa.getDispatchQueue())
.withAttributeNames("ApproximateNumberOfMessages");
Integer sqsResult = null;
try {
sqsResult =
Integer.valueOf(
sqsClient
.getQueueAttributes(sqsRequest)
.getAttributes()
.get("ApproximateNumberOfMessages"));
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.err.println("Error getting list of instances");
System.err.println("Caught Exception: " + e.getMessage());
System.err.println("Reponse Status Code: " + e.getStatusCode());
System.err.println("Error Code: " + e.getErrorCode());
System.err.println("Request ID: " + e.getRequestId());
} catch (AmazonClientException e) {
logger.warn(e);
continue; // if the error is due to being unable to connect to sqs it is likely transient,
// and we should keep going until SQS comes back
} catch (Exception e) {
// caught another exception
System.err.println(e);
System.err.println("Did not exit cleanly");
// System.exit(1);
}
if (sqsResult == null) {
System.err.println(
"Couldn't get the number of items in the queue reverting to static functioning");
logger.error(
"Couldn't get the number of items in the queue reverting to static functioning");
break;
}
System.out.println("Queue size=" + sqsResult);
// check the number of running instances.
DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest();
// get the number of running instances with our image ID
describeInstancesRequest.withFilters(
new Filter("image-id").withValues(imageID),
new Filter("instance-state-name").withValues("running"));
logger.debug("Getting list of active cumulus drones");
Integer numInstances = null;
Integer numSpotRequests = null;
Initializer.serialize("/tmp/dump.obj", papa.unitsOnServer); // TODO test this.
// check number of open spot requests
DescribeSpotInstanceRequestsRequest describeSpotInstancesRequest =
new DescribeSpotInstanceRequestsRequest();
describeSpotInstancesRequest.withFilters(new Filter("state").withValues("open"));
try {
// Parse returned instances
DescribeInstancesResult describeInstancesResult =
ec2Client.describeInstances(describeInstancesRequest);
if (describeInstancesResult.getReservations().size() == 0) {
numInstances = 0;
} else {
Integer totalInstances = 0;
for (Reservation i : describeInstancesResult.getReservations()) {
totalInstances += i.getInstances().size();
}
numInstances = totalInstances;
}
// Parse returned open spot requests
DescribeSpotInstanceRequestsResult describeSpotReservationsResult =
ec2Client.describeSpotInstanceRequests(describeSpotInstancesRequest);
if (describeSpotReservationsResult.getSpotInstanceRequests().size() == 0) {
numSpotRequests = 0;
} else {
Integer totalInstances = 0;
for (SpotInstanceRequest i : describeSpotReservationsResult.getSpotInstanceRequests()) {
totalInstances++;
}
numSpotRequests = totalInstances;
}
// System.out.println("num instances="+numInstances);
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.err.println("Error getting list of instances");
System.err.println("Caught Exception: " + e.getMessage());
System.err.println("Reponse Status Code: " + e.getStatusCode());
System.err.println("Error Code: " + e.getErrorCode());
System.err.println("Request ID: " + e.getRequestId());
// System.exit(1);
} catch (Exception e) {
// caught another exception
System.err.println(e);
System.err.println("Did not exit cleanly");
// System.exit(1);
}
if (numInstances == null || numSpotRequests == null) {
logger.error(
"Couldn't get the number of machines or requests reverting to static functioning");
break;
}
System.out.println("num instances=" + (numInstances + numSpotRequests));
// if queue is longer than the number of instances * idealMaxUnitsPerInstance
Integer idealMaxUnitsPerInstance = Integer.valueOf(Constants.idealMaxUnitsPerInstance);
if (sqsResult
> (numInstances + numSpotRequests)
* idealMaxUnitsPerInstance) { // we count pending spot instances too.
Integer numToCreate = (sqsResult / idealMaxUnitsPerInstance) - numInstances + 1;
// check current pricing of spot instances
DescribeSpotPriceHistoryRequest historyRequest =
new DescribeSpotPriceHistoryRequest()
.withAvailabilityZone("us-east-1c")
.withInstanceTypes(Constants.instanceType)
.withProductDescriptions("Linux/UNIX (Amazon VPC)")
.withStartTime(new Date(System.currentTimeMillis()));
DescribeSpotPriceHistoryResult result = ec2Client.describeSpotPriceHistory(historyRequest);
Double nowPrice = Double.valueOf(result.getSpotPriceHistory().get(0).getSpotPrice());
// If spot price is < constants.spotPrice launch numToCreate*Constants.percentSpot spot
// instances
Integer spotToCreate = 0;
if (nowPrice < Double.valueOf(Constants.spotPrice)) {
Double percentSpot = Double.valueOf(Constants.percentSpot);
spotToCreate = (int) (numToCreate * percentSpot);
numToCreate = (int) (numToCreate * (1 - percentSpot));
}
try {
System.out.println("regular instances: " + numToCreate);
System.out.println("spot instances: " + spotToCreate);
// Create regular and spot instances
if (numToCreate > 0) {
papa.createInstances(ec2Client, numToCreate, imageID);
}
if (spotToCreate > 0) {
papa.createSpotInstances(ec2Client, spotToCreate, Constants.imageID); // FIXME
}
System.out.println("created instances");
} catch (Exception e) {
// Report ec2 exceptions
logger.error(e);
e.printStackTrace();
continue;
}
}
try {
Thread.sleep(60000); // we wait for any new servers to start up
} catch (InterruptedException e) {
logger.error("Sqslistener was interrupted");
if (papa.getShuttingDown()) {
break;
} else {
continue;
}
}
}
}
public static void main(String[] args) throws IOException {
/*
* This credentials provider implementation loads your AWS credentials
* from a properties file at the root of your classpath.
*
* Important: Be sure to fill in your AWS access credentials in the
* AwsCredentials.properties file before you try to run this
* sample.
* http://aws.amazon.com/security-credentials
*/
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
String key = "MyObjectKey";
System.out.println("===========================================");
System.out.println("Getting Started with Amazon S3");
System.out.println("===========================================\n");
try {
/*
* Create a new S3 bucket - Amazon S3 bucket names are globally unique,
* so once a bucket name has been taken by any user, you can't create
* another bucket with that same name.
*
* You can optionally specify a location for your bucket if you want to
* keep your data closer to your applications or users.
*/
System.out.println("Creating bucket " + bucketName + "\n");
s3.createBucket(bucketName);
/*
* List the buckets in your account
*/
System.out.println("Listing buckets");
for (Bucket bucket : s3.listBuckets()) {
System.out.println(" - " + bucket.getName());
}
System.out.println();
/*
* Upload an object to your bucket - You can easily upload a file to
* S3, or upload directly an InputStream if you know the length of
* the data in the stream. You can also specify your own metadata
* when uploading to S3, which allows you set a variety of options
* like content-type and content-encoding, plus additional metadata
* specific to your applications.
*/
System.out.println("Uploading a new object to S3 from a file\n");
s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));
/*
* Download an object - When you download an object, you get all of
* the object's metadata and a stream from which to read the contents.
* It's important to read the contents of the stream as quickly as
* possibly since the data is streamed directly from Amazon S3 and your
* network connection will remain open until you read all the data or
* close the input stream.
*
* GetObjectRequest also supports several other options, including
* conditional downloading of objects based on modification times,
* ETags, and selectively downloading a range of an object.
*/
System.out.println("Downloading an object");
S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
displayTextInputStream(object.getObjectContent());
/*
* List objects in your bucket by prefix - There are many options for
* listing the objects in your bucket. Keep in mind that buckets with
* many objects might truncate their results when listing their objects,
* so be sure to check if the returned object listing is truncated, and
* use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
* additional results.
*/
System.out.println("Listing objects");
ObjectListing objectListing =
s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
System.out.println(
" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")");
}
System.out.println();
/*
* Delete an object - Unless versioning has been turned on for your bucket,
* there is no way to undelete an object, so use caution when deleting objects.
*/
System.out.println("Deleting an object\n");
s3.deleteObject(bucketName, key);
/*
* Delete a bucket - A bucket must be completely empty before it can be
* deleted, so remember to delete any objects from your buckets before
* you try to delete them.
*/
System.out.println("Deleting bucket " + bucketName + "\n");
s3.deleteBucket(bucketName);
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
/** @param args */
public static void main(String[] args) {
// ============================================================================================//
// =============================== Submitting a Request
// =======================================//
// ============================================================================================//
// Create the AmazonEC2Client object so we can call various APIs.
AmazonEC2 ec2 = new AmazonEC2Client(new ClasspathPropertiesFileCredentialsProvider());
Region usWest2 = Region.getRegion(Regions.US_WEST_2);
ec2.setRegion(usWest2);
// Initializes a Spot Instance Request
RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest();
// *************************** Required Parameters Settings ************************//
// Request 1 x t1.micro instance with a bid price of $0.03.
requestRequest.setSpotPrice("0.03");
requestRequest.setInstanceCount(Integer.valueOf(1));
// Setup the specifications of the launch. This includes the instance type (e.g. t1.micro)
// and the latest Amazon Linux AMI id available. Note, you should always use the latest
// Amazon Linux AMI id or another of your choosing.
LaunchSpecification launchSpecification = new LaunchSpecification();
launchSpecification.setImageId("ami-8c1fece5");
launchSpecification.setInstanceType("t1.micro");
// Add the security group to the request.
ArrayList<String> securityGroups = new ArrayList<String>();
securityGroups.add("GettingStartedGroup");
launchSpecification.setSecurityGroups(securityGroups);
// *************************** Bid Type Settings ************************//
// Set the type of the bid to persistent.
requestRequest.setType("persistent");
// *************************** Valid From/To Settings ************************//
// Set the valid start time to be two minutes from now.
Calendar from = Calendar.getInstance();
from.add(Calendar.MINUTE, 2);
requestRequest.setValidFrom(from.getTime());
// Set the valid end time to be two minutes and two hours from now.
Calendar until = (Calendar) from.clone();
until.add(Calendar.HOUR, 2);
requestRequest.setValidUntil(until.getTime());
// *************************** Launch Group Settings ************************//
// Set the launch group.
requestRequest.setLaunchGroup("ADVANCED-DEMO-LAUNCH-GROUP");
// *************************** Availability Zone Group Settings ************************//
// Set the availability zone group.
requestRequest.setAvailabilityZoneGroup("ADVANCED-DEMO-AZ-GROUP");
// *************************** Add the block device mapping ************************//
// Goal: Setup block device mappings to ensure that we will not delete
// the root partition on termination.
// Create the block device mapping to describe the root partition.
BlockDeviceMapping blockDeviceMapping = new BlockDeviceMapping();
blockDeviceMapping.setDeviceName("/dev/sda1");
// Set the delete on termination flag to false.
EbsBlockDevice ebs = new EbsBlockDevice();
ebs.setDeleteOnTermination(Boolean.FALSE);
blockDeviceMapping.setEbs(ebs);
// Add the block device mapping to the block list.
ArrayList<BlockDeviceMapping> blockList = new ArrayList<BlockDeviceMapping>();
blockList.add(blockDeviceMapping);
// Set the block device mapping configuration in the launch specifications.
launchSpecification.setBlockDeviceMappings(blockList);
// *************************** Add the availability zone ************************//
// Setup the availability zone to use. Note we could retrieve the availability
// zones using the ec2.describeAvailabilityZones() API. For this demo we will just use
// us-east-1b.
SpotPlacement placement = new SpotPlacement("us-east-1b");
launchSpecification.setPlacement(placement);
// *************************** Add the placement group ************************//
// Setup the placement group to use with whatever name you desire.
// For this demo we will just use "ADVANCED-DEMO-PLACEMENT-GROUP".
// Note: We have commented this out, because we are not leveraging cc1.4xlarge or
// cg1.4xlarge in this example.
/*
SpotPlacement pg = new SpotPlacement();
pg.setGroupName("ADVANCED-DEMO-PLACEMENT-GROUP");
launchSpecification.setPlacement(pg);
*/
// *************************** Add the launch specification ************************//
// Add the launch specification.
requestRequest.setLaunchSpecification(launchSpecification);
// ============================================================================================//
// =========================== Getting the Request ID from the Request
// ========================//
// ============================================================================================//
// Call the RequestSpotInstance API.
RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest);
List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();
// Setup an arraylist to collect all of the request ids we want to watch hit the running
// state.
ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();
// Add all of the request ids to the hashset, so we can determine when they hit the
// active state.
for (SpotInstanceRequest requestResponse : requestResponses) {
System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId());
spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId());
}
// ============================================================================================//
// =========================== Determining the State of the Spot Request
// ======================//
// ============================================================================================//
// Create a variable that will track whether there are any requests still in the open state.
boolean anyOpen;
// Initialize variables.
ArrayList<String> instanceIds = new ArrayList<String>();
do {
// Create the describeRequest with tall of the request id to monitor (e.g. that we started).
DescribeSpotInstanceRequestsRequest describeRequest =
new DescribeSpotInstanceRequestsRequest();
describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);
// Initialize the anyOpen variable to false ??? which assumes there are no requests open
// unless
// we find one that is still open.
anyOpen = false;
try {
// Retrieve all of the requests we want to monitor.
DescribeSpotInstanceRequestsResult describeResult =
ec2.describeSpotInstanceRequests(describeRequest);
List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();
// Look through each request and determine if they are all in the active state.
for (SpotInstanceRequest describeResponse : describeResponses) {
// If the state is open, it hasn't changed since we attempted to request it.
// There is the potential for it to transition almost immediately to closed or
// cancelled so we compare against open instead of active.
if (describeResponse.getState().equals("open")) {
anyOpen = true;
break;
}
// Add the instance id to the list we will eventually terminate.
instanceIds.add(describeResponse.getInstanceId());
}
} catch (AmazonServiceException e) {
// If we have an exception, ensure we don't break out of the loop.
// This prevents the scenario where there was blip on the wire.
anyOpen = true;
}
try {
// Sleep for 60 seconds.
Thread.sleep(60 * 1000);
} catch (Exception e) {
// Do nothing because it woke up early.
}
} while (anyOpen);
// ============================================================================================//
// ====================================== Canceling the Request ==============================//
// ============================================================================================//
try {
// Cancel requests.
CancelSpotInstanceRequestsRequest cancelRequest =
new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds);
ec2.cancelSpotInstanceRequests(cancelRequest);
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.out.println("Error cancelling instances");
System.out.println("Caught Exception: " + e.getMessage());
System.out.println("Reponse Status Code: " + e.getStatusCode());
System.out.println("Error Code: " + e.getErrorCode());
System.out.println("Request ID: " + e.getRequestId());
}
// ============================================================================================//
// =================================== Terminating any Instances
// ==============================//
// ============================================================================================//
try {
// Terminate instances.
TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds);
ec2.terminateInstances(terminateRequest);
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.out.println("Error terminating instances");
System.out.println("Caught Exception: " + e.getMessage());
System.out.println("Reponse Status Code: " + e.getStatusCode());
System.out.println("Error Code: " + e.getErrorCode());
System.out.println("Request ID: " + e.getRequestId());
}
}
public static void main(String[] args) throws Exception {
init();
try {
String violationsTable = "violations-table";
String citationsTable = "citations-table";
String warrantsTable = "warrants-table";
String contactsTable = "contacts-table";
String notificationsTable = "notifications-table";
// VIOLATIONS TABLE
//
//
// Create violationsTable if it does not exist yet
if (Tables.doesTableExist(dynamoDB, violationsTable)) {
System.out.println("Table " + violationsTable + " is already ACTIVE");
} else {
// Create a table with a primary hash key named 'name', which holds a string
CreateTableRequest createTableRequest =
new CreateTableRequest()
.withTableName(violationsTable)
.withKeySchema(
new KeySchemaElement()
.withAttributeName("citation_number")
.withKeyType(KeyType.HASH))
.withAttributeDefinitions(
new AttributeDefinition()
.withAttributeName("citation_number")
.withAttributeType(ScalarAttributeType.N))
.withProvisionedThroughput(
new ProvisionedThroughput()
.withReadCapacityUnits(25L)
.withWriteCapacityUnits(25L));
TableDescription createdTableDescription =
dynamoDB.createTable(createTableRequest).getTableDescription();
System.out.println("Created Table: " + createdTableDescription);
// Wait for it to become active
System.out.println("Waiting for " + violationsTable + " to become ACTIVE...");
Tables.awaitTableToBecomeActive(dynamoDB, violationsTable);
}
// Describe our new table
DescribeTableRequest describeTableRequest =
new DescribeTableRequest().withTableName(violationsTable);
TableDescription tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
System.out.println("Table Description: " + tableDescription);
// CITATIONS TABLE
//
//
// Create citationsTable if it does not exist yet
if (Tables.doesTableExist(dynamoDB, citationsTable)) {
System.out.println("Table " + citationsTable + " is already ACTIVE");
} else {
// Create a table with a primary hash key named 'name', which holds a string
/*
GlobalSecondaryIndex _firstLast = new GlobalSecondaryIndex()
.withIndexName("first_last")
.withProvisionedThroughput(new ProvisionedThroughput()
.withReadCapacityUnits((long) 10)
.withWriteCapacityUnits((long) 1))
.withProjection(new Projection().withProjectionType(ProjectionType.ALL));
GlobalSecondaryIndex _firstLastDOB = new GlobalSecondaryIndex()
.withIndexName("first_last_dob")
.withProvisionedThroughput(new ProvisionedThroughput()
.withReadCapacityUnits((long) 10)
.withWriteCapacityUnits((long) 1))
.withProjection(new Projection().withProjectionType(ProjectionType.ALL));
*/
CreateTableRequest createTableRequest =
new CreateTableRequest()
.withTableName(citationsTable)
.withKeySchema(
new KeySchemaElement()
.withAttributeName("first_last")
.withKeyType(KeyType.HASH))
.withAttributeDefinitions(
new AttributeDefinition()
.withAttributeName("first_last")
.withAttributeType(ScalarAttributeType.S))
.withProvisionedThroughput(
new ProvisionedThroughput()
.withReadCapacityUnits(25L)
.withWriteCapacityUnits(25L));
// .withGlobalSecondaryIndexes(_firstLast, _firstLastDOB);
TableDescription createdTableDescription =
dynamoDB.createTable(createTableRequest).getTableDescription();
System.out.println("Created Table: " + createdTableDescription);
// Wait for it to become active
System.out.println("Waiting for " + citationsTable + " to become ACTIVE...");
Tables.awaitTableToBecomeActive(dynamoDB, citationsTable);
}
// Describe our new table
describeTableRequest = new DescribeTableRequest().withTableName(citationsTable);
tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
System.out.println("Table Description: " + tableDescription);
// WARRANTS TABLE
//
//
// Create warrantsTable if it does not exist yet
if (Tables.doesTableExist(dynamoDB, warrantsTable)) {
System.out.println("Table " + warrantsTable + " is already ACTIVE");
} else {
/*
GlobalSecondaryIndex _firstLastDOB = new GlobalSecondaryIndex()
.withIndexName("first_last_dob")
.withProvisionedThroughput(new ProvisionedThroughput()
.withReadCapacityUnits((long) 10)
.withWriteCapacityUnits((long) 1))
.withProjection(new Projection().withProjectionType(ProjectionType.ALL));
*/
// Create a table with a primary hash key named 'name', which holds a string
CreateTableRequest createTableRequest =
new CreateTableRequest()
.withTableName(warrantsTable)
.withKeySchema(
new KeySchemaElement().withAttributeName("id").withKeyType(KeyType.HASH))
.withAttributeDefinitions(
new AttributeDefinition()
.withAttributeName("id")
.withAttributeType(ScalarAttributeType.S))
.withProvisionedThroughput(
new ProvisionedThroughput()
.withReadCapacityUnits(25L)
.withWriteCapacityUnits(25L));
// .withGlobalSecondaryIndexes(_firstLastDOB);
TableDescription createdTableDescription =
dynamoDB.createTable(createTableRequest).getTableDescription();
System.out.println("Created Table: " + createdTableDescription);
// Wait for it to become active
System.out.println("Waiting for " + warrantsTable + " to become ACTIVE...");
Tables.awaitTableToBecomeActive(dynamoDB, warrantsTable);
}
// Describe our new table
describeTableRequest = new DescribeTableRequest().withTableName(warrantsTable);
tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
System.out.println("Table Description: " + tableDescription);
// CONTACTS TABLE
//
//
// Create violationsTable if it does not exist yet
if (Tables.doesTableExist(dynamoDB, contactsTable)) {
System.out.println("Table " + contactsTable + " is already ACTIVE");
} else {
// Create a table with a primary hash key named 'name', which holds a string
CreateTableRequest createTableRequest =
new CreateTableRequest()
.withTableName(contactsTable)
.withKeySchema(
new KeySchemaElement()
.withAttributeName("first_last")
.withKeyType(KeyType.HASH))
.withAttributeDefinitions(
new AttributeDefinition()
.withAttributeName("first_last")
.withAttributeType(ScalarAttributeType.S))
.withProvisionedThroughput(
new ProvisionedThroughput()
.withReadCapacityUnits(25L)
.withWriteCapacityUnits(25L));
TableDescription createdTableDescription =
dynamoDB.createTable(createTableRequest).getTableDescription();
System.out.println("Created Table: " + createdTableDescription);
// Wait for it to become active
System.out.println("Waiting for " + contactsTable + " to become ACTIVE...");
Tables.awaitTableToBecomeActive(dynamoDB, contactsTable);
}
// Describe our new table
describeTableRequest = new DescribeTableRequest().withTableName(violationsTable);
tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
System.out.println("Table Description: " + tableDescription);
ssnHashMap = new HashMap<>();
saltHashMap = new HashMap<>();
hashedssnHashMap = new HashMap<>();
ssnList = new ArrayList<>();
for (int i = 1010001; i < 1200001; i++) {
SocialSecurityNumber ssn = new SocialSecurityNumber(i);
String ssnString = ssn.toString();
ssnList.add(ssnString);
}
/*
for (String str : ssnList) {
System.out.println("SSN: " + str);
}
*/
// oneTimeAddViolations();
// oneTimeAddCitations();
oneTimeAddWarrants();
// oneTimeAddContacts();
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to AWS, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with AWS, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
private void jButton1ActionPerformed(
java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jButton1ActionPerformed
// TODO add your handling code here: upload file
JFileChooser jfc = new JFileChooser();
jfc.setAcceptAllFileFilterUsed(false);
jfc.setFileFilter(new FileNameExtensionFilter("Text Files", "txt"));
int returnVal = jfc.showOpenDialog(this);
if (returnVal == JFileChooser.APPROVE_OPTION) {
File file = jfc.getSelectedFile();
try {
filepath = file.getCanonicalPath();
FileReader fr = new FileReader(filepath);
BufferedReader br = new BufferedReader(fr);
String line;
int total_no_of_words = 0;
while ((line = br.readLine()) != null) {
String[] words_arr = line.split(" ");
total_no_of_words += words_arr.length;
}
jLabel4.setText("word count :" + total_no_of_words);
System.out.println("Total no of words = " + total_no_of_words);
if (total_no_of_words != 0) {
File userFile = new File(filepath);
keyName = userFile.getName();
String myAccessKeyID = "AKIAI7PATIX6LHJA3A3Q";
String mySecretKey = "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";
AWSCredentials myCredentials = new BasicAWSCredentials(myAccessKeyID, mySecretKey);
ClientConfiguration clientConfig = new ClientConfiguration();
clientConfig.setProtocol(Protocol.HTTP);
AmazonS3 conn = new AmazonS3Client(myCredentials, clientConfig);
try {
File file1 = new File(filepath);
file_size = file1.length();
if (file_size < 100000) {
conn.putObject(new PutObjectRequest(bucketName, keyName, file1));
jLabel2.setText("Uploading complete !");
} else {
jLabel2.setText("Uploading failed size greater than 100kb...");
}
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response"
+ " for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
} /*catch (InterruptedException ex) {
Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
}*/
} else {
jLabel2.setText("Uploading failed file empty !");
}
} catch (IOException ex) {
Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
}
}
if (returnVal == JFileChooser.CANCEL_OPTION) {
System.exit(0);
}
} // GEN-LAST:event_jButton1ActionPerformed
public static double getCPUUsage(AmazonCloudWatchClient cloudWatch, String instanceId) {
try {
// create request message
GetMetricStatisticsRequest statRequest = new GetMetricStatisticsRequest();
// set up request message
statRequest.setNamespace("AWS/EC2"); // namespace
statRequest.setPeriod(60); // period of data
ArrayList<String> stats = new ArrayList<String>();
// Use one of these strings: Average, Maximum, Minimum, SampleCount, Sum
stats.add("Average");
stats.add("Sum");
stats.add("Maximum");
statRequest.setStatistics(stats);
// Use one of these strings: CPUUtilization, NetworkIn, NetworkOut, DiskReadBytes,
// DiskWriteBytes, DiskReadOperations
statRequest.setMetricName("CPUUtilization");
// set time
GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
calendar.add(
GregorianCalendar.SECOND, -1 * calendar.get(GregorianCalendar.SECOND)); // 1 second ago
Date endTime = calendar.getTime();
calendar.add(GregorianCalendar.MINUTE, -1); // 1 minutes ago
Date startTime = calendar.getTime();
statRequest.setStartTime(startTime);
statRequest.setEndTime(endTime);
// specify an instance
ArrayList<Dimension> dimensions = new ArrayList<Dimension>();
dimensions.add(new Dimension().withName("InstanceId").withValue(instanceId));
statRequest.setDimensions(dimensions);
// get statistics
GetMetricStatisticsResult statResult = cloudWatch.getMetricStatistics(statRequest);
// display
// System.out.println(statResult.toString());
List<Datapoint> dataList = statResult.getDatapoints();
Double averageCPU = null;
Date timeStamp = null;
for (Datapoint data : dataList) {
averageCPU = data.getAverage();
timeStamp = data.getTimestamp();
// System.out.println("Average CPU utlilization for last 1 minute since "
// +timeStamp.toString() + ": " +averageCPU);
// System.out.println("Max CPU utlilization for last 1 minute since "+timeStamp.toString() +
// ": " + data.getMaximum());
// System.out.println("Total CPU utlilization for last 1 minutes since
// "+timeStamp.toString() + ": " + data.getSum());
}
if (averageCPU == null) {
System.out.println("Average CPU utlilization for last 1 minute: 0");
return 0;
} else {
System.out.println(
"Average CPU utlilization for last 1 minute since "
+ timeStamp.toString()
+ ": "
+ averageCPU);
return averageCPU;
}
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
return 0;
}
}
