public static String setUpSpotInstanceWithImage(String image) { System.out.println("Putting spot requet for image: " + image); RequestSpotInstancesRequest loadGeneratorRequest = new RequestSpotInstancesRequest(); loadGeneratorRequest.setSpotPrice("0.03"); loadGeneratorRequest.setInstanceCount(Integer.valueOf(1)); LaunchSpecification loadGeneratorSpecification = new LaunchSpecification(); loadGeneratorSpecification.setImageId(image); loadGeneratorSpecification.setInstanceType("m3.medium"); ArrayList<String> securityGroups = new ArrayList<String>(); securityGroups.add("SecurityGroupForLoadGenerator"); loadGeneratorSpecification.setSecurityGroups(securityGroups); loadGeneratorRequest.setLaunchSpecification(loadGeneratorSpecification); RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(loadGeneratorRequest); List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests(); String requestId = requestResponses.get(0).getSpotInstanceRequestId(); ArrayList<String> spotInstanceRequestIds = new ArrayList<String>(); spotInstanceRequestIds.add(requestId); System.out.println("Finding Instance ID.."); boolean anyOpen; ArrayList<String> instanceIds = new ArrayList<String>(); do { DescribeSpotInstanceRequestsRequest describeRequest = new DescribeSpotInstanceRequestsRequest(); describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds); anyOpen = false; try { DescribeSpotInstanceRequestsResult describeResult = ec2.describeSpotInstanceRequests(describeRequest); List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests(); for (SpotInstanceRequest describeResponse : describeResponses) { if (describeResponse.getState().equals("open")) { anyOpen = true; break; } instanceIds.add(describeResponse.getInstanceId()); } } catch (AmazonServiceException e) { anyOpen = true; } sleep(30); } while (anyOpen); return instanceIds.get(0); }
/** @param args */ public static void main(String[] args) { // ============================================================================================// // =============================== Submitting a Request // =======================================// // ============================================================================================// // Retrieves the credentials from an AWSCredentials.properties file. AWSCredentials credentials = null; try { credentials = new PropertiesCredentials( InlineTaggingCodeSampleApp.class.getResourceAsStream("AwsCredentials.properties")); } catch (IOException e1) { System.out.println("Credentials were not properly entered into AwsCredentials.properties."); System.out.println(e1.getMessage()); System.exit(-1); } // Create the AmazonEC2Client object so we can call various APIs. AmazonEC2 ec2 = new AmazonEC2Client(credentials); // Initializes a Spot Instance Request RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest(); // Request 1 x t1.micro instance with a bid price of $0.03. requestRequest.setSpotPrice("0.03"); requestRequest.setInstanceCount(Integer.valueOf(1)); // Setup the specifications of the launch. This includes the instance type (e.g. t1.micro) // and the latest Amazon Linux AMI id available. Note, you should always use the latest // Amazon Linux AMI id or another of your choosing. LaunchSpecification launchSpecification = new LaunchSpecification(); launchSpecification.setImageId("ami-8c1fece5"); launchSpecification.setInstanceType("t1.micro"); // Add the security group to the request. ArrayList<String> securityGroups = new ArrayList<String>(); securityGroups.add("GettingStartedGroup"); launchSpecification.setSecurityGroups(securityGroups); // Add the launch specifications to the request. requestRequest.setLaunchSpecification(launchSpecification); // ============================================================================================// // =========================== Getting the Request ID from the Request // ========================// // ============================================================================================// // Call the RequestSpotInstance API. RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest); List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests(); // Setup an arraylist to collect all of the request ids we want to watch hit the running // state. ArrayList<String> spotInstanceRequestIds = new ArrayList<String>(); // Add all of the request ids to the hashset, so we can determine when they hit the // active state. for (SpotInstanceRequest requestResponse : requestResponses) { System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId()); spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId()); } // ============================================================================================// // ====================================== Tag the Spot Requests // ===============================// // ============================================================================================// // Create the list of tags we want to create ArrayList<Tag> requestTags = new ArrayList<Tag>(); requestTags.add(new Tag("keyname1", "value1")); // Create a tag request for requests. CreateTagsRequest createTagsRequest_requests = new CreateTagsRequest(); createTagsRequest_requests.setResources(spotInstanceRequestIds); createTagsRequest_requests.setTags(requestTags); // Try to tag the Spot request submitted. try { ec2.createTags(createTagsRequest_requests); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.out.println("Error terminating instances"); System.out.println("Caught Exception: " + e.getMessage()); System.out.println("Reponse Status Code: " + e.getStatusCode()); System.out.println("Error Code: " + e.getErrorCode()); System.out.println("Request ID: " + e.getRequestId()); } // ============================================================================================// // =========================== Determining the State of the Spot Request // ======================// // ============================================================================================// // Create a variable that will track whether there are any requests still in the open state. boolean anyOpen; // Initialize variables. ArrayList<String> instanceIds = new ArrayList<String>(); do { // Create the describeRequest with tall of the request id to monitor (e.g. that we started). DescribeSpotInstanceRequestsRequest describeRequest = new DescribeSpotInstanceRequestsRequest(); describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds); // Initialize the anyOpen variable to false – which assumes there are no requests open unless // we find one that is still open. anyOpen = false; try { // Retrieve all of the requests we want to monitor. DescribeSpotInstanceRequestsResult describeResult = ec2.describeSpotInstanceRequests(describeRequest); List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests(); // Look through each request and determine if they are all in the active state. for (SpotInstanceRequest describeResponse : describeResponses) { // If the state is open, it hasn't changed since we attempted to request it. // There is the potential for it to transition almost immediately to closed or // cancelled so we compare against open instead of active. if (describeResponse.getState().equals("open")) { anyOpen = true; break; } // Add the instance id to the list we will eventually terminate. instanceIds.add(describeResponse.getInstanceId()); } } catch (AmazonServiceException e) { // If we have an exception, ensure we don't break out of the loop. // This prevents the scenario where there was blip on the wire. anyOpen = true; } try { // Sleep for 60 seconds. Thread.sleep(60 * 1000); } catch (Exception e) { // Do nothing because it woke up early. } } while (anyOpen); // ============================================================================================// // ====================================== Tag the Spot Instances // ===============================// // ============================================================================================// // Create the list of tags we want to create ArrayList<Tag> instanceTags = new ArrayList<Tag>(); instanceTags.add(new Tag("keyname1", "value1")); // Create a tag request for instances. CreateTagsRequest createTagsRequest_instances = new CreateTagsRequest(); createTagsRequest_instances.setResources(instanceIds); createTagsRequest_instances.setTags(instanceTags); // Try to tag the Spot instance started. try { ec2.createTags(createTagsRequest_instances); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.out.println("Error terminating instances"); System.out.println("Caught Exception: " + e.getMessage()); System.out.println("Reponse Status Code: " + e.getStatusCode()); System.out.println("Error Code: " + e.getErrorCode()); System.out.println("Request ID: " + e.getRequestId()); } // ============================================================================================// // ====================================== Canceling the Request ==============================// // ============================================================================================// try { // Cancel requests. CancelSpotInstanceRequestsRequest cancelRequest = new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds); ec2.cancelSpotInstanceRequests(cancelRequest); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.out.println("Error cancelling instances"); System.out.println("Caught Exception: " + e.getMessage()); System.out.println("Reponse Status Code: " + e.getStatusCode()); System.out.println("Error Code: " + e.getErrorCode()); System.out.println("Request ID: " + e.getRequestId()); } // ============================================================================================// // =================================== Terminating any Instances // ==============================// // ============================================================================================// try { // Terminate instances. TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds); ec2.terminateInstances(terminateRequest); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.out.println("Error terminating instances"); System.out.println("Caught Exception: " + e.getMessage()); System.out.println("Reponse Status Code: " + e.getStatusCode()); System.out.println("Error Code: " + e.getErrorCode()); System.out.println("Request ID: " + e.getRequestId()); } }
/** @param args */ public static void main(String[] args) { // ============================================================================================// // =============================== Submitting a Request // =======================================// // ============================================================================================// // Create the AmazonEC2Client object so we can call various APIs. AmazonEC2 ec2 = new AmazonEC2Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.US_WEST_2); ec2.setRegion(usWest2); // Initializes a Spot Instance Request RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest(); // *************************** Required Parameters Settings ************************// // Request 1 x t1.micro instance with a bid price of $0.03. requestRequest.setSpotPrice("0.03"); requestRequest.setInstanceCount(Integer.valueOf(1)); // Setup the specifications of the launch. This includes the instance type (e.g. t1.micro) // and the latest Amazon Linux AMI id available. Note, you should always use the latest // Amazon Linux AMI id or another of your choosing. LaunchSpecification launchSpecification = new LaunchSpecification(); launchSpecification.setImageId("ami-8c1fece5"); launchSpecification.setInstanceType("t1.micro"); // Add the security group to the request. ArrayList<String> securityGroups = new ArrayList<String>(); securityGroups.add("GettingStartedGroup"); launchSpecification.setSecurityGroups(securityGroups); // *************************** Bid Type Settings ************************// // Set the type of the bid to persistent. requestRequest.setType("persistent"); // *************************** Valid From/To Settings ************************// // Set the valid start time to be two minutes from now. Calendar from = Calendar.getInstance(); from.add(Calendar.MINUTE, 2); requestRequest.setValidFrom(from.getTime()); // Set the valid end time to be two minutes and two hours from now. Calendar until = (Calendar) from.clone(); until.add(Calendar.HOUR, 2); requestRequest.setValidUntil(until.getTime()); // *************************** Launch Group Settings ************************// // Set the launch group. requestRequest.setLaunchGroup("ADVANCED-DEMO-LAUNCH-GROUP"); // *************************** Availability Zone Group Settings ************************// // Set the availability zone group. requestRequest.setAvailabilityZoneGroup("ADVANCED-DEMO-AZ-GROUP"); // *************************** Add the block device mapping ************************// // Goal: Setup block device mappings to ensure that we will not delete // the root partition on termination. // Create the block device mapping to describe the root partition. BlockDeviceMapping blockDeviceMapping = new BlockDeviceMapping(); blockDeviceMapping.setDeviceName("/dev/sda1"); // Set the delete on termination flag to false. EbsBlockDevice ebs = new EbsBlockDevice(); ebs.setDeleteOnTermination(Boolean.FALSE); blockDeviceMapping.setEbs(ebs); // Add the block device mapping to the block list. ArrayList<BlockDeviceMapping> blockList = new ArrayList<BlockDeviceMapping>(); blockList.add(blockDeviceMapping); // Set the block device mapping configuration in the launch specifications. launchSpecification.setBlockDeviceMappings(blockList); // *************************** Add the availability zone ************************// // Setup the availability zone to use. Note we could retrieve the availability // zones using the ec2.describeAvailabilityZones() API. For this demo we will just use // us-east-1b. SpotPlacement placement = new SpotPlacement("us-east-1b"); launchSpecification.setPlacement(placement); // *************************** Add the placement group ************************// // Setup the placement group to use with whatever name you desire. // For this demo we will just use "ADVANCED-DEMO-PLACEMENT-GROUP". // Note: We have commented this out, because we are not leveraging cc1.4xlarge or // cg1.4xlarge in this example. /* SpotPlacement pg = new SpotPlacement(); pg.setGroupName("ADVANCED-DEMO-PLACEMENT-GROUP"); launchSpecification.setPlacement(pg); */ // *************************** Add the launch specification ************************// // Add the launch specification. requestRequest.setLaunchSpecification(launchSpecification); // ============================================================================================// // =========================== Getting the Request ID from the Request // ========================// // ============================================================================================// // Call the RequestSpotInstance API. RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest); List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests(); // Setup an arraylist to collect all of the request ids we want to watch hit the running // state. ArrayList<String> spotInstanceRequestIds = new ArrayList<String>(); // Add all of the request ids to the hashset, so we can determine when they hit the // active state. for (SpotInstanceRequest requestResponse : requestResponses) { System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId()); spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId()); } // ============================================================================================// // =========================== Determining the State of the Spot Request // ======================// // ============================================================================================// // Create a variable that will track whether there are any requests still in the open state. boolean anyOpen; // Initialize variables. ArrayList<String> instanceIds = new ArrayList<String>(); do { // Create the describeRequest with tall of the request id to monitor (e.g. that we started). DescribeSpotInstanceRequestsRequest describeRequest = new DescribeSpotInstanceRequestsRequest(); describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds); // Initialize the anyOpen variable to false ??? which assumes there are no requests open // unless // we find one that is still open. anyOpen = false; try { // Retrieve all of the requests we want to monitor. DescribeSpotInstanceRequestsResult describeResult = ec2.describeSpotInstanceRequests(describeRequest); List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests(); // Look through each request and determine if they are all in the active state. for (SpotInstanceRequest describeResponse : describeResponses) { // If the state is open, it hasn't changed since we attempted to request it. // There is the potential for it to transition almost immediately to closed or // cancelled so we compare against open instead of active. if (describeResponse.getState().equals("open")) { anyOpen = true; break; } // Add the instance id to the list we will eventually terminate. instanceIds.add(describeResponse.getInstanceId()); } } catch (AmazonServiceException e) { // If we have an exception, ensure we don't break out of the loop. // This prevents the scenario where there was blip on the wire. anyOpen = true; } try { // Sleep for 60 seconds. Thread.sleep(60 * 1000); } catch (Exception e) { // Do nothing because it woke up early. } } while (anyOpen); // ============================================================================================// // ====================================== Canceling the Request ==============================// // ============================================================================================// try { // Cancel requests. CancelSpotInstanceRequestsRequest cancelRequest = new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds); ec2.cancelSpotInstanceRequests(cancelRequest); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.out.println("Error cancelling instances"); System.out.println("Caught Exception: " + e.getMessage()); System.out.println("Reponse Status Code: " + e.getStatusCode()); System.out.println("Error Code: " + e.getErrorCode()); System.out.println("Request ID: " + e.getRequestId()); } // ============================================================================================// // =================================== Terminating any Instances // ==============================// // ============================================================================================// try { // Terminate instances. TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds); ec2.terminateInstances(terminateRequest); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.out.println("Error terminating instances"); System.out.println("Caught Exception: " + e.getMessage()); System.out.println("Reponse Status Code: " + e.getStatusCode()); System.out.println("Error Code: " + e.getErrorCode()); System.out.println("Request ID: " + e.getRequestId()); } }
public void run() { logger.debug("started GridManager"); System.out.println("started GridManager"); AmazonSQSClient sqsClient = new AmazonSQSClient(papa.getCredentials()); AmazonEC2Client ec2Client = new AmazonEC2Client(papa.getCredentials()); // Get current imageID from the metadata table String imageID = Constants.imageID; // get the default imageID from Constants try { Connection conn = PooledConnectionFactory.INSTANCE.getCumulusConnection(); Statement stmt = conn.createStatement(); String imageQuery = "SELECT mvalue FROM cumulus.metadata WHERE mkey='imageID';"; ResultSet results = stmt.executeQuery(imageQuery); results.next(); // move the cursor into the results imageID = results.getString(1); } catch (SQLException e) { logger.warn(e); } if (imageID == null || imageID.equals("")) { // if we get an empty imageID revert to the hardcoded one imageID = Constants.imageID; } logger.debug(imageID); System.out.println(imageID); while (true) { // loop from startup to shutdown // check the number of messages in the queue. GetQueueAttributesRequest sqsRequest = new GetQueueAttributesRequest(papa.getDispatchQueue()) .withAttributeNames("ApproximateNumberOfMessages"); Integer sqsResult = null; try { sqsResult = Integer.valueOf( sqsClient .getQueueAttributes(sqsRequest) .getAttributes() .get("ApproximateNumberOfMessages")); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.err.println("Error getting list of instances"); System.err.println("Caught Exception: " + e.getMessage()); System.err.println("Reponse Status Code: " + e.getStatusCode()); System.err.println("Error Code: " + e.getErrorCode()); System.err.println("Request ID: " + e.getRequestId()); } catch (AmazonClientException e) { logger.warn(e); continue; // if the error is due to being unable to connect to sqs it is likely transient, // and we should keep going until SQS comes back } catch (Exception e) { // caught another exception System.err.println(e); System.err.println("Did not exit cleanly"); // System.exit(1); } if (sqsResult == null) { System.err.println( "Couldn't get the number of items in the queue reverting to static functioning"); logger.error( "Couldn't get the number of items in the queue reverting to static functioning"); break; } System.out.println("Queue size=" + sqsResult); // check the number of running instances. DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest(); // get the number of running instances with our image ID describeInstancesRequest.withFilters( new Filter("image-id").withValues(imageID), new Filter("instance-state-name").withValues("running")); logger.debug("Getting list of active cumulus drones"); Integer numInstances = null; Integer numSpotRequests = null; Initializer.serialize("/tmp/dump.obj", papa.unitsOnServer); // TODO test this. // check number of open spot requests DescribeSpotInstanceRequestsRequest describeSpotInstancesRequest = new DescribeSpotInstanceRequestsRequest(); describeSpotInstancesRequest.withFilters(new Filter("state").withValues("open")); try { // Parse returned instances DescribeInstancesResult describeInstancesResult = ec2Client.describeInstances(describeInstancesRequest); if (describeInstancesResult.getReservations().size() == 0) { numInstances = 0; } else { Integer totalInstances = 0; for (Reservation i : describeInstancesResult.getReservations()) { totalInstances += i.getInstances().size(); } numInstances = totalInstances; } // Parse returned open spot requests DescribeSpotInstanceRequestsResult describeSpotReservationsResult = ec2Client.describeSpotInstanceRequests(describeSpotInstancesRequest); if (describeSpotReservationsResult.getSpotInstanceRequests().size() == 0) { numSpotRequests = 0; } else { Integer totalInstances = 0; for (SpotInstanceRequest i : describeSpotReservationsResult.getSpotInstanceRequests()) { totalInstances++; } numSpotRequests = totalInstances; } // System.out.println("num instances="+numInstances); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.err.println("Error getting list of instances"); System.err.println("Caught Exception: " + e.getMessage()); System.err.println("Reponse Status Code: " + e.getStatusCode()); System.err.println("Error Code: " + e.getErrorCode()); System.err.println("Request ID: " + e.getRequestId()); // System.exit(1); } catch (Exception e) { // caught another exception System.err.println(e); System.err.println("Did not exit cleanly"); // System.exit(1); } if (numInstances == null || numSpotRequests == null) { logger.error( "Couldn't get the number of machines or requests reverting to static functioning"); break; } System.out.println("num instances=" + (numInstances + numSpotRequests)); // if queue is longer than the number of instances * idealMaxUnitsPerInstance Integer idealMaxUnitsPerInstance = Integer.valueOf(Constants.idealMaxUnitsPerInstance); if (sqsResult > (numInstances + numSpotRequests) * idealMaxUnitsPerInstance) { // we count pending spot instances too. Integer numToCreate = (sqsResult / idealMaxUnitsPerInstance) - numInstances + 1; // check current pricing of spot instances DescribeSpotPriceHistoryRequest historyRequest = new DescribeSpotPriceHistoryRequest() .withAvailabilityZone("us-east-1c") .withInstanceTypes(Constants.instanceType) .withProductDescriptions("Linux/UNIX (Amazon VPC)") .withStartTime(new Date(System.currentTimeMillis())); DescribeSpotPriceHistoryResult result = ec2Client.describeSpotPriceHistory(historyRequest); Double nowPrice = Double.valueOf(result.getSpotPriceHistory().get(0).getSpotPrice()); // If spot price is < constants.spotPrice launch numToCreate*Constants.percentSpot spot // instances Integer spotToCreate = 0; if (nowPrice < Double.valueOf(Constants.spotPrice)) { Double percentSpot = Double.valueOf(Constants.percentSpot); spotToCreate = (int) (numToCreate * percentSpot); numToCreate = (int) (numToCreate * (1 - percentSpot)); } try { System.out.println("regular instances: " + numToCreate); System.out.println("spot instances: " + spotToCreate); // Create regular and spot instances if (numToCreate > 0) { papa.createInstances(ec2Client, numToCreate, imageID); } if (spotToCreate > 0) { papa.createSpotInstances(ec2Client, spotToCreate, Constants.imageID); // FIXME } System.out.println("created instances"); } catch (Exception e) { // Report ec2 exceptions logger.error(e); e.printStackTrace(); continue; } } try { Thread.sleep(60000); // we wait for any new servers to start up } catch (InterruptedException e) { logger.error("Sqslistener was interrupted"); if (papa.getShuttingDown()) { break; } else { continue; } } } }