public static void main(String[] args) throws UnknownHostException {
AmazonEC2 ec2 = new AmazonEC2Client(new ClasspathPropertiesFileCredentialsProvider());
Region usWest2 = Region.getRegion(Regions.US_WEST_2);
ec2.setRegion(usWest2);
List<String> instanceIds = new ArrayList<>();
String instanceid = null;
DescribeInstancesRequest request = new DescribeInstancesRequest();
List<String> valuesT1 = new ArrayList<String>();
valuesT1.add("hz-nodes");
Filter filter = new Filter("tag-value", valuesT1);
DescribeInstancesResult result = ec2.describeInstances(request.withFilters(filter));
List<Reservation> reservations = result.getReservations();
for (Reservation reservation : reservations) {
List<Instance> instances = reservation.getInstances();
for (Instance instance : instances) {
instanceid = instance.getInstanceId();
PrintManager.PrintMessage(instanceid);
instanceIds.add(instanceid);
List<Tag> abc = instance.getTags();
for (Tag aaa : abc) {
PrintManager.PrintMessage(aaa.getKey() + " : " + aaa.getValue());
}
}
}
instanceIds.remove("i-cb45adfc");
StopInstancesRequest stopReq = new StopInstancesRequest(instanceIds);
ec2.stopInstances(stopReq);
RebootInstancesRequest requestRe = new RebootInstancesRequest(instanceIds);
ec2.rebootInstances(requestRe);
}
/**
* Counts the number of instances in EC2 currently running that are using the specified image and
* a template.
*
* @param ami If AMI is left null, then all instances are counted and template description is
* ignored.
* <p>This includes those instances that may be started outside Jenkins.
* @param templateDesc
*/
public int countCurrentEC2Slaves(String ami, String templateDesc) throws AmazonClientException {
int n = 0;
for (Reservation r : connect().describeInstances().getReservations()) {
for (Instance i : r.getInstances()) {
if (isEc2ProvisionedAmiSlave(i, ami, templateDesc)) {
InstanceStateName stateName = InstanceStateName.fromValue(i.getState().getName());
if (stateName == InstanceStateName.Pending || stateName == InstanceStateName.Running) {
EC2AbstractSlave foundSlave = null;
for (EC2AbstractSlave ec2Node : NodeIterator.nodes(EC2AbstractSlave.class)) {
if (ec2Node.getInstanceId().equals(i.getInstanceId())) {
foundSlave = ec2Node;
break;
}
}
// Don't count disconnected slaves as being used, we will connected them later is
// required
if (foundSlave != null && foundSlave.toComputer().isOffline()) continue;
n++;
}
}
}
}
// Count pending spot requests too
for (SpotInstanceRequest sir :
connect().describeSpotInstanceRequests().getSpotInstanceRequests()) {
// Count Spot requests that are open and still have a
// chance to be active.
if (sir.getState().equals("open")) {
n++;
}
}
return n;
}
public void run() {
LOG.debug("Running describeInstances() in " + region.getRegionName());
List<Reservation> reservations = ec2Client.describeInstances().getReservations();
for (Reservation reservation : reservations) {
instances.addAll(reservation.getInstances());
}
LOG.debug("Completed describeInstances() in " + region.getRegionName());
}
public static String getPublicDns(String instanceId) {
System.out.println("Retreiving public dns.");
DescribeInstancesResult results = ec2.describeInstances();
List<Reservation> reservations = results.getReservations();
for (Reservation reservation : reservations) {
for (Instance instance : reservation.getInstances()) {
if (instance.getInstanceId().equals(instanceId)) return instance.getPublicDnsName();
}
}
return "NA";
}
/** Query amazon to get ASG name. Currently not available as part of instance info api. */
private String populateASGName(String region, String instanceId) {
AmazonEC2 client = new AmazonEC2Client(provider.getCredentials());
client.setEndpoint("ec2." + region + ".amazonaws.com");
DescribeInstancesRequest desc = new DescribeInstancesRequest().withInstanceIds(instanceId);
DescribeInstancesResult res = client.describeInstances(desc);
for (Reservation resr : res.getReservations()) {
for (Instance ins : resr.getInstances()) {
for (com.amazonaws.services.ec2.model.Tag tag : ins.getTags()) {
if (tag.getKey().equals("aws:autoscaling:groupName")) return tag.getValue();
}
}
}
return null;
}
@Override
public String retriableCall() throws IllegalStateException {
DescribeInstancesRequest desc = new DescribeInstancesRequest().withInstanceIds(instanceId);
DescribeInstancesResult res = client.describeInstances(desc);
for (Reservation resr : res.getReservations()) {
for (Instance ins : resr.getInstances()) {
for (com.amazonaws.services.ec2.model.Tag tag : ins.getTags()) {
if (tag.getKey().equals("aws:autoscaling:groupName")) return tag.getValue();
}
}
}
logger.warn("Couldn't determine ASG name");
throw new IllegalStateException("Couldn't determine ASG name");
}
public String getDnsMaster(String clusterId) {
String master = null;
DescribeInstancesResult describeInstancesRequest = ec2.describeInstances();
List<Reservation> reservations = describeInstancesRequest.getReservations();
int count = 0;
for (Reservation reservation : reservations) {
List<com.amazonaws.services.ec2.model.Tag> tags = reservation.getInstances().get(0).getTags();
for (int i = 0; i < tags.size(); i++) {
if ((tags.get(i).getValue().equals(clusterId))
|| (tags.get(i).getValue().equals("MASTER"))) {
count++;
}
}
if (count == 2) master = reservation.getInstances().get(0).getPublicDnsName();
count = 0;
}
return master;
}
public Instance startInstance(AWSCredentials credentials) {
AmazonEC2Client amazonEC2Client = new AmazonEC2Client(credentials);
RunInstancesRequest runInstancesRequest =
new RunInstancesRequest()
.withImageId("")
.withInstanceType("m1.small")
.withMinCount(1)
.withMaxCount(1);
RunInstancesResult runInstancesResult = amazonEC2Client.runInstances(runInstancesRequest);
Reservation reservation = runInstancesResult.getReservation();
List<Instance> instances = reservation.getInstances();
// there will be exactly one instance in this list, otherwise
// runInstances() would have thrown an exception
return instances.get(0);
}
private static void putCache() throws CacheException {
if (amazon == null) amazon = JCS.getInstance("Instances");
synchronized (AmazonService.class) {
amazon.clear();
DescribeInstancesResult result = amazonClient.describeInstances();
for (Reservation r : result.getReservations()) {
for (Instance i : r.getInstances()) {
amazon.putInGroup(i.getInstanceId(), "Instances", i);
}
}
DescribeImagesRequest ir = new DescribeImagesRequest().withOwners("self");
DescribeImagesResult images = amazonClient.describeImages(ir);
for (Image r : images.getImages()) {
amazon.putInGroup(r.getImageId(), "AMIs", r);
}
log.debug("put cache finished");
}
}
/**
* Describe a set of specific instances.
*
* @param instanceIds the instance ids
* @return the instances
*/
public List<Instance> describeInstances(String... instanceIds) {
if (instanceIds == null || instanceIds.length == 0) {
LOGGER.info("Getting all EC2 instances.");
} else {
LOGGER.info(String.format("Getting EC2 instances for %d ids.", instanceIds.length));
}
List<Instance> instances = new LinkedList<Instance>();
AmazonEC2 ec2Client = ec2Client();
DescribeInstancesRequest request = new DescribeInstancesRequest();
if (instanceIds != null) {
request.withInstanceIds(Arrays.asList(instanceIds));
}
DescribeInstancesResult result = ec2Client.describeInstances(request);
for (Reservation reservation : result.getReservations()) {
instances.addAll(reservation.getInstances());
}
LOGGER.info(String.format("Got %d EC2 instances.", instances.size()));
return instances;
}
// Returns pending/running instance ID list.
public static ArrayList<String> getInstanceIdsByTag(AmazonEC2 ec2, String key, String value) {
logger.fine("Requesting instances by tag " + key + "=" + value);
// Create tag request.
DescribeInstancesRequest instanceReq = new DescribeInstancesRequest();
Filter managerFilter = new Filter("tag:" + key).withValues(value),
activeFilter =
new Filter("instance-state-code").withValues("0", "16"); // 0||16 == pending||running
// Send request.
DescribeInstancesResult result =
ec2.describeInstances(instanceReq.withFilters(managerFilter, activeFilter));
// Build instance ID list.
ArrayList<String> ids = new ArrayList<String>();
for (Reservation reservation : result.getReservations()) {
for (Instance instance : reservation.getInstances()) {
ids.add(instance.getInstanceId());
}
}
logger.fine(ids.size() + " instances found with tag " + key + "=" + value);
return ids;
}
public static void main(String[] args) throws Exception {
System.out.println("===========================================");
System.out.println("Welcome to the AWS Java SDK!");
System.out.println("===========================================");
init();
try {
/*
* The Amazon EC2 client allows you to easily launch and configure
* computing capacity in AWS datacenters.
*
* In this sample, we use the EC2 client to list the availability zones
* in a region, and then list the instances running in those zones.
*/
DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
List<AvailabilityZone> availabilityZones = availabilityZonesResult.getAvailabilityZones();
System.out.println("You have access to " + availabilityZones.size() + " availability zones:");
for (AvailabilityZone zone : availabilityZones) {
System.out.println(" - " + zone.getZoneName() + " (" + zone.getRegionName() + ")");
}
DescribeInstancesResult describeInstancesResult = ec2.describeInstances();
Set<Instance> instances = new HashSet<Instance>();
for (Reservation reservation : describeInstancesResult.getReservations()) {
instances.addAll(reservation.getInstances());
}
System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");
/*
* The Amazon S3 client allows you to manage and configure buckets
* and to upload and download data.
*
* In this sample, we use the S3 client to list all the buckets in
* your account, and then iterate over the object metadata for all
* objects in one bucket to calculate the total object count and
* space usage for that one bucket. Note that this sample only
* retrieves the object's metadata and doesn't actually download the
* object's content.
*
* In addition to the low-level Amazon S3 client in the SDK, there
* is also a high-level TransferManager API that provides
* asynchronous management of uploads and downloads with an easy to
* use API:
* http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/transfer/TransferManager.html
*/
List<Bucket> buckets = s3.listBuckets();
System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s).");
if (buckets.size() > 0) {
Bucket bucket = buckets.get(0);
long totalSize = 0;
long totalItems = 0;
/*
* The S3Objects and S3Versions classes provide convenient APIs
* for iterating over the contents of your buckets, without
* having to manually deal with response pagination.
*/
for (S3ObjectSummary objectSummary : S3Objects.inBucket(s3, bucket.getName())) {
totalSize += objectSummary.getSize();
totalItems++;
}
System.out.println(
"The bucket '"
+ bucket.getName()
+ "' contains "
+ totalItems
+ " objects "
+ "with a total size of "
+ totalSize
+ " bytes.");
}
} catch (AmazonServiceException ase) {
/*
* AmazonServiceExceptions represent an error response from an AWS
* services, i.e. your request made it to AWS, but the AWS service
* either found it invalid or encountered an error trying to execute
* it.
*/
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
/*
* AmazonClientExceptions represent an error that occurred inside
* the client on the local host, either while trying to send the
* request to AWS or interpret the response. For example, if no
* network connection is available, the client won't be able to
* connect to AWS to execute a request and will throw an
* AmazonClientException.
*/
System.out.println("Error Message: " + ace.getMessage());
}
}
/**
* Provisions an On-demand EC2 slave by launching a new instance or starting a previously-stopped
* instance.
*/
private EC2AbstractSlave provisionOndemand(
TaskListener listener, Label requiredLabel, EnumSet<ProvisionOptions> provisionOptions)
throws AmazonClientException, IOException {
PrintStream logger = listener.getLogger();
AmazonEC2 ec2 = getParent().connect();
try {
logProvisionInfo(logger, "Considering launching " + ami + " for template " + description);
KeyPair keyPair = getKeyPair(ec2);
RunInstancesRequest riRequest = new RunInstancesRequest(ami, 1, 1);
InstanceNetworkInterfaceSpecification net = new InstanceNetworkInterfaceSpecification();
riRequest.setEbsOptimized(ebsOptimized);
if (useEphemeralDevices) {
setupEphemeralDeviceMapping(riRequest);
} else {
setupCustomDeviceMapping(riRequest);
}
if (stopOnTerminate) {
riRequest.setInstanceInitiatedShutdownBehavior(ShutdownBehavior.Stop);
logProvisionInfo(
logger, "Setting Instance Initiated Shutdown Behavior : ShutdownBehavior.Stop");
} else {
riRequest.setInstanceInitiatedShutdownBehavior(ShutdownBehavior.Terminate);
logProvisionInfo(
logger, "Setting Instance Initiated Shutdown Behavior : ShutdownBehavior.Terminate");
}
List<Filter> diFilters = new ArrayList<Filter>();
diFilters.add(new Filter("image-id").withValues(ami));
if (StringUtils.isNotBlank(getZone())) {
Placement placement = new Placement(getZone());
if (getUseDedicatedTenancy()) {
placement.setTenancy("dedicated");
}
riRequest.setPlacement(placement);
diFilters.add(new Filter("availability-zone").withValues(getZone()));
}
if (StringUtils.isNotBlank(getSubnetId())) {
if (getAssociatePublicIp()) {
net.setSubnetId(getSubnetId());
} else {
riRequest.setSubnetId(getSubnetId());
}
diFilters.add(new Filter("subnet-id").withValues(getSubnetId()));
/*
* If we have a subnet ID then we can only use VPC security groups
*/
if (!securityGroupSet.isEmpty()) {
List<String> groupIds = getEc2SecurityGroups(ec2);
if (!groupIds.isEmpty()) {
if (getAssociatePublicIp()) {
net.setGroups(groupIds);
} else {
riRequest.setSecurityGroupIds(groupIds);
}
diFilters.add(new Filter("instance.group-id").withValues(groupIds));
}
}
} else {
/* No subnet: we can use standard security groups by name */
riRequest.setSecurityGroups(securityGroupSet);
if (!securityGroupSet.isEmpty()) {
diFilters.add(new Filter("instance.group-name").withValues(securityGroupSet));
}
}
String userDataString = Base64.encodeBase64String(userData.getBytes(StandardCharsets.UTF_8));
riRequest.setUserData(userDataString);
riRequest.setKeyName(keyPair.getKeyName());
diFilters.add(new Filter("key-name").withValues(keyPair.getKeyName()));
riRequest.setInstanceType(type.toString());
diFilters.add(new Filter("instance-type").withValues(type.toString()));
if (getAssociatePublicIp()) {
net.setAssociatePublicIpAddress(true);
net.setDeviceIndex(0);
riRequest.withNetworkInterfaces(net);
}
boolean hasCustomTypeTag = false;
HashSet<Tag> instTags = null;
if (tags != null && !tags.isEmpty()) {
instTags = new HashSet<Tag>();
for (EC2Tag t : tags) {
instTags.add(new Tag(t.getName(), t.getValue()));
diFilters.add(new Filter("tag:" + t.getName()).withValues(t.getValue()));
if (StringUtils.equals(t.getName(), EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE)) {
hasCustomTypeTag = true;
}
}
}
if (!hasCustomTypeTag) {
if (instTags == null) {
instTags = new HashSet<Tag>();
}
// Append template description as well to identify slaves provisioned per template
instTags.add(
new Tag(
EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE,
EC2Cloud.getSlaveTypeTagValue(EC2Cloud.EC2_SLAVE_TYPE_DEMAND, description)));
}
DescribeInstancesRequest diRequest = new DescribeInstancesRequest();
diRequest.setFilters(diFilters);
logProvision(logger, "Looking for existing instances with describe-instance: " + diRequest);
DescribeInstancesResult diResult = ec2.describeInstances(diRequest);
EC2AbstractSlave[] ec2Node = new EC2AbstractSlave[1];
Instance existingInstance = null;
if (!provisionOptions.contains(ProvisionOptions.FORCE_CREATE)) {
reservationLoop:
for (Reservation reservation : diResult.getReservations()) {
for (Instance instance : reservation.getInstances()) {
if (checkInstance(logger, instance, requiredLabel, ec2Node)) {
existingInstance = instance;
logProvision(
logger,
"Found existing instance: "
+ existingInstance
+ ((ec2Node[0] != null) ? (" node: " + ec2Node[0].getInstanceId()) : ""));
break reservationLoop;
}
}
}
}
if (existingInstance == null) {
if (!provisionOptions.contains(ProvisionOptions.FORCE_CREATE)
&& !provisionOptions.contains(ProvisionOptions.ALLOW_CREATE)) {
logProvision(logger, "No existing instance found - but cannot create new instance");
return null;
}
if (StringUtils.isNotBlank(getIamInstanceProfile())) {
riRequest.setIamInstanceProfile(
new IamInstanceProfileSpecification().withArn(getIamInstanceProfile()));
}
// Have to create a new instance
Instance inst = ec2.runInstances(riRequest).getReservation().getInstances().get(0);
/* Now that we have our instance, we can set tags on it */
if (instTags != null) {
updateRemoteTags(ec2, instTags, "InvalidInstanceID.NotFound", inst.getInstanceId());
// That was a remote request - we should also update our
// local instance data.
inst.setTags(instTags);
}
logProvisionInfo(logger, "No existing instance found - created new instance: " + inst);
return newOndemandSlave(inst);
}
if (existingInstance
.getState()
.getName()
.equalsIgnoreCase(InstanceStateName.Stopping.toString())
|| existingInstance
.getState()
.getName()
.equalsIgnoreCase(InstanceStateName.Stopped.toString())) {
List<String> instances = new ArrayList<String>();
instances.add(existingInstance.getInstanceId());
StartInstancesRequest siRequest = new StartInstancesRequest(instances);
StartInstancesResult siResult = ec2.startInstances(siRequest);
logProvisionInfo(
logger,
"Found stopped instance - starting it: " + existingInstance + " result:" + siResult);
} else {
// Should be pending or running at this point, just let it come up
logProvisionInfo(
logger,
"Found existing pending or running: "
+ existingInstance.getState().getName()
+ " instance: "
+ existingInstance);
}
if (ec2Node[0] != null) {
logProvisionInfo(logger, "Using existing slave: " + ec2Node[0].getInstanceId());
return ec2Node[0];
}
// Existing slave not found
logProvision(logger, "Creating new slave for existing instance: " + existingInstance);
return newOndemandSlave(existingInstance);
} catch (FormException e) {
throw new AssertionError(e); // we should have discovered all
// configuration issues upfront
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
@Override
public List<DiscoveryNode> buildDynamicNodes() {
List<DiscoveryNode> discoNodes = new ArrayList<>();
DescribeInstancesResult descInstances;
try {
// Query EC2 API based on AZ, instance state, and tag.
// NOTE: we don't filter by security group during the describe instances request for two
// reasons:
// 1. differences in VPCs require different parameters during query (ID vs Name)
// 2. We want to use two different strategies: (all security groups vs. any security groups)
descInstances = client.describeInstances(buildDescribeInstancesRequest());
} catch (AmazonClientException e) {
logger.info("Exception while retrieving instance list from AWS API: {}", e.getMessage());
logger.debug("Full exception:", e);
return discoNodes;
}
logger.trace("building dynamic unicast discovery nodes...");
for (Reservation reservation : descInstances.getReservations()) {
for (Instance instance : reservation.getInstances()) {
// lets see if we can filter based on groups
if (!groups.isEmpty()) {
List<GroupIdentifier> instanceSecurityGroups = instance.getSecurityGroups();
ArrayList<String> securityGroupNames = new ArrayList<String>();
ArrayList<String> securityGroupIds = new ArrayList<String>();
for (GroupIdentifier sg : instanceSecurityGroups) {
securityGroupNames.add(sg.getGroupName());
securityGroupIds.add(sg.getGroupId());
}
if (bindAnyGroup) {
// We check if we can find at least one group name or one group id in groups.
if (Collections.disjoint(securityGroupNames, groups)
&& Collections.disjoint(securityGroupIds, groups)) {
logger.trace(
"filtering out instance {} based on groups {}, not part of {}",
instance.getInstanceId(),
instanceSecurityGroups,
groups);
// continue to the next instance
continue;
}
} else {
// We need tp match all group names or group ids, otherwise we ignore this instance
if (!(securityGroupNames.containsAll(groups) || securityGroupIds.containsAll(groups))) {
logger.trace(
"filtering out instance {} based on groups {}, does not include all of {}",
instance.getInstanceId(),
instanceSecurityGroups,
groups);
// continue to the next instance
continue;
}
}
}
String address = null;
switch (hostType) {
case PRIVATE_DNS:
address = instance.getPrivateDnsName();
break;
case PRIVATE_IP:
address = instance.getPrivateIpAddress();
break;
case PUBLIC_DNS:
address = instance.getPublicDnsName();
break;
case PUBLIC_IP:
address = instance.getPublicIpAddress();
break;
}
if (address != null) {
try {
TransportAddress[] addresses = transportService.addressesFromString(address);
// we only limit to 1 addresses, makes no sense to ping 100 ports
for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) {
logger.trace(
"adding {}, address {}, transport_address {}",
instance.getInstanceId(),
address,
addresses[i]);
discoNodes.add(
new DiscoveryNode(
"#cloud-" + instance.getInstanceId() + "-" + i,
addresses[i],
version.minimumCompatibilityVersion()));
}
} catch (Exception e) {
logger.warn("failed ot add {}, address {}", e, instance.getInstanceId(), address);
}
} else {
logger.trace(
"not adding {}, address is null, host_type {}", instance.getInstanceId(), hostType);
}
}
}
logger.debug("using dynamic discovery nodes {}", discoNodes);
return discoNodes;
}
public void run() {
logger.debug("started GridManager");
System.out.println("started GridManager");
AmazonSQSClient sqsClient = new AmazonSQSClient(papa.getCredentials());
AmazonEC2Client ec2Client = new AmazonEC2Client(papa.getCredentials());
// Get current imageID from the metadata table
String imageID = Constants.imageID; // get the default imageID from Constants
try {
Connection conn = PooledConnectionFactory.INSTANCE.getCumulusConnection();
Statement stmt = conn.createStatement();
String imageQuery = "SELECT mvalue FROM cumulus.metadata WHERE mkey='imageID';";
ResultSet results = stmt.executeQuery(imageQuery);
results.next(); // move the cursor into the results
imageID = results.getString(1);
} catch (SQLException e) {
logger.warn(e);
}
if (imageID == null
|| imageID.equals("")) { // if we get an empty imageID revert to the hardcoded one
imageID = Constants.imageID;
}
logger.debug(imageID);
System.out.println(imageID);
while (true) { // loop from startup to shutdown
// check the number of messages in the queue.
GetQueueAttributesRequest sqsRequest =
new GetQueueAttributesRequest(papa.getDispatchQueue())
.withAttributeNames("ApproximateNumberOfMessages");
Integer sqsResult = null;
try {
sqsResult =
Integer.valueOf(
sqsClient
.getQueueAttributes(sqsRequest)
.getAttributes()
.get("ApproximateNumberOfMessages"));
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.err.println("Error getting list of instances");
System.err.println("Caught Exception: " + e.getMessage());
System.err.println("Reponse Status Code: " + e.getStatusCode());
System.err.println("Error Code: " + e.getErrorCode());
System.err.println("Request ID: " + e.getRequestId());
} catch (AmazonClientException e) {
logger.warn(e);
continue; // if the error is due to being unable to connect to sqs it is likely transient,
// and we should keep going until SQS comes back
} catch (Exception e) {
// caught another exception
System.err.println(e);
System.err.println("Did not exit cleanly");
// System.exit(1);
}
if (sqsResult == null) {
System.err.println(
"Couldn't get the number of items in the queue reverting to static functioning");
logger.error(
"Couldn't get the number of items in the queue reverting to static functioning");
break;
}
System.out.println("Queue size=" + sqsResult);
// check the number of running instances.
DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest();
// get the number of running instances with our image ID
describeInstancesRequest.withFilters(
new Filter("image-id").withValues(imageID),
new Filter("instance-state-name").withValues("running"));
logger.debug("Getting list of active cumulus drones");
Integer numInstances = null;
Integer numSpotRequests = null;
Initializer.serialize("/tmp/dump.obj", papa.unitsOnServer); // TODO test this.
// check number of open spot requests
DescribeSpotInstanceRequestsRequest describeSpotInstancesRequest =
new DescribeSpotInstanceRequestsRequest();
describeSpotInstancesRequest.withFilters(new Filter("state").withValues("open"));
try {
// Parse returned instances
DescribeInstancesResult describeInstancesResult =
ec2Client.describeInstances(describeInstancesRequest);
if (describeInstancesResult.getReservations().size() == 0) {
numInstances = 0;
} else {
Integer totalInstances = 0;
for (Reservation i : describeInstancesResult.getReservations()) {
totalInstances += i.getInstances().size();
}
numInstances = totalInstances;
}
// Parse returned open spot requests
DescribeSpotInstanceRequestsResult describeSpotReservationsResult =
ec2Client.describeSpotInstanceRequests(describeSpotInstancesRequest);
if (describeSpotReservationsResult.getSpotInstanceRequests().size() == 0) {
numSpotRequests = 0;
} else {
Integer totalInstances = 0;
for (SpotInstanceRequest i : describeSpotReservationsResult.getSpotInstanceRequests()) {
totalInstances++;
}
numSpotRequests = totalInstances;
}
// System.out.println("num instances="+numInstances);
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.err.println("Error getting list of instances");
System.err.println("Caught Exception: " + e.getMessage());
System.err.println("Reponse Status Code: " + e.getStatusCode());
System.err.println("Error Code: " + e.getErrorCode());
System.err.println("Request ID: " + e.getRequestId());
// System.exit(1);
} catch (Exception e) {
// caught another exception
System.err.println(e);
System.err.println("Did not exit cleanly");
// System.exit(1);
}
if (numInstances == null || numSpotRequests == null) {
logger.error(
"Couldn't get the number of machines or requests reverting to static functioning");
break;
}
System.out.println("num instances=" + (numInstances + numSpotRequests));
// if queue is longer than the number of instances * idealMaxUnitsPerInstance
Integer idealMaxUnitsPerInstance = Integer.valueOf(Constants.idealMaxUnitsPerInstance);
if (sqsResult
> (numInstances + numSpotRequests)
* idealMaxUnitsPerInstance) { // we count pending spot instances too.
Integer numToCreate = (sqsResult / idealMaxUnitsPerInstance) - numInstances + 1;
// check current pricing of spot instances
DescribeSpotPriceHistoryRequest historyRequest =
new DescribeSpotPriceHistoryRequest()
.withAvailabilityZone("us-east-1c")
.withInstanceTypes(Constants.instanceType)
.withProductDescriptions("Linux/UNIX (Amazon VPC)")
.withStartTime(new Date(System.currentTimeMillis()));
DescribeSpotPriceHistoryResult result = ec2Client.describeSpotPriceHistory(historyRequest);
Double nowPrice = Double.valueOf(result.getSpotPriceHistory().get(0).getSpotPrice());
// If spot price is < constants.spotPrice launch numToCreate*Constants.percentSpot spot
// instances
Integer spotToCreate = 0;
if (nowPrice < Double.valueOf(Constants.spotPrice)) {
Double percentSpot = Double.valueOf(Constants.percentSpot);
spotToCreate = (int) (numToCreate * percentSpot);
numToCreate = (int) (numToCreate * (1 - percentSpot));
}
try {
System.out.println("regular instances: " + numToCreate);
System.out.println("spot instances: " + spotToCreate);
// Create regular and spot instances
if (numToCreate > 0) {
papa.createInstances(ec2Client, numToCreate, imageID);
}
if (spotToCreate > 0) {
papa.createSpotInstances(ec2Client, spotToCreate, Constants.imageID); // FIXME
}
System.out.println("created instances");
} catch (Exception e) {
// Report ec2 exceptions
logger.error(e);
e.printStackTrace();
continue;
}
}
try {
Thread.sleep(60000); // we wait for any new servers to start up
} catch (InterruptedException e) {
logger.error("Sqslistener was interrupted");
if (papa.getShuttingDown()) {
break;
} else {
continue;
}
}
}
}
