public static void main(String[] args) throws UnknownHostException {
AmazonEC2 ec2 = new AmazonEC2Client(new ClasspathPropertiesFileCredentialsProvider());
Region usWest2 = Region.getRegion(Regions.US_WEST_2);
ec2.setRegion(usWest2);
List<String> instanceIds = new ArrayList<>();
String instanceid = null;
DescribeInstancesRequest request = new DescribeInstancesRequest();
List<String> valuesT1 = new ArrayList<String>();
valuesT1.add("hz-nodes");
Filter filter = new Filter("tag-value", valuesT1);
DescribeInstancesResult result = ec2.describeInstances(request.withFilters(filter));
List<Reservation> reservations = result.getReservations();
for (Reservation reservation : reservations) {
List<Instance> instances = reservation.getInstances();
for (Instance instance : instances) {
instanceid = instance.getInstanceId();
PrintManager.PrintMessage(instanceid);
instanceIds.add(instanceid);
List<Tag> abc = instance.getTags();
for (Tag aaa : abc) {
PrintManager.PrintMessage(aaa.getKey() + " : " + aaa.getValue());
}
}
}
instanceIds.remove("i-cb45adfc");
StopInstancesRequest stopReq = new StopInstancesRequest(instanceIds);
ec2.stopInstances(stopReq);
RebootInstancesRequest requestRe = new RebootInstancesRequest(instanceIds);
ec2.rebootInstances(requestRe);
}
private DescribeInstancesRequest buildDescribeInstancesRequest() {
DescribeInstancesRequest describeInstancesRequest =
new DescribeInstancesRequest()
.withFilters(new Filter("instance-state-name").withValues("running", "pending"));
for (Map.Entry<String, String> tagFilter : tags.entrySet()) {
// for a given tag key, OR relationship for multiple different values
describeInstancesRequest.withFilters(
new Filter("tag:" + tagFilter.getKey()).withValues(tagFilter.getValue()));
}
if (!availabilityZones.isEmpty()) {
// OR relationship amongst multiple values of the availability-zone filter
describeInstancesRequest.withFilters(
new Filter("availability-zone").withValues(availabilityZones));
}
return describeInstancesRequest;
}
// Returns pending/running instance ID list.
public static ArrayList<String> getInstanceIdsByTag(AmazonEC2 ec2, String key, String value) {
logger.fine("Requesting instances by tag " + key + "=" + value);
// Create tag request.
DescribeInstancesRequest instanceReq = new DescribeInstancesRequest();
Filter managerFilter = new Filter("tag:" + key).withValues(value),
activeFilter =
new Filter("instance-state-code").withValues("0", "16"); // 0||16 == pending||running
// Send request.
DescribeInstancesResult result =
ec2.describeInstances(instanceReq.withFilters(managerFilter, activeFilter));
// Build instance ID list.
ArrayList<String> ids = new ArrayList<String>();
for (Reservation reservation : result.getReservations()) {
for (Instance instance : reservation.getInstances()) {
ids.add(instance.getInstanceId());
}
}
logger.fine(ids.size() + " instances found with tag " + key + "=" + value);
return ids;
}
public void run() {
logger.debug("started GridManager");
System.out.println("started GridManager");
AmazonSQSClient sqsClient = new AmazonSQSClient(papa.getCredentials());
AmazonEC2Client ec2Client = new AmazonEC2Client(papa.getCredentials());
// Get current imageID from the metadata table
String imageID = Constants.imageID; // get the default imageID from Constants
try {
Connection conn = PooledConnectionFactory.INSTANCE.getCumulusConnection();
Statement stmt = conn.createStatement();
String imageQuery = "SELECT mvalue FROM cumulus.metadata WHERE mkey='imageID';";
ResultSet results = stmt.executeQuery(imageQuery);
results.next(); // move the cursor into the results
imageID = results.getString(1);
} catch (SQLException e) {
logger.warn(e);
}
if (imageID == null
|| imageID.equals("")) { // if we get an empty imageID revert to the hardcoded one
imageID = Constants.imageID;
}
logger.debug(imageID);
System.out.println(imageID);
while (true) { // loop from startup to shutdown
// check the number of messages in the queue.
GetQueueAttributesRequest sqsRequest =
new GetQueueAttributesRequest(papa.getDispatchQueue())
.withAttributeNames("ApproximateNumberOfMessages");
Integer sqsResult = null;
try {
sqsResult =
Integer.valueOf(
sqsClient
.getQueueAttributes(sqsRequest)
.getAttributes()
.get("ApproximateNumberOfMessages"));
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.err.println("Error getting list of instances");
System.err.println("Caught Exception: " + e.getMessage());
System.err.println("Reponse Status Code: " + e.getStatusCode());
System.err.println("Error Code: " + e.getErrorCode());
System.err.println("Request ID: " + e.getRequestId());
} catch (AmazonClientException e) {
logger.warn(e);
continue; // if the error is due to being unable to connect to sqs it is likely transient,
// and we should keep going until SQS comes back
} catch (Exception e) {
// caught another exception
System.err.println(e);
System.err.println("Did not exit cleanly");
// System.exit(1);
}
if (sqsResult == null) {
System.err.println(
"Couldn't get the number of items in the queue reverting to static functioning");
logger.error(
"Couldn't get the number of items in the queue reverting to static functioning");
break;
}
System.out.println("Queue size=" + sqsResult);
// check the number of running instances.
DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest();
// get the number of running instances with our image ID
describeInstancesRequest.withFilters(
new Filter("image-id").withValues(imageID),
new Filter("instance-state-name").withValues("running"));
logger.debug("Getting list of active cumulus drones");
Integer numInstances = null;
Integer numSpotRequests = null;
Initializer.serialize("/tmp/dump.obj", papa.unitsOnServer); // TODO test this.
// check number of open spot requests
DescribeSpotInstanceRequestsRequest describeSpotInstancesRequest =
new DescribeSpotInstanceRequestsRequest();
describeSpotInstancesRequest.withFilters(new Filter("state").withValues("open"));
try {
// Parse returned instances
DescribeInstancesResult describeInstancesResult =
ec2Client.describeInstances(describeInstancesRequest);
if (describeInstancesResult.getReservations().size() == 0) {
numInstances = 0;
} else {
Integer totalInstances = 0;
for (Reservation i : describeInstancesResult.getReservations()) {
totalInstances += i.getInstances().size();
}
numInstances = totalInstances;
}
// Parse returned open spot requests
DescribeSpotInstanceRequestsResult describeSpotReservationsResult =
ec2Client.describeSpotInstanceRequests(describeSpotInstancesRequest);
if (describeSpotReservationsResult.getSpotInstanceRequests().size() == 0) {
numSpotRequests = 0;
} else {
Integer totalInstances = 0;
for (SpotInstanceRequest i : describeSpotReservationsResult.getSpotInstanceRequests()) {
totalInstances++;
}
numSpotRequests = totalInstances;
}
// System.out.println("num instances="+numInstances);
} catch (AmazonServiceException e) {
// Write out any exceptions that may have occurred.
System.err.println("Error getting list of instances");
System.err.println("Caught Exception: " + e.getMessage());
System.err.println("Reponse Status Code: " + e.getStatusCode());
System.err.println("Error Code: " + e.getErrorCode());
System.err.println("Request ID: " + e.getRequestId());
// System.exit(1);
} catch (Exception e) {
// caught another exception
System.err.println(e);
System.err.println("Did not exit cleanly");
// System.exit(1);
}
if (numInstances == null || numSpotRequests == null) {
logger.error(
"Couldn't get the number of machines or requests reverting to static functioning");
break;
}
System.out.println("num instances=" + (numInstances + numSpotRequests));
// if queue is longer than the number of instances * idealMaxUnitsPerInstance
Integer idealMaxUnitsPerInstance = Integer.valueOf(Constants.idealMaxUnitsPerInstance);
if (sqsResult
> (numInstances + numSpotRequests)
* idealMaxUnitsPerInstance) { // we count pending spot instances too.
Integer numToCreate = (sqsResult / idealMaxUnitsPerInstance) - numInstances + 1;
// check current pricing of spot instances
DescribeSpotPriceHistoryRequest historyRequest =
new DescribeSpotPriceHistoryRequest()
.withAvailabilityZone("us-east-1c")
.withInstanceTypes(Constants.instanceType)
.withProductDescriptions("Linux/UNIX (Amazon VPC)")
.withStartTime(new Date(System.currentTimeMillis()));
DescribeSpotPriceHistoryResult result = ec2Client.describeSpotPriceHistory(historyRequest);
Double nowPrice = Double.valueOf(result.getSpotPriceHistory().get(0).getSpotPrice());
// If spot price is < constants.spotPrice launch numToCreate*Constants.percentSpot spot
// instances
Integer spotToCreate = 0;
if (nowPrice < Double.valueOf(Constants.spotPrice)) {
Double percentSpot = Double.valueOf(Constants.percentSpot);
spotToCreate = (int) (numToCreate * percentSpot);
numToCreate = (int) (numToCreate * (1 - percentSpot));
}
try {
System.out.println("regular instances: " + numToCreate);
System.out.println("spot instances: " + spotToCreate);
// Create regular and spot instances
if (numToCreate > 0) {
papa.createInstances(ec2Client, numToCreate, imageID);
}
if (spotToCreate > 0) {
papa.createSpotInstances(ec2Client, spotToCreate, Constants.imageID); // FIXME
}
System.out.println("created instances");
} catch (Exception e) {
// Report ec2 exceptions
logger.error(e);
e.printStackTrace();
continue;
}
}
try {
Thread.sleep(60000); // we wait for any new servers to start up
} catch (InterruptedException e) {
logger.error("Sqslistener was interrupted");
if (papa.getShuttingDown()) {
break;
} else {
continue;
}
}
}
}
