Esempio n. 1
0
 /**
  * Counts the number of instances in EC2 currently running that are using the specified image and
  * a template.
  *
  * @param ami If AMI is left null, then all instances are counted and template description is
  *     ignored.
  *     <p>This includes those instances that may be started outside Jenkins.
  * @param templateDesc
  */
 public int countCurrentEC2Slaves(String ami, String templateDesc) throws AmazonClientException {
   int n = 0;
   for (Reservation r : connect().describeInstances().getReservations()) {
     for (Instance i : r.getInstances()) {
       if (isEc2ProvisionedAmiSlave(i, ami, templateDesc)) {
         InstanceStateName stateName = InstanceStateName.fromValue(i.getState().getName());
         if (stateName == InstanceStateName.Pending || stateName == InstanceStateName.Running) {
           EC2AbstractSlave foundSlave = null;
           for (EC2AbstractSlave ec2Node : NodeIterator.nodes(EC2AbstractSlave.class)) {
             if (ec2Node.getInstanceId().equals(i.getInstanceId())) {
               foundSlave = ec2Node;
               break;
             }
           }
           // Don't count disconnected slaves as being used, we will connected them later is
           // required
           if (foundSlave != null && foundSlave.toComputer().isOffline()) continue;
           n++;
         }
       }
     }
   }
   // Count pending spot requests too
   for (SpotInstanceRequest sir :
       connect().describeSpotInstanceRequests().getSpotInstanceRequests()) {
     // Count Spot requests that are open and still have a
     // chance to be active.
     if (sir.getState().equals("open")) {
       n++;
     }
   }
   return n;
 }
Esempio n. 2
0
 protected EC2SpotSlave newSpotSlave(SpotInstanceRequest sir, String name)
     throws FormException, IOException {
   return new EC2SpotSlave(
       name,
       sir.getSpotInstanceRequestId(),
       description,
       remoteFS,
       getNumExecutors(),
       mode,
       initScript,
       tmpDir,
       labels,
       remoteAdmin,
       jvmopts,
       idleTerminationMinutes,
       EC2Tag.fromAmazonTags(sir.getTags()),
       parent.name,
       usePrivateDnsName,
       getLaunchTimeout(),
       amiType);
 }
  public static String setUpSpotInstanceWithImage(String image) {
    System.out.println("Putting spot requet for image: " + image);

    RequestSpotInstancesRequest loadGeneratorRequest = new RequestSpotInstancesRequest();

    loadGeneratorRequest.setSpotPrice("0.03");
    loadGeneratorRequest.setInstanceCount(Integer.valueOf(1));

    LaunchSpecification loadGeneratorSpecification = new LaunchSpecification();
    loadGeneratorSpecification.setImageId(image);
    loadGeneratorSpecification.setInstanceType("m3.medium");

    ArrayList<String> securityGroups = new ArrayList<String>();
    securityGroups.add("SecurityGroupForLoadGenerator");
    loadGeneratorSpecification.setSecurityGroups(securityGroups);

    loadGeneratorRequest.setLaunchSpecification(loadGeneratorSpecification);

    RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(loadGeneratorRequest);
    List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();

    String requestId = requestResponses.get(0).getSpotInstanceRequestId();

    ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();
    spotInstanceRequestIds.add(requestId);

    System.out.println("Finding Instance ID..");

    boolean anyOpen;

    ArrayList<String> instanceIds = new ArrayList<String>();

    do {
      DescribeSpotInstanceRequestsRequest describeRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);

      anyOpen = false;

      try {
        DescribeSpotInstanceRequestsResult describeResult =
            ec2.describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();

        for (SpotInstanceRequest describeResponse : describeResponses) {
          if (describeResponse.getState().equals("open")) {
            anyOpen = true;
            break;
          }

          instanceIds.add(describeResponse.getInstanceId());
        }
      } catch (AmazonServiceException e) {
        anyOpen = true;
      }

      sleep(30);
    } while (anyOpen);

    return instanceIds.get(0);
  }
Esempio n. 4
0
  /** Provision a new slave for an EC2 spot instance to call back to Jenkins */
  private EC2AbstractSlave provisionSpot(TaskListener listener)
      throws AmazonClientException, IOException {
    PrintStream logger = listener.getLogger();
    AmazonEC2 ec2 = getParent().connect();

    try {
      logger.println("Launching " + ami + " for template " + description);
      LOGGER.info("Launching " + ami + " for template " + description);

      KeyPair keyPair = getKeyPair(ec2);

      RequestSpotInstancesRequest spotRequest = new RequestSpotInstancesRequest();

      // Validate spot bid before making the request
      if (getSpotMaxBidPrice() == null) {
        // throw new FormException("Invalid Spot price specified: " +
        // getSpotMaxBidPrice(), "spotMaxBidPrice");
        throw new AmazonClientException("Invalid Spot price specified: " + getSpotMaxBidPrice());
      }

      spotRequest.setSpotPrice(getSpotMaxBidPrice());
      spotRequest.setInstanceCount(1);

      LaunchSpecification launchSpecification = new LaunchSpecification();
      InstanceNetworkInterfaceSpecification net = new InstanceNetworkInterfaceSpecification();

      launchSpecification.setImageId(ami);
      launchSpecification.setInstanceType(type);

      if (StringUtils.isNotBlank(getZone())) {
        SpotPlacement placement = new SpotPlacement(getZone());
        launchSpecification.setPlacement(placement);
      }

      if (StringUtils.isNotBlank(getSubnetId())) {
        if (getAssociatePublicIp()) {
          net.setSubnetId(getSubnetId());
        } else {
          launchSpecification.setSubnetId(getSubnetId());
        }

        /*
         * If we have a subnet ID then we can only use VPC security groups
         */
        if (!securityGroupSet.isEmpty()) {
          List<String> groupIds = getEc2SecurityGroups(ec2);
          if (!groupIds.isEmpty()) {
            if (getAssociatePublicIp()) {
              net.setGroups(groupIds);
            } else {
              ArrayList<GroupIdentifier> groups = new ArrayList<GroupIdentifier>();

              for (String group_id : groupIds) {
                GroupIdentifier group = new GroupIdentifier();
                group.setGroupId(group_id);
                groups.add(group);
              }
              if (!groups.isEmpty()) launchSpecification.setAllSecurityGroups(groups);
            }
          }
        }
      } else {
        /* No subnet: we can use standard security groups by name */
        if (!securityGroupSet.isEmpty()) {
          launchSpecification.setSecurityGroups(securityGroupSet);
        }
      }

      String userDataString = Base64.encodeBase64String(userData.getBytes(StandardCharsets.UTF_8));

      launchSpecification.setUserData(userDataString);
      launchSpecification.setKeyName(keyPair.getKeyName());
      launchSpecification.setInstanceType(type.toString());

      if (getAssociatePublicIp()) {
        net.setAssociatePublicIpAddress(true);
        net.setDeviceIndex(0);
        launchSpecification.withNetworkInterfaces(net);
      }

      boolean hasCustomTypeTag = false;
      HashSet<Tag> instTags = null;
      if (tags != null && !tags.isEmpty()) {
        instTags = new HashSet<Tag>();
        for (EC2Tag t : tags) {
          instTags.add(new Tag(t.getName(), t.getValue()));
          if (StringUtils.equals(t.getName(), EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE)) {
            hasCustomTypeTag = true;
          }
        }
      }
      if (!hasCustomTypeTag) {
        if (instTags != null)
          instTags.add(
              new Tag(
                  EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE,
                  EC2Cloud.getSlaveTypeTagValue(EC2Cloud.EC2_SLAVE_TYPE_SPOT, description)));
      }

      if (StringUtils.isNotBlank(getIamInstanceProfile())) {
        launchSpecification.setIamInstanceProfile(
            new IamInstanceProfileSpecification().withArn(getIamInstanceProfile()));
      }

      if (useEphemeralDevices) {
        setupEphemeralDeviceMapping(launchSpecification);
      } else {
        setupCustomDeviceMapping(launchSpecification);
      }

      spotRequest.setLaunchSpecification(launchSpecification);

      // Make the request for a new Spot instance
      RequestSpotInstancesResult reqResult = ec2.requestSpotInstances(spotRequest);

      List<SpotInstanceRequest> reqInstances = reqResult.getSpotInstanceRequests();
      if (reqInstances.isEmpty()) {
        throw new AmazonClientException("No spot instances found");
      }

      SpotInstanceRequest spotInstReq = reqInstances.get(0);
      if (spotInstReq == null) {
        throw new AmazonClientException("Spot instance request is null");
      }
      String slaveName = spotInstReq.getSpotInstanceRequestId();

      /* Now that we have our Spot request, we can set tags on it */
      if (instTags != null) {
        updateRemoteTags(
            ec2,
            instTags,
            "InvalidSpotInstanceRequestID.NotFound",
            spotInstReq.getSpotInstanceRequestId());

        // That was a remote request - we should also update our local
        // instance data.
        spotInstReq.setTags(instTags);
      }

      logger.println("Spot instance id in provision: " + spotInstReq.getSpotInstanceRequestId());
      LOGGER.info("Spot instance id in provision: " + spotInstReq.getSpotInstanceRequestId());

      return newSpotSlave(spotInstReq, slaveName);

    } catch (FormException e) {
      throw new AssertionError(); // we should have discovered all
      // configuration issues upfront
    } catch (InterruptedException e) {
      throw new RuntimeException(e);
    }
  }
  /** @param args */
  public static void main(String[] args) {
    // ============================================================================================//
    // =============================== Submitting a Request
    // =======================================//
    // ============================================================================================//

    // Create the AmazonEC2Client object so we can call various APIs.
    AmazonEC2 ec2 = new AmazonEC2Client(new ClasspathPropertiesFileCredentialsProvider());
    Region usWest2 = Region.getRegion(Regions.US_WEST_2);
    ec2.setRegion(usWest2);

    // Initializes a Spot Instance Request
    RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest();

    // *************************** Required Parameters Settings ************************//
    // Request 1 x t1.micro instance with a bid price of $0.03.
    requestRequest.setSpotPrice("0.03");
    requestRequest.setInstanceCount(Integer.valueOf(1));

    // Setup the specifications of the launch. This includes the instance type (e.g. t1.micro)
    // and the latest Amazon Linux AMI id available. Note, you should always use the latest
    // Amazon Linux AMI id or another of your choosing.
    LaunchSpecification launchSpecification = new LaunchSpecification();
    launchSpecification.setImageId("ami-8c1fece5");
    launchSpecification.setInstanceType("t1.micro");

    // Add the security group to the request.
    ArrayList<String> securityGroups = new ArrayList<String>();
    securityGroups.add("GettingStartedGroup");
    launchSpecification.setSecurityGroups(securityGroups);

    // *************************** Bid Type Settings ************************//
    // Set the type of the bid to persistent.
    requestRequest.setType("persistent");

    // *************************** Valid From/To Settings ************************//
    // Set the valid start time to be two minutes from now.
    Calendar from = Calendar.getInstance();
    from.add(Calendar.MINUTE, 2);
    requestRequest.setValidFrom(from.getTime());

    // Set the valid end time to be two minutes and two hours from now.
    Calendar until = (Calendar) from.clone();
    until.add(Calendar.HOUR, 2);
    requestRequest.setValidUntil(until.getTime());

    // *************************** Launch Group Settings ************************//
    // Set the launch group.
    requestRequest.setLaunchGroup("ADVANCED-DEMO-LAUNCH-GROUP");

    // *************************** Availability Zone Group Settings ************************//
    // Set the availability zone group.
    requestRequest.setAvailabilityZoneGroup("ADVANCED-DEMO-AZ-GROUP");

    // *************************** Add the block device mapping ************************//

    // Goal: Setup block device mappings to ensure that we will not delete
    // the root partition on termination.

    // Create the block device mapping to describe the root partition.
    BlockDeviceMapping blockDeviceMapping = new BlockDeviceMapping();
    blockDeviceMapping.setDeviceName("/dev/sda1");

    // Set the delete on termination flag to false.
    EbsBlockDevice ebs = new EbsBlockDevice();
    ebs.setDeleteOnTermination(Boolean.FALSE);
    blockDeviceMapping.setEbs(ebs);

    // Add the block device mapping to the block list.
    ArrayList<BlockDeviceMapping> blockList = new ArrayList<BlockDeviceMapping>();
    blockList.add(blockDeviceMapping);

    // Set the block device mapping configuration in the launch specifications.
    launchSpecification.setBlockDeviceMappings(blockList);

    // *************************** Add the availability zone ************************//
    // Setup the availability zone to use. Note we could retrieve the availability
    // zones using the ec2.describeAvailabilityZones() API. For this demo we will just use
    // us-east-1b.
    SpotPlacement placement = new SpotPlacement("us-east-1b");
    launchSpecification.setPlacement(placement);

    // *************************** Add the placement group ************************//
    // Setup the placement group to use with whatever name you desire.
    // For this demo we will just use "ADVANCED-DEMO-PLACEMENT-GROUP".
    // Note: We have commented this out, because we are not leveraging cc1.4xlarge or
    // cg1.4xlarge in this example.
    /*
    SpotPlacement pg = new SpotPlacement();
    pg.setGroupName("ADVANCED-DEMO-PLACEMENT-GROUP");
    launchSpecification.setPlacement(pg);
    */

    // *************************** Add the launch specification ************************//
    // Add the launch specification.
    requestRequest.setLaunchSpecification(launchSpecification);

    // ============================================================================================//
    // =========================== Getting the Request ID from the Request
    // ========================//
    // ============================================================================================//

    // Call the RequestSpotInstance API.
    RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest);
    List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();

    // Setup an arraylist to collect all of the request ids we want to watch hit the running
    // state.
    ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();

    // Add all of the request ids to the hashset, so we can determine when they hit the
    // active state.
    for (SpotInstanceRequest requestResponse : requestResponses) {
      System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId());
      spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId());
    }

    // ============================================================================================//
    // =========================== Determining the State of the Spot Request
    // ======================//
    // ============================================================================================//

    // Create a variable that will track whether there are any requests still in the open state.
    boolean anyOpen;

    // Initialize variables.
    ArrayList<String> instanceIds = new ArrayList<String>();

    do {
      // Create the describeRequest with tall of the request id to monitor (e.g. that we started).
      DescribeSpotInstanceRequestsRequest describeRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);

      // Initialize the anyOpen variable to false ??? which assumes there are no requests open
      // unless
      // we find one that is still open.
      anyOpen = false;

      try {
        // Retrieve all of the requests we want to monitor.
        DescribeSpotInstanceRequestsResult describeResult =
            ec2.describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();

        // Look through each request and determine if they are all in the active state.
        for (SpotInstanceRequest describeResponse : describeResponses) {
          // If the state is open, it hasn't changed since we attempted to request it.
          // There is the potential for it to transition almost immediately to closed or
          // cancelled so we compare against open instead of active.
          if (describeResponse.getState().equals("open")) {
            anyOpen = true;
            break;
          }

          // Add the instance id to the list we will eventually terminate.
          instanceIds.add(describeResponse.getInstanceId());
        }
      } catch (AmazonServiceException e) {
        // If we have an exception, ensure we don't break out of the loop.
        // This prevents the scenario where there was blip on the wire.
        anyOpen = true;
      }

      try {
        // Sleep for 60 seconds.
        Thread.sleep(60 * 1000);
      } catch (Exception e) {
        // Do nothing because it woke up early.
      }
    } while (anyOpen);

    // ============================================================================================//
    // ====================================== Canceling the Request ==============================//
    // ============================================================================================//

    try {
      // Cancel requests.
      CancelSpotInstanceRequestsRequest cancelRequest =
          new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds);
      ec2.cancelSpotInstanceRequests(cancelRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error cancelling instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // =================================== Terminating any Instances
    // ==============================//
    // ============================================================================================//
    try {
      // Terminate instances.
      TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds);
      ec2.terminateInstances(terminateRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }
  }
Esempio n. 6
0
  @Override
  public Collection<PlannedNode> provision(Label label, int excessWorkload) {
    try {
      // Count number of pending executors from spot requests
      for (EC2SpotSlave n : NodeIterator.nodes(EC2SpotSlave.class)) {
        // If the slave is online then it is already counted by Jenkins
        // We only want to count potential additional Spot instance
        // slaves
        if (n.getComputer().isOffline() && label.matches(n.getAssignedLabels())) {
          DescribeSpotInstanceRequestsRequest dsir =
              new DescribeSpotInstanceRequestsRequest()
                  .withSpotInstanceRequestIds(n.getSpotInstanceRequestId());

          for (SpotInstanceRequest sir :
              connect().describeSpotInstanceRequests(dsir).getSpotInstanceRequests()) {
            // Count Spot requests that are open and still have a
            // chance to be active
            // A request can be active and not yet registered as a
            // slave. We check above
            // to ensure only unregistered slaves get counted
            if (sir.getState().equals("open") || sir.getState().equals("active")) {
              excessWorkload -= n.getNumExecutors();
            }
          }
        }
      }
      LOGGER.log(Level.INFO, "Excess workload after pending Spot instances: " + excessWorkload);

      List<PlannedNode> r = new ArrayList<PlannedNode>();

      final SlaveTemplate t = getTemplate(label);
      int amiCap = t.getInstanceCap();

      while (excessWorkload > 0) {

        if (!addProvisionedSlave(t.ami, amiCap, t.description)) {
          break;
        }

        r.add(
            new PlannedNode(
                t.getDisplayName(),
                Computer.threadPoolForRemoting.submit(
                    new Callable<Node>() {
                      public Node call() throws Exception {
                        // TODO: record the output somewhere
                        try {
                          EC2AbstractSlave s = t.provision(StreamTaskListener.fromStdout());
                          Hudson.getInstance().addNode(s);
                          // EC2 instances may have a long init script. If we
                          // declare
                          // the provisioning complete by returning without
                          // the connect
                          // operation, NodeProvisioner may decide that it
                          // still wants
                          // one more instance, because it sees that (1) all
                          // the slaves
                          // are offline (because it's still being launched)
                          // and
                          // (2) there's no capacity provisioned yet.
                          //
                          // deferring the completion of provisioning until
                          // the launch
                          // goes successful prevents this problem.
                          s.toComputer().connect(false).get();
                          return s;
                        } finally {
                          decrementAmiSlaveProvision(t.ami);
                        }
                      }
                    }),
                t.getNumExecutors()));

        excessWorkload -= t.getNumExecutors();
      }
      return r;
    } catch (AmazonClientException e) {
      LOGGER.log(Level.WARNING, "Failed to count the # of live instances on EC2", e);
      return Collections.emptyList();
    }
  }
  /** @param args */
  public static void main(String[] args) {
    // ============================================================================================//
    // =============================== Submitting a Request
    // =======================================//
    // ============================================================================================//

    // Retrieves the credentials from an AWSCredentials.properties file.
    AWSCredentials credentials = null;
    try {
      credentials =
          new PropertiesCredentials(
              InlineTaggingCodeSampleApp.class.getResourceAsStream("AwsCredentials.properties"));
    } catch (IOException e1) {
      System.out.println("Credentials were not properly entered into AwsCredentials.properties.");
      System.out.println(e1.getMessage());
      System.exit(-1);
    }

    // Create the AmazonEC2Client object so we can call various APIs.
    AmazonEC2 ec2 = new AmazonEC2Client(credentials);

    // Initializes a Spot Instance Request
    RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest();

    // Request 1 x t1.micro instance with a bid price of $0.03.
    requestRequest.setSpotPrice("0.03");
    requestRequest.setInstanceCount(Integer.valueOf(1));

    // Setup the specifications of the launch. This includes the instance type (e.g. t1.micro)
    // and the latest Amazon Linux AMI id available. Note, you should always use the latest
    // Amazon Linux AMI id or another of your choosing.
    LaunchSpecification launchSpecification = new LaunchSpecification();
    launchSpecification.setImageId("ami-8c1fece5");
    launchSpecification.setInstanceType("t1.micro");

    // Add the security group to the request.
    ArrayList<String> securityGroups = new ArrayList<String>();
    securityGroups.add("GettingStartedGroup");
    launchSpecification.setSecurityGroups(securityGroups);

    // Add the launch specifications to the request.
    requestRequest.setLaunchSpecification(launchSpecification);

    // ============================================================================================//
    // =========================== Getting the Request ID from the Request
    // ========================//
    // ============================================================================================//

    // Call the RequestSpotInstance API.
    RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest);
    List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();

    // Setup an arraylist to collect all of the request ids we want to watch hit the running
    // state.
    ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();

    // Add all of the request ids to the hashset, so we can determine when they hit the
    // active state.
    for (SpotInstanceRequest requestResponse : requestResponses) {
      System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId());
      spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId());
    }

    // ============================================================================================//
    // ====================================== Tag the Spot Requests
    // ===============================//
    // ============================================================================================//

    // Create the list of tags we want to create
    ArrayList<Tag> requestTags = new ArrayList<Tag>();
    requestTags.add(new Tag("keyname1", "value1"));

    // Create a tag request for requests.
    CreateTagsRequest createTagsRequest_requests = new CreateTagsRequest();
    createTagsRequest_requests.setResources(spotInstanceRequestIds);
    createTagsRequest_requests.setTags(requestTags);

    // Try to tag the Spot request submitted.
    try {
      ec2.createTags(createTagsRequest_requests);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // =========================== Determining the State of the Spot Request
    // ======================//
    // ============================================================================================//

    // Create a variable that will track whether there are any requests still in the open state.
    boolean anyOpen;

    // Initialize variables.
    ArrayList<String> instanceIds = new ArrayList<String>();

    do {
      // Create the describeRequest with tall of the request id to monitor (e.g. that we started).
      DescribeSpotInstanceRequestsRequest describeRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);

      // Initialize the anyOpen variable to false – which assumes there are no requests open unless
      // we find one that is still open.
      anyOpen = false;

      try {
        // Retrieve all of the requests we want to monitor.
        DescribeSpotInstanceRequestsResult describeResult =
            ec2.describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();

        // Look through each request and determine if they are all in the active state.
        for (SpotInstanceRequest describeResponse : describeResponses) {
          // If the state is open, it hasn't changed since we attempted to request it.
          // There is the potential for it to transition almost immediately to closed or
          // cancelled so we compare against open instead of active.
          if (describeResponse.getState().equals("open")) {
            anyOpen = true;
            break;
          }

          // Add the instance id to the list we will eventually terminate.
          instanceIds.add(describeResponse.getInstanceId());
        }
      } catch (AmazonServiceException e) {
        // If we have an exception, ensure we don't break out of the loop.
        // This prevents the scenario where there was blip on the wire.
        anyOpen = true;
      }

      try {
        // Sleep for 60 seconds.
        Thread.sleep(60 * 1000);
      } catch (Exception e) {
        // Do nothing because it woke up early.
      }
    } while (anyOpen);

    // ============================================================================================//
    // ====================================== Tag the Spot Instances
    // ===============================//
    // ============================================================================================//

    // Create the list of tags we want to create
    ArrayList<Tag> instanceTags = new ArrayList<Tag>();
    instanceTags.add(new Tag("keyname1", "value1"));

    // Create a tag request for instances.
    CreateTagsRequest createTagsRequest_instances = new CreateTagsRequest();
    createTagsRequest_instances.setResources(instanceIds);
    createTagsRequest_instances.setTags(instanceTags);

    // Try to tag the Spot instance started.
    try {
      ec2.createTags(createTagsRequest_instances);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // ====================================== Canceling the Request ==============================//
    // ============================================================================================//

    try {
      // Cancel requests.
      CancelSpotInstanceRequestsRequest cancelRequest =
          new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds);
      ec2.cancelSpotInstanceRequests(cancelRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error cancelling instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // =================================== Terminating any Instances
    // ==============================//
    // ============================================================================================//
    try {
      // Terminate instances.
      TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds);
      ec2.terminateInstances(terminateRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }
  }