public static String setUpSpotInstanceWithImage(String image) {
    System.out.println("Putting spot requet for image: " + image);

    RequestSpotInstancesRequest loadGeneratorRequest = new RequestSpotInstancesRequest();

    loadGeneratorRequest.setSpotPrice("0.03");
    loadGeneratorRequest.setInstanceCount(Integer.valueOf(1));

    LaunchSpecification loadGeneratorSpecification = new LaunchSpecification();
    loadGeneratorSpecification.setImageId(image);
    loadGeneratorSpecification.setInstanceType("m3.medium");

    ArrayList<String> securityGroups = new ArrayList<String>();
    securityGroups.add("SecurityGroupForLoadGenerator");
    loadGeneratorSpecification.setSecurityGroups(securityGroups);

    loadGeneratorRequest.setLaunchSpecification(loadGeneratorSpecification);

    RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(loadGeneratorRequest);
    List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();

    String requestId = requestResponses.get(0).getSpotInstanceRequestId();

    ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();
    spotInstanceRequestIds.add(requestId);

    System.out.println("Finding Instance ID..");

    boolean anyOpen;

    ArrayList<String> instanceIds = new ArrayList<String>();

    do {
      DescribeSpotInstanceRequestsRequest describeRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);

      anyOpen = false;

      try {
        DescribeSpotInstanceRequestsResult describeResult =
            ec2.describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();

        for (SpotInstanceRequest describeResponse : describeResponses) {
          if (describeResponse.getState().equals("open")) {
            anyOpen = true;
            break;
          }

          instanceIds.add(describeResponse.getInstanceId());
        }
      } catch (AmazonServiceException e) {
        anyOpen = true;
      }

      sleep(30);
    } while (anyOpen);

    return instanceIds.get(0);
  }
  /** @param args */
  public static void main(String[] args) {
    // ============================================================================================//
    // =============================== Submitting a Request
    // =======================================//
    // ============================================================================================//

    // Retrieves the credentials from an AWSCredentials.properties file.
    AWSCredentials credentials = null;
    try {
      credentials =
          new PropertiesCredentials(
              InlineTaggingCodeSampleApp.class.getResourceAsStream("AwsCredentials.properties"));
    } catch (IOException e1) {
      System.out.println("Credentials were not properly entered into AwsCredentials.properties.");
      System.out.println(e1.getMessage());
      System.exit(-1);
    }

    // Create the AmazonEC2Client object so we can call various APIs.
    AmazonEC2 ec2 = new AmazonEC2Client(credentials);

    // Initializes a Spot Instance Request
    RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest();

    // Request 1 x t1.micro instance with a bid price of $0.03.
    requestRequest.setSpotPrice("0.03");
    requestRequest.setInstanceCount(Integer.valueOf(1));

    // Setup the specifications of the launch. This includes the instance type (e.g. t1.micro)
    // and the latest Amazon Linux AMI id available. Note, you should always use the latest
    // Amazon Linux AMI id or another of your choosing.
    LaunchSpecification launchSpecification = new LaunchSpecification();
    launchSpecification.setImageId("ami-8c1fece5");
    launchSpecification.setInstanceType("t1.micro");

    // Add the security group to the request.
    ArrayList<String> securityGroups = new ArrayList<String>();
    securityGroups.add("GettingStartedGroup");
    launchSpecification.setSecurityGroups(securityGroups);

    // Add the launch specifications to the request.
    requestRequest.setLaunchSpecification(launchSpecification);

    // ============================================================================================//
    // =========================== Getting the Request ID from the Request
    // ========================//
    // ============================================================================================//

    // Call the RequestSpotInstance API.
    RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest);
    List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();

    // Setup an arraylist to collect all of the request ids we want to watch hit the running
    // state.
    ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();

    // Add all of the request ids to the hashset, so we can determine when they hit the
    // active state.
    for (SpotInstanceRequest requestResponse : requestResponses) {
      System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId());
      spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId());
    }

    // ============================================================================================//
    // ====================================== Tag the Spot Requests
    // ===============================//
    // ============================================================================================//

    // Create the list of tags we want to create
    ArrayList<Tag> requestTags = new ArrayList<Tag>();
    requestTags.add(new Tag("keyname1", "value1"));

    // Create a tag request for requests.
    CreateTagsRequest createTagsRequest_requests = new CreateTagsRequest();
    createTagsRequest_requests.setResources(spotInstanceRequestIds);
    createTagsRequest_requests.setTags(requestTags);

    // Try to tag the Spot request submitted.
    try {
      ec2.createTags(createTagsRequest_requests);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // =========================== Determining the State of the Spot Request
    // ======================//
    // ============================================================================================//

    // Create a variable that will track whether there are any requests still in the open state.
    boolean anyOpen;

    // Initialize variables.
    ArrayList<String> instanceIds = new ArrayList<String>();

    do {
      // Create the describeRequest with tall of the request id to monitor (e.g. that we started).
      DescribeSpotInstanceRequestsRequest describeRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);

      // Initialize the anyOpen variable to false – which assumes there are no requests open unless
      // we find one that is still open.
      anyOpen = false;

      try {
        // Retrieve all of the requests we want to monitor.
        DescribeSpotInstanceRequestsResult describeResult =
            ec2.describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();

        // Look through each request and determine if they are all in the active state.
        for (SpotInstanceRequest describeResponse : describeResponses) {
          // If the state is open, it hasn't changed since we attempted to request it.
          // There is the potential for it to transition almost immediately to closed or
          // cancelled so we compare against open instead of active.
          if (describeResponse.getState().equals("open")) {
            anyOpen = true;
            break;
          }

          // Add the instance id to the list we will eventually terminate.
          instanceIds.add(describeResponse.getInstanceId());
        }
      } catch (AmazonServiceException e) {
        // If we have an exception, ensure we don't break out of the loop.
        // This prevents the scenario where there was blip on the wire.
        anyOpen = true;
      }

      try {
        // Sleep for 60 seconds.
        Thread.sleep(60 * 1000);
      } catch (Exception e) {
        // Do nothing because it woke up early.
      }
    } while (anyOpen);

    // ============================================================================================//
    // ====================================== Tag the Spot Instances
    // ===============================//
    // ============================================================================================//

    // Create the list of tags we want to create
    ArrayList<Tag> instanceTags = new ArrayList<Tag>();
    instanceTags.add(new Tag("keyname1", "value1"));

    // Create a tag request for instances.
    CreateTagsRequest createTagsRequest_instances = new CreateTagsRequest();
    createTagsRequest_instances.setResources(instanceIds);
    createTagsRequest_instances.setTags(instanceTags);

    // Try to tag the Spot instance started.
    try {
      ec2.createTags(createTagsRequest_instances);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // ====================================== Canceling the Request ==============================//
    // ============================================================================================//

    try {
      // Cancel requests.
      CancelSpotInstanceRequestsRequest cancelRequest =
          new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds);
      ec2.cancelSpotInstanceRequests(cancelRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error cancelling instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // =================================== Terminating any Instances
    // ==============================//
    // ============================================================================================//
    try {
      // Terminate instances.
      TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds);
      ec2.terminateInstances(terminateRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }
  }
  /** @param args */
  public static void main(String[] args) {
    // ============================================================================================//
    // =============================== Submitting a Request
    // =======================================//
    // ============================================================================================//

    // Create the AmazonEC2Client object so we can call various APIs.
    AmazonEC2 ec2 = new AmazonEC2Client(new ClasspathPropertiesFileCredentialsProvider());
    Region usWest2 = Region.getRegion(Regions.US_WEST_2);
    ec2.setRegion(usWest2);

    // Initializes a Spot Instance Request
    RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest();

    // *************************** Required Parameters Settings ************************//
    // Request 1 x t1.micro instance with a bid price of $0.03.
    requestRequest.setSpotPrice("0.03");
    requestRequest.setInstanceCount(Integer.valueOf(1));

    // Setup the specifications of the launch. This includes the instance type (e.g. t1.micro)
    // and the latest Amazon Linux AMI id available. Note, you should always use the latest
    // Amazon Linux AMI id or another of your choosing.
    LaunchSpecification launchSpecification = new LaunchSpecification();
    launchSpecification.setImageId("ami-8c1fece5");
    launchSpecification.setInstanceType("t1.micro");

    // Add the security group to the request.
    ArrayList<String> securityGroups = new ArrayList<String>();
    securityGroups.add("GettingStartedGroup");
    launchSpecification.setSecurityGroups(securityGroups);

    // *************************** Bid Type Settings ************************//
    // Set the type of the bid to persistent.
    requestRequest.setType("persistent");

    // *************************** Valid From/To Settings ************************//
    // Set the valid start time to be two minutes from now.
    Calendar from = Calendar.getInstance();
    from.add(Calendar.MINUTE, 2);
    requestRequest.setValidFrom(from.getTime());

    // Set the valid end time to be two minutes and two hours from now.
    Calendar until = (Calendar) from.clone();
    until.add(Calendar.HOUR, 2);
    requestRequest.setValidUntil(until.getTime());

    // *************************** Launch Group Settings ************************//
    // Set the launch group.
    requestRequest.setLaunchGroup("ADVANCED-DEMO-LAUNCH-GROUP");

    // *************************** Availability Zone Group Settings ************************//
    // Set the availability zone group.
    requestRequest.setAvailabilityZoneGroup("ADVANCED-DEMO-AZ-GROUP");

    // *************************** Add the block device mapping ************************//

    // Goal: Setup block device mappings to ensure that we will not delete
    // the root partition on termination.

    // Create the block device mapping to describe the root partition.
    BlockDeviceMapping blockDeviceMapping = new BlockDeviceMapping();
    blockDeviceMapping.setDeviceName("/dev/sda1");

    // Set the delete on termination flag to false.
    EbsBlockDevice ebs = new EbsBlockDevice();
    ebs.setDeleteOnTermination(Boolean.FALSE);
    blockDeviceMapping.setEbs(ebs);

    // Add the block device mapping to the block list.
    ArrayList<BlockDeviceMapping> blockList = new ArrayList<BlockDeviceMapping>();
    blockList.add(blockDeviceMapping);

    // Set the block device mapping configuration in the launch specifications.
    launchSpecification.setBlockDeviceMappings(blockList);

    // *************************** Add the availability zone ************************//
    // Setup the availability zone to use. Note we could retrieve the availability
    // zones using the ec2.describeAvailabilityZones() API. For this demo we will just use
    // us-east-1b.
    SpotPlacement placement = new SpotPlacement("us-east-1b");
    launchSpecification.setPlacement(placement);

    // *************************** Add the placement group ************************//
    // Setup the placement group to use with whatever name you desire.
    // For this demo we will just use "ADVANCED-DEMO-PLACEMENT-GROUP".
    // Note: We have commented this out, because we are not leveraging cc1.4xlarge or
    // cg1.4xlarge in this example.
    /*
    SpotPlacement pg = new SpotPlacement();
    pg.setGroupName("ADVANCED-DEMO-PLACEMENT-GROUP");
    launchSpecification.setPlacement(pg);
    */

    // *************************** Add the launch specification ************************//
    // Add the launch specification.
    requestRequest.setLaunchSpecification(launchSpecification);

    // ============================================================================================//
    // =========================== Getting the Request ID from the Request
    // ========================//
    // ============================================================================================//

    // Call the RequestSpotInstance API.
    RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest);
    List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();

    // Setup an arraylist to collect all of the request ids we want to watch hit the running
    // state.
    ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();

    // Add all of the request ids to the hashset, so we can determine when they hit the
    // active state.
    for (SpotInstanceRequest requestResponse : requestResponses) {
      System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId());
      spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId());
    }

    // ============================================================================================//
    // =========================== Determining the State of the Spot Request
    // ======================//
    // ============================================================================================//

    // Create a variable that will track whether there are any requests still in the open state.
    boolean anyOpen;

    // Initialize variables.
    ArrayList<String> instanceIds = new ArrayList<String>();

    do {
      // Create the describeRequest with tall of the request id to monitor (e.g. that we started).
      DescribeSpotInstanceRequestsRequest describeRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);

      // Initialize the anyOpen variable to false ??? which assumes there are no requests open
      // unless
      // we find one that is still open.
      anyOpen = false;

      try {
        // Retrieve all of the requests we want to monitor.
        DescribeSpotInstanceRequestsResult describeResult =
            ec2.describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();

        // Look through each request and determine if they are all in the active state.
        for (SpotInstanceRequest describeResponse : describeResponses) {
          // If the state is open, it hasn't changed since we attempted to request it.
          // There is the potential for it to transition almost immediately to closed or
          // cancelled so we compare against open instead of active.
          if (describeResponse.getState().equals("open")) {
            anyOpen = true;
            break;
          }

          // Add the instance id to the list we will eventually terminate.
          instanceIds.add(describeResponse.getInstanceId());
        }
      } catch (AmazonServiceException e) {
        // If we have an exception, ensure we don't break out of the loop.
        // This prevents the scenario where there was blip on the wire.
        anyOpen = true;
      }

      try {
        // Sleep for 60 seconds.
        Thread.sleep(60 * 1000);
      } catch (Exception e) {
        // Do nothing because it woke up early.
      }
    } while (anyOpen);

    // ============================================================================================//
    // ====================================== Canceling the Request ==============================//
    // ============================================================================================//

    try {
      // Cancel requests.
      CancelSpotInstanceRequestsRequest cancelRequest =
          new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds);
      ec2.cancelSpotInstanceRequests(cancelRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error cancelling instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // =================================== Terminating any Instances
    // ==============================//
    // ============================================================================================//
    try {
      // Terminate instances.
      TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds);
      ec2.terminateInstances(terminateRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }
  }
Пример #4
0
  public void run() {
    logger.debug("started GridManager");
    System.out.println("started GridManager");
    AmazonSQSClient sqsClient = new AmazonSQSClient(papa.getCredentials());
    AmazonEC2Client ec2Client = new AmazonEC2Client(papa.getCredentials());

    // Get current imageID from the metadata table
    String imageID = Constants.imageID; // get the default imageID from Constants
    try {
      Connection conn = PooledConnectionFactory.INSTANCE.getCumulusConnection();
      Statement stmt = conn.createStatement();
      String imageQuery = "SELECT mvalue FROM cumulus.metadata WHERE mkey='imageID';";
      ResultSet results = stmt.executeQuery(imageQuery);
      results.next(); // move the cursor into the results
      imageID = results.getString(1);
    } catch (SQLException e) {
      logger.warn(e);
    }
    if (imageID == null
        || imageID.equals("")) { // if we get an empty imageID revert to the hardcoded one
      imageID = Constants.imageID;
    }
    logger.debug(imageID);
    System.out.println(imageID);

    while (true) { // loop from startup to shutdown
      // check the number of messages in the queue.
      GetQueueAttributesRequest sqsRequest =
          new GetQueueAttributesRequest(papa.getDispatchQueue())
              .withAttributeNames("ApproximateNumberOfMessages");
      Integer sqsResult = null;
      try {
        sqsResult =
            Integer.valueOf(
                sqsClient
                    .getQueueAttributes(sqsRequest)
                    .getAttributes()
                    .get("ApproximateNumberOfMessages"));
      } catch (AmazonServiceException e) {
        // Write out any exceptions that may have occurred.
        System.err.println("Error getting list of instances");
        System.err.println("Caught Exception: " + e.getMessage());
        System.err.println("Reponse Status Code: " + e.getStatusCode());
        System.err.println("Error Code: " + e.getErrorCode());
        System.err.println("Request ID: " + e.getRequestId());
      } catch (AmazonClientException e) {
        logger.warn(e);
        continue; // if the error is due to being unable to connect to sqs it is likely transient,
                  // and we should keep going until SQS comes back
      } catch (Exception e) {
        // caught another exception
        System.err.println(e);
        System.err.println("Did not exit cleanly");
        // System.exit(1);
      }
      if (sqsResult == null) {
        System.err.println(
            "Couldn't get the number of items in the queue reverting to static functioning");
        logger.error(
            "Couldn't get the number of items in the queue reverting to static functioning");
        break;
      }
      System.out.println("Queue size=" + sqsResult);
      // check the number of running instances.
      DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest();
      // get the number of running instances with our image ID
      describeInstancesRequest.withFilters(
          new Filter("image-id").withValues(imageID),
          new Filter("instance-state-name").withValues("running"));
      logger.debug("Getting list of active cumulus drones");
      Integer numInstances = null;
      Integer numSpotRequests = null;
      Initializer.serialize("/tmp/dump.obj", papa.unitsOnServer); // TODO test this.
      // check number of open spot requests
      DescribeSpotInstanceRequestsRequest describeSpotInstancesRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeSpotInstancesRequest.withFilters(new Filter("state").withValues("open"));
      try {
        // Parse returned instances
        DescribeInstancesResult describeInstancesResult =
            ec2Client.describeInstances(describeInstancesRequest);
        if (describeInstancesResult.getReservations().size() == 0) {
          numInstances = 0;
        } else {
          Integer totalInstances = 0;
          for (Reservation i : describeInstancesResult.getReservations()) {
            totalInstances += i.getInstances().size();
          }
          numInstances = totalInstances;
        }
        // Parse returned open spot requests
        DescribeSpotInstanceRequestsResult describeSpotReservationsResult =
            ec2Client.describeSpotInstanceRequests(describeSpotInstancesRequest);
        if (describeSpotReservationsResult.getSpotInstanceRequests().size() == 0) {
          numSpotRequests = 0;
        } else {
          Integer totalInstances = 0;
          for (SpotInstanceRequest i : describeSpotReservationsResult.getSpotInstanceRequests()) {
            totalInstances++;
          }
          numSpotRequests = totalInstances;
        }

        // System.out.println("num instances="+numInstances);
      } catch (AmazonServiceException e) {
        // Write out any exceptions that may have occurred.
        System.err.println("Error getting list of instances");
        System.err.println("Caught Exception: " + e.getMessage());
        System.err.println("Reponse Status Code: " + e.getStatusCode());
        System.err.println("Error Code: " + e.getErrorCode());
        System.err.println("Request ID: " + e.getRequestId());
        // System.exit(1);
      } catch (Exception e) {
        // caught another exception
        System.err.println(e);
        System.err.println("Did not exit cleanly");
        // System.exit(1);
      }
      if (numInstances == null || numSpotRequests == null) {
        logger.error(
            "Couldn't get the number of machines or requests reverting to static functioning");
        break;
      }
      System.out.println("num instances=" + (numInstances + numSpotRequests));

      // if queue is longer than the number of instances * idealMaxUnitsPerInstance
      Integer idealMaxUnitsPerInstance = Integer.valueOf(Constants.idealMaxUnitsPerInstance);
      if (sqsResult
          > (numInstances + numSpotRequests)
              * idealMaxUnitsPerInstance) { // we count pending spot instances too.
        Integer numToCreate = (sqsResult / idealMaxUnitsPerInstance) - numInstances + 1;
        // check current pricing of spot instances
        DescribeSpotPriceHistoryRequest historyRequest =
            new DescribeSpotPriceHistoryRequest()
                .withAvailabilityZone("us-east-1c")
                .withInstanceTypes(Constants.instanceType)
                .withProductDescriptions("Linux/UNIX (Amazon VPC)")
                .withStartTime(new Date(System.currentTimeMillis()));
        DescribeSpotPriceHistoryResult result = ec2Client.describeSpotPriceHistory(historyRequest);
        Double nowPrice = Double.valueOf(result.getSpotPriceHistory().get(0).getSpotPrice());
        // If spot price is < constants.spotPrice launch numToCreate*Constants.percentSpot spot
        // instances
        Integer spotToCreate = 0;
        if (nowPrice < Double.valueOf(Constants.spotPrice)) {
          Double percentSpot = Double.valueOf(Constants.percentSpot);
          spotToCreate = (int) (numToCreate * percentSpot);
          numToCreate = (int) (numToCreate * (1 - percentSpot));
        }
        try {
          System.out.println("regular instances: " + numToCreate);
          System.out.println("spot instances: " + spotToCreate);
          // Create regular and spot instances
          if (numToCreate > 0) {
            papa.createInstances(ec2Client, numToCreate, imageID);
          }
          if (spotToCreate > 0) {
            papa.createSpotInstances(ec2Client, spotToCreate, Constants.imageID); // FIXME
          }
          System.out.println("created instances");
        } catch (Exception e) {
          // Report ec2 exceptions
          logger.error(e);
          e.printStackTrace();
          continue;
        }
      }
      try {
        Thread.sleep(60000); // we wait for any new servers to start up
      } catch (InterruptedException e) {
        logger.error("Sqslistener was interrupted");
        if (papa.getShuttingDown()) {
          break;
        } else {
          continue;
        }
      }
    }
  }