@Override
    public void run() {
      // set up new data center
      RunInstancesRequest dataCenterRequest = createDataCenterRequest();
      RunInstancesResult runDataCenterResult = ec2.runInstances(dataCenterRequest);
      Instance dataCenterInstance = runDataCenterResult.getReservation().getInstances().get(0);
      String dataCenterId = dataCenterInstance.getInstanceId();
      createTag(ec2, dataCenterInstance);

      // wait for 1 min for the data center running
      try {
        Thread.sleep(60 * 1000);
      } catch (InterruptedException e1) {
        e1.printStackTrace();
      }

      // wait for data center ready for the web server
      String testSentence = null;
      while (testSentence == null) {
        System.out.println("RunInstance Thread: Waiting for data center ready");

        try {
          Thread.sleep(5000);
        } catch (InterruptedException e) {
          e.printStackTrace();
        }

        List<Instance> allInstances = listInstance(ec2);
        for (Instance inst : allInstances) {
          if (inst.getInstanceId().equals(dataCenterId)) {
            dataCenterInstance = inst;
            break;
          }
        }

        String randomUserUrl = "http://" + dataCenterInstance.getPublicDnsName() + "/lookup/random";
        try {
          testSentence = sendGet(randomUserUrl);
        } catch (Exception e) {
          // Do nothing
        }
      }

      String newDataCenterUrl = "http://" + dataCenterInstance.getPublicDnsName();

      // update the data center to in-service instances
      for (int i = 0; i < 3; i++) {
        if (instances[i].getUrl() == null) {
          instances[i].changeUrl(newDataCenterUrl);
          isHealthy[i] = true;
          updataHealthCount();
        }
      }
      // complete adding instance
      isAddingInstance = false;
    }
Example #2
0
 /**
  * Counts the number of instances in EC2 currently running that are using the specified image and
  * a template.
  *
  * @param ami If AMI is left null, then all instances are counted and template description is
  *     ignored.
  *     <p>This includes those instances that may be started outside Jenkins.
  * @param templateDesc
  */
 public int countCurrentEC2Slaves(String ami, String templateDesc) throws AmazonClientException {
   int n = 0;
   for (Reservation r : connect().describeInstances().getReservations()) {
     for (Instance i : r.getInstances()) {
       if (isEc2ProvisionedAmiSlave(i, ami, templateDesc)) {
         InstanceStateName stateName = InstanceStateName.fromValue(i.getState().getName());
         if (stateName == InstanceStateName.Pending || stateName == InstanceStateName.Running) {
           EC2AbstractSlave foundSlave = null;
           for (EC2AbstractSlave ec2Node : NodeIterator.nodes(EC2AbstractSlave.class)) {
             if (ec2Node.getInstanceId().equals(i.getInstanceId())) {
               foundSlave = ec2Node;
               break;
             }
           }
           // Don't count disconnected slaves as being used, we will connected them later is
           // required
           if (foundSlave != null && foundSlave.toComputer().isOffline()) continue;
           n++;
         }
       }
     }
   }
   // Count pending spot requests too
   for (SpotInstanceRequest sir :
       connect().describeSpotInstanceRequests().getSpotInstanceRequests()) {
     // Count Spot requests that are open and still have a
     // chance to be active.
     if (sir.getState().equals("open")) {
       n++;
     }
   }
   return n;
 }
Example #3
0
  // Creates new AMI instances from pre-defined snapshot,
  // and returns launched instance IDs.
  public static List<String> createAmiFromSnapshot(AmazonEC2 ec2, int amount, String userData) {
    logger.info("Launching " + amount + " instances.");

    ArrayList<String> securityGroups = new ArrayList<String>();
    securityGroups.add(securityGroup);

    RunInstancesRequest request = new RunInstancesRequest();

    // Send launch request.
    request
        .withImageId(imageId) // Utils.imageId
        .withInstanceType(instanceType)
        .withUserData(userData)
        .withKeyName(keyName)
        .withSecurityGroups(securityGroups)
        .withMinCount(amount)
        .withMaxCount(amount);

    // Build instance id list.
    ArrayList<String> ids = new ArrayList<String>();
    for (Instance instance : ec2.runInstances(request).getReservation().getInstances()) {
      ids.add(instance.getInstanceId());
    }

    return ids;
  }
Example #4
0
  public static void main(String[] args) throws UnknownHostException {
    AmazonEC2 ec2 = new AmazonEC2Client(new ClasspathPropertiesFileCredentialsProvider());
    Region usWest2 = Region.getRegion(Regions.US_WEST_2);
    ec2.setRegion(usWest2);
    List<String> instanceIds = new ArrayList<>();
    String instanceid = null;
    DescribeInstancesRequest request = new DescribeInstancesRequest();

    List<String> valuesT1 = new ArrayList<String>();
    valuesT1.add("hz-nodes");
    Filter filter = new Filter("tag-value", valuesT1);

    DescribeInstancesResult result = ec2.describeInstances(request.withFilters(filter));

    List<Reservation> reservations = result.getReservations();

    for (Reservation reservation : reservations) {
      List<Instance> instances = reservation.getInstances();
      for (Instance instance : instances) {
        instanceid = instance.getInstanceId();
        PrintManager.PrintMessage(instanceid);
        instanceIds.add(instanceid);
        List<Tag> abc = instance.getTags();
        for (Tag aaa : abc) {
          PrintManager.PrintMessage(aaa.getKey() + " : " + aaa.getValue());
        }
      }
    }
    instanceIds.remove("i-cb45adfc");
    StopInstancesRequest stopReq = new StopInstancesRequest(instanceIds);
    ec2.stopInstances(stopReq);
    RebootInstancesRequest requestRe = new RebootInstancesRequest(instanceIds);
    ec2.rebootInstances(requestRe);
  }
  @Override
  public void execute(Context context) throws Exception {
    RunInstancesRequest request =
        new RunInstancesRequest()
            .withKeyName(resource.keyPair.remoteKeyPair.getKeyName())
            .withInstanceType(resource.instanceType)
            .withImageId(resource.ami.imageId())
            .withMinCount(addedCount)
            .withMaxCount(addedCount)
            .withUserData(Base64.encodeBase64String(Strings.bytes(userData(context.env))));

    if (resource.instanceProfile != null)
      request.withIamInstanceProfile(
          new IamInstanceProfileSpecification()
              .withName(resource.instanceProfile.remoteInstanceProfile.getInstanceProfileName()));

    String sgId = resource.securityGroup.remoteSecurityGroup.getGroupId();

    request
        .getNetworkInterfaces()
        .add(
            new InstanceNetworkInterfaceSpecification()
                .withDeviceIndex(0)
                .withSubnetId(resource.subnet.firstRemoteSubnet().getSubnetId())
                .withGroups(sgId)
                .withAssociatePublicIpAddress(resource.subnet.type == SubnetType.PUBLIC));

    if (resource.ebs.rootVolumeSize != null) {
      request
          .getBlockDeviceMappings()
          .add(
              new BlockDeviceMapping()
                  .withDeviceName("/dev/sda1")
                  .withEbs(new EbsBlockDevice().withVolumeSize(resource.ebs.rootVolumeSize)));
    }

    List<com.amazonaws.services.ec2.model.Instance> remoteInstances =
        AWS.ec2.runInstances(request, tags(context.env));
    resource.remoteInstances.addAll(remoteInstances);

    for (com.amazonaws.services.ec2.model.Instance remoteInstance : remoteInstances) {
      String key = String.format("instance/%s/%s", resource.id, remoteInstance.getInstanceId());
      StringBuilder builder = new StringBuilder();
      builder.append("privateIP=").append(remoteInstance.getPrivateIpAddress());
      if (resource.subnet == null || resource.subnet.type == SubnetType.PUBLIC) {
        builder.append(", publicDNS=").append(remoteInstance.getPublicDnsName());
      }
      context.output(key, builder.toString());
    }

    if (resource.elb != null) {
      List<String> instanceIds =
          remoteInstances
              .stream()
              .map(com.amazonaws.services.ec2.model.Instance::getInstanceId)
              .collect(Collectors.toList());
      AWS.elb.attachInstances(
          resource.elb.remoteELB.getLoadBalancerName(), instanceIds, waitUntilInService);
    }
  }
 private void createTag(AmazonEC2Client ec2, Instance instance) {
   // Create Tags
   Tag tag = new Tag("Project", "2.3");
   CreateTagsRequest createTagsRequest = new CreateTagsRequest();
   createTagsRequest.withResources(instance.getInstanceId()).withTags(tag);
   ec2.createTags(createTagsRequest);
 }
Example #7
0
  /**
   * Determines whether the AMI of the given instance matches the AMI of template and has the
   * required label (if requiredLabel is non-null)
   */
  private boolean checkInstance(
      PrintStream logger,
      Instance existingInstance,
      Label requiredLabel,
      EC2AbstractSlave[] returnNode) {
    logProvision(logger, "checkInstance: " + existingInstance);
    if (StringUtils.isNotBlank(getIamInstanceProfile())) {
      if (existingInstance.getIamInstanceProfile() != null) {
        if (!existingInstance.getIamInstanceProfile().getArn().equals(getIamInstanceProfile())) {
          logProvision(logger, " false - IAM Instance profile does not match");
          return false;
        }
        // Match, fall through
      } else {
        logProvision(logger, " false - Null IAM Instance profile");
        return false;
      }
    }

    if (existingInstance
            .getState()
            .getName()
            .equalsIgnoreCase(InstanceStateName.Terminated.toString())
        || existingInstance
            .getState()
            .getName()
            .equalsIgnoreCase(InstanceStateName.ShuttingDown.toString())) {
      logProvision(logger, " false - Instance is terminated or shutting down");
      return false;
    }
    // See if we know about this and it has capacity
    for (EC2AbstractSlave node : NodeIterator.nodes(EC2AbstractSlave.class)) {
      if (node.getInstanceId().equals(existingInstance.getInstanceId())) {
        logProvision(logger, "Found existing corresponding Jenkins slave: " + node.getInstanceId());
        if (!node.toComputer().isPartiallyIdle()) {
          logProvision(logger, " false - Node is not partially idle");
          return false;
        }
        // REMOVEME - this was added to force provision to work, but might not allow
        // stopped instances to be found - need to investigate further
        else if (false && node.toComputer().isOffline()) {
          logProvision(logger, " false - Node is offline");
          return false;
        } else if (requiredLabel != null && !requiredLabel.matches(node.getAssignedLabels())) {
          logProvision(logger, " false - we need a Node having label " + requiredLabel);
          return false;
        } else {
          logProvision(logger, " true - Node has capacity - can use it");
          returnNode[0] = node;
          return true;
        }
      }
    }
    logProvision(logger, " true - Instance has no node, but can be used");
    return true;
  }
Example #8
0
  public String toString() {
    String ret = "\n";
    while (true) {
      if (amazon == null)
        try {
          amazon = JCS.getInstance("Instances");
        } catch (CacheException e) {

          e.printStackTrace();
          continue;
        }
      synchronized (AmazonService.class) {
        for (Object key : amazon.getGroupKeys("Instances")) {
          Instance i = (Instance) amazon.getFromGroup(key, "Instances");
          if (i == null) continue;
          ret +=
              (i.getInstanceId()
                      + " "
                      + i.getImageId()
                      + " "
                      + i.getImageId()
                      + " "
                      + i.getArchitecture()
                      + " "
                      + i.getPrivateDnsName()
                      + " "
                      + i.getState().getName())
                  + "\n";
        }
        for (Object key : amazon.getGroupKeys("AMIs")) {
          Image r = (Image) amazon.getFromGroup(key, "AMIs");
          ;
          if (r == null) continue;
          ret += (r.getImageId() + " " + r.getDescription() + " " + r.getState()) + "\n";
        }
      }
      if (ret.length() > 10) break;
      else {
        try {
          Thread.sleep(1000 * 5);
        } catch (InterruptedException e) {

          e.printStackTrace();
        }
        continue;
      }
    }

    return ret;
  }
  public static String getPublicDns(String instanceId) {

    System.out.println("Retreiving public dns.");

    DescribeInstancesResult results = ec2.describeInstances();
    List<Reservation> reservations = results.getReservations();

    for (Reservation reservation : reservations) {
      for (Instance instance : reservation.getInstances()) {
        if (instance.getInstanceId().equals(instanceId)) return instance.getPublicDnsName();
      }
    }
    return "NA";
  }
  // Gives fresh copy of instance
  public static Instance updatedInstance(Instance ins) {

    int i, j;
    List<Reservation> reserveList = ec2.describeInstances().getReservations();

    for (i = 0; i < reserveList.size(); i++) {
      List<Instance> instanceList = reserveList.get(i).getInstances();
      for (j = 0; j < instanceList.size(); j++) {
        if (instanceList.get(j).getInstanceId().equalsIgnoreCase(ins.getInstanceId())) {
          return instanceList.get(j);
        }
      }
    }
    return null;
  }
  /*
   * Sets tags on an instance. This will not clear existing tag data, so call clearLiveInstancedata if needed
   */
  protected void pushLiveInstancedata() throws AmazonClientException {
    Instance inst = getInstance(getInstanceId(), getCloud());

    /* Now that we have our instance, we can set tags on it */
    if (inst != null && tags != null && !tags.isEmpty()) {
      HashSet<Tag> inst_tags = new HashSet<Tag>();

      for (EC2Tag t : tags) {
        inst_tags.add(new Tag(t.getName(), t.getValue()));
      }

      CreateTagsRequest tag_request = new CreateTagsRequest();
      tag_request.withResources(inst.getInstanceId()).setTags(inst_tags);
      getCloud().connect().createTags(tag_request);
    }
  }
  // Allocates new elastic IP addresses.
  public static Instance allocateElasticIP(Instance ins) {
    // allocate
    AllocateAddressResult elasticResult = ec2.allocateAddress();
    String elasticIp = elasticResult.getPublicIp();
    System.out.println("New elastic IP: " + elasticIp);

    // associate
    AssociateAddressRequest aar = new AssociateAddressRequest();
    aar.setInstanceId(ins.getInstanceId());
    aar.setPublicIp(elasticIp);
    ec2.associateAddress(aar);

    ins = updatedInstance(ins);

    return ins;
  }
Example #13
0
 private Instance locateInstanceToSSH(List<Instance> instances) {
   for (int i = 0; i < instances.size(); i++) {
     com.amazonaws.services.ec2.model.Instance remoteInstance = instances.get(i);
     logger.info(
         "index={}, instanceId={}, state={}, publicDNS={}, privateDNS={}",
         i,
         remoteInstance.getInstanceId(),
         remoteInstance.getState().getName(),
         remoteInstance.getPublicDnsName(),
         remoteInstance.getPrivateDnsName());
   }
   Asserts.isTrue(
       instances.size() == 1 || instanceIndex != null,
       "more than one remoteInstance, use --{} to specify index",
       Param.INSTANCE_INDEX.key);
   return instances.size() == 1 ? instances.get(0) : instances.get(instanceIndex);
 }
 /** Test of getMaster method, of class HadoopCluster. */
 @Test
 public void testGetMaster() {
   System.out.println("getMaster");
   ClusterInstance result = cluster.getMaster();
   assertNotNull(result);
   assertEquals(cluster.getMasterGroupName(), result.getSecurityGroups().get(0));
   Instance slaveInstance = result.getInstance();
   StringBuilder sb = new StringBuilder("\t");
   sb.append(slaveInstance.getInstanceId());
   sb.append(" ");
   sb.append(slaveInstance.getInstanceType());
   sb.append(" ");
   sb.append(slaveInstance.getLaunchTime().toString());
   sb.append(" ");
   sb.append(slaveInstance.getImageId());
   out.println(sb.toString());
 }
Example #15
0
  private static void putCache() throws CacheException {
    if (amazon == null) amazon = JCS.getInstance("Instances");
    synchronized (AmazonService.class) {
      amazon.clear();
      DescribeInstancesResult result = amazonClient.describeInstances();
      for (Reservation r : result.getReservations()) {
        for (Instance i : r.getInstances()) {
          amazon.putInGroup(i.getInstanceId(), "Instances", i);
        }
      }

      DescribeImagesRequest ir = new DescribeImagesRequest().withOwners("self");
      DescribeImagesResult images = amazonClient.describeImages(ir);
      for (Image r : images.getImages()) {
        amazon.putInGroup(r.getImageId(), "AMIs", r);
      }
      log.debug("put cache finished");
    }
  }
 /** Test of getSlaves method, of class HadoopCluster. */
 @Test
 public void testGetSlaves() {
   out.println("getSlaves");
   List<ClusterInstance> result = cluster.getSlaves();
   out.println("found " + result.size() + " slaves for group " + TEST_GROUP);
   assertNotNull(result);
   StringBuilder sb = new StringBuilder("\t");
   for (ClusterInstance slave : result) {
     assertEquals(cluster.getGroupName(), slave.getSecurityGroups().get(0));
     Instance slaveInstance = slave.getInstance();
     if (sb.length() > 1) sb.delete(1, sb.length());
     sb.append(slaveInstance.getInstanceId());
     sb.append(" ");
     sb.append(slaveInstance.getInstanceType());
     sb.append(" ");
     sb.append(slaveInstance.getLaunchTime().toString());
     sb.append(" ");
     sb.append(slaveInstance.getImageId());
     out.println(sb.toString());
   }
 }
Example #17
0
 protected EC2OndemandSlave newOndemandSlave(Instance inst) throws FormException, IOException {
   return new EC2OndemandSlave(
       inst.getInstanceId(),
       description,
       remoteFS,
       getNumExecutors(),
       labels,
       mode,
       initScript,
       tmpDir,
       remoteAdmin,
       jvmopts,
       stopOnTerminate,
       idleTerminationMinutes,
       inst.getPublicDnsName(),
       inst.getPrivateDnsName(),
       EC2Tag.fromAmazonTags(inst.getTags()),
       parent.name,
       usePrivateDnsName,
       useDedicatedTenancy,
       getLaunchTimeout(),
       amiType);
 }
Example #18
0
  // Returns pending/running instance ID list.
  public static ArrayList<String> getInstanceIdsByTag(AmazonEC2 ec2, String key, String value) {
    logger.fine("Requesting instances by tag " + key + "=" + value);

    // Create tag request.
    DescribeInstancesRequest instanceReq = new DescribeInstancesRequest();
    Filter managerFilter = new Filter("tag:" + key).withValues(value),
        activeFilter =
            new Filter("instance-state-code").withValues("0", "16"); // 0||16 == pending||running

    // Send request.
    DescribeInstancesResult result =
        ec2.describeInstances(instanceReq.withFilters(managerFilter, activeFilter));

    // Build instance ID list.
    ArrayList<String> ids = new ArrayList<String>();
    for (Reservation reservation : result.getReservations()) {
      for (Instance instance : reservation.getInstances()) {
        ids.add(instance.getInstanceId());
      }
    }

    logger.fine(ids.size() + " instances found with tag " + key + "=" + value);
    return ids;
  }
  public static void start() {
    AWSCredentials credentials = null;
    try {
      credentials =
          new PropertiesCredentials(
              CreateInitialVMs.class.getResourceAsStream("AwsCredentials.properties"));
    } catch (IOException e1) {
      // TODO Auto-generated catch block
      e1.printStackTrace();
    }

    /**
     * *******************************************
     *
     * <p>#1 Create Amazon Client object
     *
     * <p>*******************************************
     */
    System.out.println("#1 Create Amazon Client object");
    ec2 = new AmazonEC2Client(credentials);

    try {

      /**
       * *******************************************
       *
       * <p>#6 Create an Instance
       *
       * <p>*******************************************
       */
      System.out.println("#5 Create an Instance");

      String imageId = "ami-76f0061f"; // Basic 32-bit Amazon Linux AMI
      int minInstanceCount = 1;
      int maxInstanceCount = 2; // create 2 instances
      String keyPairName = "testElastic";
      String securityGroupName = "Prachi";
      ArrayList<String> securityGroup = new ArrayList<String>();
      securityGroup.add(securityGroupName);

      RunInstancesRequest rir =
          new RunInstancesRequest(imageId, minInstanceCount, maxInstanceCount);
      rir.setKeyName(keyPairName);
      rir.setSecurityGroups(securityGroup);
      Placement p = new Placement();
      p.setAvailabilityZone("us-east-1a");
      rir.setPlacement(p);

      RunInstancesResult result = ec2.runInstances(rir);

      // get instanceId from the result
      List<Instance> resultInstance = result.getReservation().getInstances();
      String createdInstanceId = null;
      for (Instance ins : resultInstance) {
        createdInstanceId = ins.getInstanceId();

        System.out.println("State of instance " + ins.getInstanceId() + " : ");
        while (true) {
          try {
            Thread.sleep(1000);
          } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
          }
          System.out.print(ins.getState().getName() + "\n");

          ins = updatedInstance(ins);
          // if(ins.getPublicIpAddress()!= null){
          // if(ins.getState().getName().equals("running")){
          if (ins.getPublicIpAddress() != null && ins.getState().getName().equals("running")) {
            break;
          }
        }

        System.out.println("State: " + ins.getState().getName());

        System.out.println("New instance has been created: " + ins.getInstanceId());
        System.out.println("Instance Key: " + ins.getKeyName());
        System.out.println("Public DNS Name: " + ins.getPublicDnsName());
        System.out.println("Public DNS IP Address: " + ins.getPublicIpAddress());
        System.out.println("Instance ID: " + ins.getInstanceId());

        VMNode vmn = new VMNode();

        vmn.setInstanceId(ins.getInstanceId());

        /**
         * ******************************************* Allocate elastic IP addresses.
         * *******************************************
         */
        System.out.println("Public IP before association: " + ins.getPublicIpAddress());

        ins = allocateElasticIP(ins);

        vmn.setElasticIp(ins.getPublicIpAddress());

        System.out.println("Public IP after association: " + ins.getPublicIpAddress());

        /**
         * ******************************************* Create a volume
         * *******************************************
         */
        String createdVolumeId = createVolume();

        /**
         * ******************************************* Attach the volume to the instance
         * *******************************************
         */
        attachVolume(createdInstanceId, createdVolumeId);

        vmn.setVolumeId(createdVolumeId);

        /**
         * ******************************************* Detach the volume from the instance
         * *******************************************
         */
        detachVolume(createdInstanceId, createdVolumeId);

        /**
         * ********************************* Create an AMI from an instance
         * *******************************
         */
        String createdImageId = createAmi(createdInstanceId);

        vmn.setAmiId(createdImageId);
        nodes.add(vmn);

        /** ********************************* Stop the instance ******************************* */
        // stopInstance(ins.getInstanceId());
        // Diassociate Elastic IP from instance
        disassociateElasticIp(ins.getPublicIpAddress());
      }

      System.out.println("Listing Nodes: ");
      for (VMNode n : nodes) {
        System.out.println("Instance ID: " + n.getInstanceid());
        System.out.println("Elastic IP: " + n.getElasticIp());
        System.out.println("Volume ID: " + n.getVolumeId());
        System.out.println("AMI ID: " + n.getAmiId());
        System.out.println();
      }

      /**
       * *******************************************
       *
       * <p>#10 shutdown client object
       *
       * <p>*******************************************
       */
      ec2.shutdown();

    } catch (AmazonServiceException ase) {
      System.out.println("Caught Exception: " + ase.getMessage());
      System.out.println("Reponse Status Code: " + ase.getStatusCode());
      System.out.println("Error Code: " + ase.getErrorCode());
      System.out.println("Request ID: " + ase.getRequestId());
    }
  }
  @Override
  public List<DiscoveryNode> buildDynamicNodes() {
    List<DiscoveryNode> discoNodes = new ArrayList<>();

    DescribeInstancesResult descInstances;
    try {
      // Query EC2 API based on AZ, instance state, and tag.

      // NOTE: we don't filter by security group during the describe instances request for two
      // reasons:
      // 1. differences in VPCs require different parameters during query (ID vs Name)
      // 2. We want to use two different strategies: (all security groups vs. any security groups)
      descInstances = client.describeInstances(buildDescribeInstancesRequest());
    } catch (AmazonClientException e) {
      logger.info("Exception while retrieving instance list from AWS API: {}", e.getMessage());
      logger.debug("Full exception:", e);
      return discoNodes;
    }

    logger.trace("building dynamic unicast discovery nodes...");
    for (Reservation reservation : descInstances.getReservations()) {
      for (Instance instance : reservation.getInstances()) {
        // lets see if we can filter based on groups
        if (!groups.isEmpty()) {
          List<GroupIdentifier> instanceSecurityGroups = instance.getSecurityGroups();
          ArrayList<String> securityGroupNames = new ArrayList<String>();
          ArrayList<String> securityGroupIds = new ArrayList<String>();
          for (GroupIdentifier sg : instanceSecurityGroups) {
            securityGroupNames.add(sg.getGroupName());
            securityGroupIds.add(sg.getGroupId());
          }
          if (bindAnyGroup) {
            // We check if we can find at least one group name or one group id in groups.
            if (Collections.disjoint(securityGroupNames, groups)
                && Collections.disjoint(securityGroupIds, groups)) {
              logger.trace(
                  "filtering out instance {} based on groups {}, not part of {}",
                  instance.getInstanceId(),
                  instanceSecurityGroups,
                  groups);
              // continue to the next instance
              continue;
            }
          } else {
            // We need tp match all group names or group ids, otherwise we ignore this instance
            if (!(securityGroupNames.containsAll(groups) || securityGroupIds.containsAll(groups))) {
              logger.trace(
                  "filtering out instance {} based on groups {}, does not include all of {}",
                  instance.getInstanceId(),
                  instanceSecurityGroups,
                  groups);
              // continue to the next instance
              continue;
            }
          }
        }

        String address = null;
        switch (hostType) {
          case PRIVATE_DNS:
            address = instance.getPrivateDnsName();
            break;
          case PRIVATE_IP:
            address = instance.getPrivateIpAddress();
            break;
          case PUBLIC_DNS:
            address = instance.getPublicDnsName();
            break;
          case PUBLIC_IP:
            address = instance.getPublicIpAddress();
            break;
        }
        if (address != null) {
          try {
            TransportAddress[] addresses = transportService.addressesFromString(address);
            // we only limit to 1 addresses, makes no sense to ping 100 ports
            for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) {
              logger.trace(
                  "adding {}, address {}, transport_address {}",
                  instance.getInstanceId(),
                  address,
                  addresses[i]);
              discoNodes.add(
                  new DiscoveryNode(
                      "#cloud-" + instance.getInstanceId() + "-" + i,
                      addresses[i],
                      version.minimumCompatibilityVersion()));
            }
          } catch (Exception e) {
            logger.warn("failed ot add {}, address {}", e, instance.getInstanceId(), address);
          }
        } else {
          logger.trace(
              "not adding {}, address is null, host_type {}", instance.getInstanceId(), hostType);
        }
      }
    }

    logger.debug("using dynamic discovery nodes {}", discoNodes);

    return discoNodes;
  }
Example #21
0
  /**
   * Provisions an On-demand EC2 slave by launching a new instance or starting a previously-stopped
   * instance.
   */
  private EC2AbstractSlave provisionOndemand(
      TaskListener listener, Label requiredLabel, EnumSet<ProvisionOptions> provisionOptions)
      throws AmazonClientException, IOException {
    PrintStream logger = listener.getLogger();
    AmazonEC2 ec2 = getParent().connect();

    try {
      logProvisionInfo(logger, "Considering launching " + ami + " for template " + description);

      KeyPair keyPair = getKeyPair(ec2);

      RunInstancesRequest riRequest = new RunInstancesRequest(ami, 1, 1);
      InstanceNetworkInterfaceSpecification net = new InstanceNetworkInterfaceSpecification();

      riRequest.setEbsOptimized(ebsOptimized);

      if (useEphemeralDevices) {
        setupEphemeralDeviceMapping(riRequest);
      } else {
        setupCustomDeviceMapping(riRequest);
      }

      if (stopOnTerminate) {
        riRequest.setInstanceInitiatedShutdownBehavior(ShutdownBehavior.Stop);
        logProvisionInfo(
            logger, "Setting Instance Initiated Shutdown Behavior : ShutdownBehavior.Stop");
      } else {
        riRequest.setInstanceInitiatedShutdownBehavior(ShutdownBehavior.Terminate);
        logProvisionInfo(
            logger, "Setting Instance Initiated Shutdown Behavior : ShutdownBehavior.Terminate");
      }

      List<Filter> diFilters = new ArrayList<Filter>();
      diFilters.add(new Filter("image-id").withValues(ami));

      if (StringUtils.isNotBlank(getZone())) {
        Placement placement = new Placement(getZone());
        if (getUseDedicatedTenancy()) {
          placement.setTenancy("dedicated");
        }
        riRequest.setPlacement(placement);
        diFilters.add(new Filter("availability-zone").withValues(getZone()));
      }

      if (StringUtils.isNotBlank(getSubnetId())) {
        if (getAssociatePublicIp()) {
          net.setSubnetId(getSubnetId());
        } else {
          riRequest.setSubnetId(getSubnetId());
        }

        diFilters.add(new Filter("subnet-id").withValues(getSubnetId()));

        /*
         * If we have a subnet ID then we can only use VPC security groups
         */
        if (!securityGroupSet.isEmpty()) {
          List<String> groupIds = getEc2SecurityGroups(ec2);

          if (!groupIds.isEmpty()) {
            if (getAssociatePublicIp()) {
              net.setGroups(groupIds);
            } else {
              riRequest.setSecurityGroupIds(groupIds);
            }

            diFilters.add(new Filter("instance.group-id").withValues(groupIds));
          }
        }
      } else {
        /* No subnet: we can use standard security groups by name */
        riRequest.setSecurityGroups(securityGroupSet);
        if (!securityGroupSet.isEmpty()) {
          diFilters.add(new Filter("instance.group-name").withValues(securityGroupSet));
        }
      }

      String userDataString = Base64.encodeBase64String(userData.getBytes(StandardCharsets.UTF_8));
      riRequest.setUserData(userDataString);
      riRequest.setKeyName(keyPair.getKeyName());
      diFilters.add(new Filter("key-name").withValues(keyPair.getKeyName()));
      riRequest.setInstanceType(type.toString());
      diFilters.add(new Filter("instance-type").withValues(type.toString()));

      if (getAssociatePublicIp()) {
        net.setAssociatePublicIpAddress(true);
        net.setDeviceIndex(0);
        riRequest.withNetworkInterfaces(net);
      }

      boolean hasCustomTypeTag = false;
      HashSet<Tag> instTags = null;
      if (tags != null && !tags.isEmpty()) {
        instTags = new HashSet<Tag>();
        for (EC2Tag t : tags) {
          instTags.add(new Tag(t.getName(), t.getValue()));
          diFilters.add(new Filter("tag:" + t.getName()).withValues(t.getValue()));
          if (StringUtils.equals(t.getName(), EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE)) {
            hasCustomTypeTag = true;
          }
        }
      }
      if (!hasCustomTypeTag) {
        if (instTags == null) {
          instTags = new HashSet<Tag>();
        }
        // Append template description as well to identify slaves provisioned per template
        instTags.add(
            new Tag(
                EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE,
                EC2Cloud.getSlaveTypeTagValue(EC2Cloud.EC2_SLAVE_TYPE_DEMAND, description)));
      }

      DescribeInstancesRequest diRequest = new DescribeInstancesRequest();
      diRequest.setFilters(diFilters);

      logProvision(logger, "Looking for existing instances with describe-instance: " + diRequest);

      DescribeInstancesResult diResult = ec2.describeInstances(diRequest);
      EC2AbstractSlave[] ec2Node = new EC2AbstractSlave[1];
      Instance existingInstance = null;
      if (!provisionOptions.contains(ProvisionOptions.FORCE_CREATE)) {
        reservationLoop:
        for (Reservation reservation : diResult.getReservations()) {
          for (Instance instance : reservation.getInstances()) {
            if (checkInstance(logger, instance, requiredLabel, ec2Node)) {
              existingInstance = instance;
              logProvision(
                  logger,
                  "Found existing instance: "
                      + existingInstance
                      + ((ec2Node[0] != null) ? (" node: " + ec2Node[0].getInstanceId()) : ""));
              break reservationLoop;
            }
          }
        }
      }

      if (existingInstance == null) {
        if (!provisionOptions.contains(ProvisionOptions.FORCE_CREATE)
            && !provisionOptions.contains(ProvisionOptions.ALLOW_CREATE)) {
          logProvision(logger, "No existing instance found - but cannot create new instance");
          return null;
        }
        if (StringUtils.isNotBlank(getIamInstanceProfile())) {
          riRequest.setIamInstanceProfile(
              new IamInstanceProfileSpecification().withArn(getIamInstanceProfile()));
        }
        // Have to create a new instance
        Instance inst = ec2.runInstances(riRequest).getReservation().getInstances().get(0);

        /* Now that we have our instance, we can set tags on it */
        if (instTags != null) {
          updateRemoteTags(ec2, instTags, "InvalidInstanceID.NotFound", inst.getInstanceId());

          // That was a remote request - we should also update our
          // local instance data.
          inst.setTags(instTags);
        }
        logProvisionInfo(logger, "No existing instance found - created new instance: " + inst);
        return newOndemandSlave(inst);
      }

      if (existingInstance
              .getState()
              .getName()
              .equalsIgnoreCase(InstanceStateName.Stopping.toString())
          || existingInstance
              .getState()
              .getName()
              .equalsIgnoreCase(InstanceStateName.Stopped.toString())) {

        List<String> instances = new ArrayList<String>();
        instances.add(existingInstance.getInstanceId());
        StartInstancesRequest siRequest = new StartInstancesRequest(instances);
        StartInstancesResult siResult = ec2.startInstances(siRequest);

        logProvisionInfo(
            logger,
            "Found stopped instance - starting it: " + existingInstance + " result:" + siResult);
      } else {
        // Should be pending or running at this point, just let it come up
        logProvisionInfo(
            logger,
            "Found existing pending or running: "
                + existingInstance.getState().getName()
                + " instance: "
                + existingInstance);
      }

      if (ec2Node[0] != null) {
        logProvisionInfo(logger, "Using existing slave: " + ec2Node[0].getInstanceId());
        return ec2Node[0];
      }

      // Existing slave not found
      logProvision(logger, "Creating new slave for existing instance: " + existingInstance);
      return newOndemandSlave(existingInstance);

    } catch (FormException e) {
      throw new AssertionError(e); // we should have discovered all
      // configuration issues upfront
    } catch (InterruptedException e) {
      throw new RuntimeException(e);
    }
  }