Exemplo n.º 1
0
  /** Get a list of security group ids for the slave */
  private List<String> getEc2SecurityGroups(AmazonEC2 ec2) throws AmazonClientException {
    List<String> groupIds = new ArrayList<String>();

    DescribeSecurityGroupsResult groupResult =
        getSecurityGroupsBy("group-name", securityGroupSet, ec2);
    if (groupResult.getSecurityGroups().size() == 0) {
      groupResult = getSecurityGroupsBy("group-id", securityGroupSet, ec2);
    }

    for (SecurityGroup group : groupResult.getSecurityGroups()) {
      if (group.getVpcId() != null && !group.getVpcId().isEmpty()) {
        List<Filter> filters = new ArrayList<Filter>();
        filters.add(new Filter("vpc-id").withValues(group.getVpcId()));
        filters.add(new Filter("state").withValues("available"));
        filters.add(new Filter("subnet-id").withValues(getSubnetId()));

        DescribeSubnetsRequest subnetReq = new DescribeSubnetsRequest();
        subnetReq.withFilters(filters);
        DescribeSubnetsResult subnetResult = ec2.describeSubnets(subnetReq);

        List<Subnet> subnets = subnetResult.getSubnets();
        if (subnets != null && !subnets.isEmpty()) {
          groupIds.add(group.getGroupId());
        }
      }
    }

    if (securityGroupSet.size() != groupIds.size()) {
      throw new AmazonClientException(
          "Security groups must all be VPC security groups to work in a VPC context");
    }

    return groupIds;
  }
Exemplo n.º 2
0
  private List<BlockDeviceMapping> getNewEphemeralDeviceMapping() {

    final List<BlockDeviceMapping> oldDeviceMapping = getAmiBlockDeviceMappings();

    final Set<String> occupiedDevices = new HashSet<String>();
    for (final BlockDeviceMapping mapping : oldDeviceMapping) {

      occupiedDevices.add(mapping.getDeviceName());
    }

    final List<String> available =
        new ArrayList<String>(
            Arrays.asList("ephemeral0", "ephemeral1", "ephemeral2", "ephemeral3"));

    final List<BlockDeviceMapping> newDeviceMapping = new ArrayList<BlockDeviceMapping>(4);
    for (char suffix = 'b'; suffix <= 'z' && !available.isEmpty(); suffix++) {

      final String deviceName = String.format("/dev/xvd%s", suffix);

      if (occupiedDevices.contains(deviceName)) continue;

      final BlockDeviceMapping newMapping =
          new BlockDeviceMapping().withDeviceName(deviceName).withVirtualName(available.get(0));

      newDeviceMapping.add(newMapping);
      available.remove(0);
    }

    return newDeviceMapping;
  }
Exemplo n.º 3
0
 /** * Check that the AMI requested is available in the cloud and can be used. */
 public FormValidation doValidateAmi(
     @QueryParameter boolean useInstanceProfileForCredentials,
     @QueryParameter String credentialsId,
     @QueryParameter String ec2endpoint,
     @QueryParameter String region,
     final @QueryParameter String ami)
     throws IOException {
   AWSCredentialsProvider credentialsProvider =
       EC2Cloud.createCredentialsProvider(useInstanceProfileForCredentials, credentialsId);
   AmazonEC2 ec2;
   if (region != null) {
     ec2 = EC2Cloud.connect(credentialsProvider, AmazonEC2Cloud.getEc2EndpointUrl(region));
   } else {
     ec2 = EC2Cloud.connect(credentialsProvider, new URL(ec2endpoint));
   }
   if (ec2 != null) {
     try {
       List<String> images = new LinkedList<String>();
       images.add(ami);
       List<String> owners = new LinkedList<String>();
       List<String> users = new LinkedList<String>();
       DescribeImagesRequest request = new DescribeImagesRequest();
       request.setImageIds(images);
       request.setOwners(owners);
       request.setExecutableUsers(users);
       List<Image> img = ec2.describeImages(request).getImages();
       if (img == null || img.isEmpty()) {
         // de-registered AMI causes an empty list to be
         // returned. so be defensive
         // against other possibilities
         return FormValidation.error("No such AMI, or not usable with this accessId: " + ami);
       }
       String ownerAlias = img.get(0).getImageOwnerAlias();
       return FormValidation.ok(
           img.get(0).getImageLocation() + (ownerAlias != null ? " by " + ownerAlias : ""));
     } catch (AmazonClientException e) {
       return FormValidation.error(e.getMessage());
     }
   } else return FormValidation.ok(); // can't test
 }
Exemplo n.º 4
0
  /** Provision a new slave for an EC2 spot instance to call back to Jenkins */
  private EC2AbstractSlave provisionSpot(TaskListener listener)
      throws AmazonClientException, IOException {
    PrintStream logger = listener.getLogger();
    AmazonEC2 ec2 = getParent().connect();

    try {
      logger.println("Launching " + ami + " for template " + description);
      LOGGER.info("Launching " + ami + " for template " + description);

      KeyPair keyPair = getKeyPair(ec2);

      RequestSpotInstancesRequest spotRequest = new RequestSpotInstancesRequest();

      // Validate spot bid before making the request
      if (getSpotMaxBidPrice() == null) {
        // throw new FormException("Invalid Spot price specified: " +
        // getSpotMaxBidPrice(), "spotMaxBidPrice");
        throw new AmazonClientException("Invalid Spot price specified: " + getSpotMaxBidPrice());
      }

      spotRequest.setSpotPrice(getSpotMaxBidPrice());
      spotRequest.setInstanceCount(1);

      LaunchSpecification launchSpecification = new LaunchSpecification();
      InstanceNetworkInterfaceSpecification net = new InstanceNetworkInterfaceSpecification();

      launchSpecification.setImageId(ami);
      launchSpecification.setInstanceType(type);

      if (StringUtils.isNotBlank(getZone())) {
        SpotPlacement placement = new SpotPlacement(getZone());
        launchSpecification.setPlacement(placement);
      }

      if (StringUtils.isNotBlank(getSubnetId())) {
        if (getAssociatePublicIp()) {
          net.setSubnetId(getSubnetId());
        } else {
          launchSpecification.setSubnetId(getSubnetId());
        }

        /*
         * If we have a subnet ID then we can only use VPC security groups
         */
        if (!securityGroupSet.isEmpty()) {
          List<String> groupIds = getEc2SecurityGroups(ec2);
          if (!groupIds.isEmpty()) {
            if (getAssociatePublicIp()) {
              net.setGroups(groupIds);
            } else {
              ArrayList<GroupIdentifier> groups = new ArrayList<GroupIdentifier>();

              for (String group_id : groupIds) {
                GroupIdentifier group = new GroupIdentifier();
                group.setGroupId(group_id);
                groups.add(group);
              }
              if (!groups.isEmpty()) launchSpecification.setAllSecurityGroups(groups);
            }
          }
        }
      } else {
        /* No subnet: we can use standard security groups by name */
        if (!securityGroupSet.isEmpty()) {
          launchSpecification.setSecurityGroups(securityGroupSet);
        }
      }

      String userDataString = Base64.encodeBase64String(userData.getBytes(StandardCharsets.UTF_8));

      launchSpecification.setUserData(userDataString);
      launchSpecification.setKeyName(keyPair.getKeyName());
      launchSpecification.setInstanceType(type.toString());

      if (getAssociatePublicIp()) {
        net.setAssociatePublicIpAddress(true);
        net.setDeviceIndex(0);
        launchSpecification.withNetworkInterfaces(net);
      }

      boolean hasCustomTypeTag = false;
      HashSet<Tag> instTags = null;
      if (tags != null && !tags.isEmpty()) {
        instTags = new HashSet<Tag>();
        for (EC2Tag t : tags) {
          instTags.add(new Tag(t.getName(), t.getValue()));
          if (StringUtils.equals(t.getName(), EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE)) {
            hasCustomTypeTag = true;
          }
        }
      }
      if (!hasCustomTypeTag) {
        if (instTags != null)
          instTags.add(
              new Tag(
                  EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE,
                  EC2Cloud.getSlaveTypeTagValue(EC2Cloud.EC2_SLAVE_TYPE_SPOT, description)));
      }

      if (StringUtils.isNotBlank(getIamInstanceProfile())) {
        launchSpecification.setIamInstanceProfile(
            new IamInstanceProfileSpecification().withArn(getIamInstanceProfile()));
      }

      if (useEphemeralDevices) {
        setupEphemeralDeviceMapping(launchSpecification);
      } else {
        setupCustomDeviceMapping(launchSpecification);
      }

      spotRequest.setLaunchSpecification(launchSpecification);

      // Make the request for a new Spot instance
      RequestSpotInstancesResult reqResult = ec2.requestSpotInstances(spotRequest);

      List<SpotInstanceRequest> reqInstances = reqResult.getSpotInstanceRequests();
      if (reqInstances.isEmpty()) {
        throw new AmazonClientException("No spot instances found");
      }

      SpotInstanceRequest spotInstReq = reqInstances.get(0);
      if (spotInstReq == null) {
        throw new AmazonClientException("Spot instance request is null");
      }
      String slaveName = spotInstReq.getSpotInstanceRequestId();

      /* Now that we have our Spot request, we can set tags on it */
      if (instTags != null) {
        updateRemoteTags(
            ec2,
            instTags,
            "InvalidSpotInstanceRequestID.NotFound",
            spotInstReq.getSpotInstanceRequestId());

        // That was a remote request - we should also update our local
        // instance data.
        spotInstReq.setTags(instTags);
      }

      logger.println("Spot instance id in provision: " + spotInstReq.getSpotInstanceRequestId());
      LOGGER.info("Spot instance id in provision: " + spotInstReq.getSpotInstanceRequestId());

      return newSpotSlave(spotInstReq, slaveName);

    } catch (FormException e) {
      throw new AssertionError(); // we should have discovered all
      // configuration issues upfront
    } catch (InterruptedException e) {
      throw new RuntimeException(e);
    }
  }
Exemplo n.º 5
0
  /**
   * Provisions an On-demand EC2 slave by launching a new instance or starting a previously-stopped
   * instance.
   */
  private EC2AbstractSlave provisionOndemand(
      TaskListener listener, Label requiredLabel, EnumSet<ProvisionOptions> provisionOptions)
      throws AmazonClientException, IOException {
    PrintStream logger = listener.getLogger();
    AmazonEC2 ec2 = getParent().connect();

    try {
      logProvisionInfo(logger, "Considering launching " + ami + " for template " + description);

      KeyPair keyPair = getKeyPair(ec2);

      RunInstancesRequest riRequest = new RunInstancesRequest(ami, 1, 1);
      InstanceNetworkInterfaceSpecification net = new InstanceNetworkInterfaceSpecification();

      riRequest.setEbsOptimized(ebsOptimized);

      if (useEphemeralDevices) {
        setupEphemeralDeviceMapping(riRequest);
      } else {
        setupCustomDeviceMapping(riRequest);
      }

      if (stopOnTerminate) {
        riRequest.setInstanceInitiatedShutdownBehavior(ShutdownBehavior.Stop);
        logProvisionInfo(
            logger, "Setting Instance Initiated Shutdown Behavior : ShutdownBehavior.Stop");
      } else {
        riRequest.setInstanceInitiatedShutdownBehavior(ShutdownBehavior.Terminate);
        logProvisionInfo(
            logger, "Setting Instance Initiated Shutdown Behavior : ShutdownBehavior.Terminate");
      }

      List<Filter> diFilters = new ArrayList<Filter>();
      diFilters.add(new Filter("image-id").withValues(ami));

      if (StringUtils.isNotBlank(getZone())) {
        Placement placement = new Placement(getZone());
        if (getUseDedicatedTenancy()) {
          placement.setTenancy("dedicated");
        }
        riRequest.setPlacement(placement);
        diFilters.add(new Filter("availability-zone").withValues(getZone()));
      }

      if (StringUtils.isNotBlank(getSubnetId())) {
        if (getAssociatePublicIp()) {
          net.setSubnetId(getSubnetId());
        } else {
          riRequest.setSubnetId(getSubnetId());
        }

        diFilters.add(new Filter("subnet-id").withValues(getSubnetId()));

        /*
         * If we have a subnet ID then we can only use VPC security groups
         */
        if (!securityGroupSet.isEmpty()) {
          List<String> groupIds = getEc2SecurityGroups(ec2);

          if (!groupIds.isEmpty()) {
            if (getAssociatePublicIp()) {
              net.setGroups(groupIds);
            } else {
              riRequest.setSecurityGroupIds(groupIds);
            }

            diFilters.add(new Filter("instance.group-id").withValues(groupIds));
          }
        }
      } else {
        /* No subnet: we can use standard security groups by name */
        riRequest.setSecurityGroups(securityGroupSet);
        if (!securityGroupSet.isEmpty()) {
          diFilters.add(new Filter("instance.group-name").withValues(securityGroupSet));
        }
      }

      String userDataString = Base64.encodeBase64String(userData.getBytes(StandardCharsets.UTF_8));
      riRequest.setUserData(userDataString);
      riRequest.setKeyName(keyPair.getKeyName());
      diFilters.add(new Filter("key-name").withValues(keyPair.getKeyName()));
      riRequest.setInstanceType(type.toString());
      diFilters.add(new Filter("instance-type").withValues(type.toString()));

      if (getAssociatePublicIp()) {
        net.setAssociatePublicIpAddress(true);
        net.setDeviceIndex(0);
        riRequest.withNetworkInterfaces(net);
      }

      boolean hasCustomTypeTag = false;
      HashSet<Tag> instTags = null;
      if (tags != null && !tags.isEmpty()) {
        instTags = new HashSet<Tag>();
        for (EC2Tag t : tags) {
          instTags.add(new Tag(t.getName(), t.getValue()));
          diFilters.add(new Filter("tag:" + t.getName()).withValues(t.getValue()));
          if (StringUtils.equals(t.getName(), EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE)) {
            hasCustomTypeTag = true;
          }
        }
      }
      if (!hasCustomTypeTag) {
        if (instTags == null) {
          instTags = new HashSet<Tag>();
        }
        // Append template description as well to identify slaves provisioned per template
        instTags.add(
            new Tag(
                EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE,
                EC2Cloud.getSlaveTypeTagValue(EC2Cloud.EC2_SLAVE_TYPE_DEMAND, description)));
      }

      DescribeInstancesRequest diRequest = new DescribeInstancesRequest();
      diRequest.setFilters(diFilters);

      logProvision(logger, "Looking for existing instances with describe-instance: " + diRequest);

      DescribeInstancesResult diResult = ec2.describeInstances(diRequest);
      EC2AbstractSlave[] ec2Node = new EC2AbstractSlave[1];
      Instance existingInstance = null;
      if (!provisionOptions.contains(ProvisionOptions.FORCE_CREATE)) {
        reservationLoop:
        for (Reservation reservation : diResult.getReservations()) {
          for (Instance instance : reservation.getInstances()) {
            if (checkInstance(logger, instance, requiredLabel, ec2Node)) {
              existingInstance = instance;
              logProvision(
                  logger,
                  "Found existing instance: "
                      + existingInstance
                      + ((ec2Node[0] != null) ? (" node: " + ec2Node[0].getInstanceId()) : ""));
              break reservationLoop;
            }
          }
        }
      }

      if (existingInstance == null) {
        if (!provisionOptions.contains(ProvisionOptions.FORCE_CREATE)
            && !provisionOptions.contains(ProvisionOptions.ALLOW_CREATE)) {
          logProvision(logger, "No existing instance found - but cannot create new instance");
          return null;
        }
        if (StringUtils.isNotBlank(getIamInstanceProfile())) {
          riRequest.setIamInstanceProfile(
              new IamInstanceProfileSpecification().withArn(getIamInstanceProfile()));
        }
        // Have to create a new instance
        Instance inst = ec2.runInstances(riRequest).getReservation().getInstances().get(0);

        /* Now that we have our instance, we can set tags on it */
        if (instTags != null) {
          updateRemoteTags(ec2, instTags, "InvalidInstanceID.NotFound", inst.getInstanceId());

          // That was a remote request - we should also update our
          // local instance data.
          inst.setTags(instTags);
        }
        logProvisionInfo(logger, "No existing instance found - created new instance: " + inst);
        return newOndemandSlave(inst);
      }

      if (existingInstance
              .getState()
              .getName()
              .equalsIgnoreCase(InstanceStateName.Stopping.toString())
          || existingInstance
              .getState()
              .getName()
              .equalsIgnoreCase(InstanceStateName.Stopped.toString())) {

        List<String> instances = new ArrayList<String>();
        instances.add(existingInstance.getInstanceId());
        StartInstancesRequest siRequest = new StartInstancesRequest(instances);
        StartInstancesResult siResult = ec2.startInstances(siRequest);

        logProvisionInfo(
            logger,
            "Found stopped instance - starting it: " + existingInstance + " result:" + siResult);
      } else {
        // Should be pending or running at this point, just let it come up
        logProvisionInfo(
            logger,
            "Found existing pending or running: "
                + existingInstance.getState().getName()
                + " instance: "
                + existingInstance);
      }

      if (ec2Node[0] != null) {
        logProvisionInfo(logger, "Using existing slave: " + ec2Node[0].getInstanceId());
        return ec2Node[0];
      }

      // Existing slave not found
      logProvision(logger, "Creating new slave for existing instance: " + existingInstance);
      return newOndemandSlave(existingInstance);

    } catch (FormException e) {
      throw new AssertionError(e); // we should have discovered all
      // configuration issues upfront
    } catch (InterruptedException e) {
      throw new RuntimeException(e);
    }
  }
  @Override
  public List<DiscoveryNode> buildDynamicNodes() {
    List<DiscoveryNode> discoNodes = new ArrayList<>();

    DescribeInstancesResult descInstances;
    try {
      // Query EC2 API based on AZ, instance state, and tag.

      // NOTE: we don't filter by security group during the describe instances request for two
      // reasons:
      // 1. differences in VPCs require different parameters during query (ID vs Name)
      // 2. We want to use two different strategies: (all security groups vs. any security groups)
      descInstances = client.describeInstances(buildDescribeInstancesRequest());
    } catch (AmazonClientException e) {
      logger.info("Exception while retrieving instance list from AWS API: {}", e.getMessage());
      logger.debug("Full exception:", e);
      return discoNodes;
    }

    logger.trace("building dynamic unicast discovery nodes...");
    for (Reservation reservation : descInstances.getReservations()) {
      for (Instance instance : reservation.getInstances()) {
        // lets see if we can filter based on groups
        if (!groups.isEmpty()) {
          List<GroupIdentifier> instanceSecurityGroups = instance.getSecurityGroups();
          ArrayList<String> securityGroupNames = new ArrayList<String>();
          ArrayList<String> securityGroupIds = new ArrayList<String>();
          for (GroupIdentifier sg : instanceSecurityGroups) {
            securityGroupNames.add(sg.getGroupName());
            securityGroupIds.add(sg.getGroupId());
          }
          if (bindAnyGroup) {
            // We check if we can find at least one group name or one group id in groups.
            if (Collections.disjoint(securityGroupNames, groups)
                && Collections.disjoint(securityGroupIds, groups)) {
              logger.trace(
                  "filtering out instance {} based on groups {}, not part of {}",
                  instance.getInstanceId(),
                  instanceSecurityGroups,
                  groups);
              // continue to the next instance
              continue;
            }
          } else {
            // We need tp match all group names or group ids, otherwise we ignore this instance
            if (!(securityGroupNames.containsAll(groups) || securityGroupIds.containsAll(groups))) {
              logger.trace(
                  "filtering out instance {} based on groups {}, does not include all of {}",
                  instance.getInstanceId(),
                  instanceSecurityGroups,
                  groups);
              // continue to the next instance
              continue;
            }
          }
        }

        String address = null;
        switch (hostType) {
          case PRIVATE_DNS:
            address = instance.getPrivateDnsName();
            break;
          case PRIVATE_IP:
            address = instance.getPrivateIpAddress();
            break;
          case PUBLIC_DNS:
            address = instance.getPublicDnsName();
            break;
          case PUBLIC_IP:
            address = instance.getPublicIpAddress();
            break;
        }
        if (address != null) {
          try {
            TransportAddress[] addresses = transportService.addressesFromString(address);
            // we only limit to 1 addresses, makes no sense to ping 100 ports
            for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) {
              logger.trace(
                  "adding {}, address {}, transport_address {}",
                  instance.getInstanceId(),
                  address,
                  addresses[i]);
              discoNodes.add(
                  new DiscoveryNode(
                      "#cloud-" + instance.getInstanceId() + "-" + i,
                      addresses[i],
                      version.minimumCompatibilityVersion()));
            }
          } catch (Exception e) {
            logger.warn("failed ot add {}, address {}", e, instance.getInstanceId(), address);
          }
        } else {
          logger.trace(
              "not adding {}, address is null, host_type {}", instance.getInstanceId(), hostType);
        }
      }
    }

    logger.debug("using dynamic discovery nodes {}", discoNodes);

    return discoNodes;
  }
Exemplo n.º 7
0
 @Override
 public void addTask(Task task) {
   tasks.add(task);
 }