/** * Counts the number of instances in EC2 currently running that are using the specified image and * a template. * * @param ami If AMI is left null, then all instances are counted and template description is * ignored. * <p>This includes those instances that may be started outside Jenkins. * @param templateDesc */ public int countCurrentEC2Slaves(String ami, String templateDesc) throws AmazonClientException { int n = 0; for (Reservation r : connect().describeInstances().getReservations()) { for (Instance i : r.getInstances()) { if (isEc2ProvisionedAmiSlave(i, ami, templateDesc)) { InstanceStateName stateName = InstanceStateName.fromValue(i.getState().getName()); if (stateName == InstanceStateName.Pending || stateName == InstanceStateName.Running) { EC2AbstractSlave foundSlave = null; for (EC2AbstractSlave ec2Node : NodeIterator.nodes(EC2AbstractSlave.class)) { if (ec2Node.getInstanceId().equals(i.getInstanceId())) { foundSlave = ec2Node; break; } } // Don't count disconnected slaves as being used, we will connected them later is // required if (foundSlave != null && foundSlave.toComputer().isOffline()) continue; n++; } } } } // Count pending spot requests too for (SpotInstanceRequest sir : connect().describeSpotInstanceRequests().getSpotInstanceRequests()) { // Count Spot requests that are open and still have a // chance to be active. if (sir.getState().equals("open")) { n++; } } return n; }
protected EC2SpotSlave newSpotSlave(SpotInstanceRequest sir, String name) throws FormException, IOException { return new EC2SpotSlave( name, sir.getSpotInstanceRequestId(), description, remoteFS, getNumExecutors(), mode, initScript, tmpDir, labels, remoteAdmin, jvmopts, idleTerminationMinutes, EC2Tag.fromAmazonTags(sir.getTags()), parent.name, usePrivateDnsName, getLaunchTimeout(), amiType); }
/** Provision a new slave for an EC2 spot instance to call back to Jenkins */ private EC2AbstractSlave provisionSpot(TaskListener listener) throws AmazonClientException, IOException { PrintStream logger = listener.getLogger(); AmazonEC2 ec2 = getParent().connect(); try { logger.println("Launching " + ami + " for template " + description); LOGGER.info("Launching " + ami + " for template " + description); KeyPair keyPair = getKeyPair(ec2); RequestSpotInstancesRequest spotRequest = new RequestSpotInstancesRequest(); // Validate spot bid before making the request if (getSpotMaxBidPrice() == null) { // throw new FormException("Invalid Spot price specified: " + // getSpotMaxBidPrice(), "spotMaxBidPrice"); throw new AmazonClientException("Invalid Spot price specified: " + getSpotMaxBidPrice()); } spotRequest.setSpotPrice(getSpotMaxBidPrice()); spotRequest.setInstanceCount(1); LaunchSpecification launchSpecification = new LaunchSpecification(); InstanceNetworkInterfaceSpecification net = new InstanceNetworkInterfaceSpecification(); launchSpecification.setImageId(ami); launchSpecification.setInstanceType(type); if (StringUtils.isNotBlank(getZone())) { SpotPlacement placement = new SpotPlacement(getZone()); launchSpecification.setPlacement(placement); } if (StringUtils.isNotBlank(getSubnetId())) { if (getAssociatePublicIp()) { net.setSubnetId(getSubnetId()); } else { launchSpecification.setSubnetId(getSubnetId()); } /* * If we have a subnet ID then we can only use VPC security groups */ if (!securityGroupSet.isEmpty()) { List<String> groupIds = getEc2SecurityGroups(ec2); if (!groupIds.isEmpty()) { if (getAssociatePublicIp()) { net.setGroups(groupIds); } else { ArrayList<GroupIdentifier> groups = new ArrayList<GroupIdentifier>(); for (String group_id : groupIds) { GroupIdentifier group = new GroupIdentifier(); group.setGroupId(group_id); groups.add(group); } if (!groups.isEmpty()) launchSpecification.setAllSecurityGroups(groups); } } } } else { /* No subnet: we can use standard security groups by name */ if (!securityGroupSet.isEmpty()) { launchSpecification.setSecurityGroups(securityGroupSet); } } String userDataString = Base64.encodeBase64String(userData.getBytes(StandardCharsets.UTF_8)); launchSpecification.setUserData(userDataString); launchSpecification.setKeyName(keyPair.getKeyName()); launchSpecification.setInstanceType(type.toString()); if (getAssociatePublicIp()) { net.setAssociatePublicIpAddress(true); net.setDeviceIndex(0); launchSpecification.withNetworkInterfaces(net); } boolean hasCustomTypeTag = false; HashSet<Tag> instTags = null; if (tags != null && !tags.isEmpty()) { instTags = new HashSet<Tag>(); for (EC2Tag t : tags) { instTags.add(new Tag(t.getName(), t.getValue())); if (StringUtils.equals(t.getName(), EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE)) { hasCustomTypeTag = true; } } } if (!hasCustomTypeTag) { if (instTags != null) instTags.add( new Tag( EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE, EC2Cloud.getSlaveTypeTagValue(EC2Cloud.EC2_SLAVE_TYPE_SPOT, description))); } if (StringUtils.isNotBlank(getIamInstanceProfile())) { launchSpecification.setIamInstanceProfile( new IamInstanceProfileSpecification().withArn(getIamInstanceProfile())); } if (useEphemeralDevices) { setupEphemeralDeviceMapping(launchSpecification); } else { setupCustomDeviceMapping(launchSpecification); } spotRequest.setLaunchSpecification(launchSpecification); // Make the request for a new Spot instance RequestSpotInstancesResult reqResult = ec2.requestSpotInstances(spotRequest); List<SpotInstanceRequest> reqInstances = reqResult.getSpotInstanceRequests(); if (reqInstances.isEmpty()) { throw new AmazonClientException("No spot instances found"); } SpotInstanceRequest spotInstReq = reqInstances.get(0); if (spotInstReq == null) { throw new AmazonClientException("Spot instance request is null"); } String slaveName = spotInstReq.getSpotInstanceRequestId(); /* Now that we have our Spot request, we can set tags on it */ if (instTags != null) { updateRemoteTags( ec2, instTags, "InvalidSpotInstanceRequestID.NotFound", spotInstReq.getSpotInstanceRequestId()); // That was a remote request - we should also update our local // instance data. spotInstReq.setTags(instTags); } logger.println("Spot instance id in provision: " + spotInstReq.getSpotInstanceRequestId()); LOGGER.info("Spot instance id in provision: " + spotInstReq.getSpotInstanceRequestId()); return newSpotSlave(spotInstReq, slaveName); } catch (FormException e) { throw new AssertionError(); // we should have discovered all // configuration issues upfront } catch (InterruptedException e) { throw new RuntimeException(e); } }
@Override public Collection<PlannedNode> provision(Label label, int excessWorkload) { try { // Count number of pending executors from spot requests for (EC2SpotSlave n : NodeIterator.nodes(EC2SpotSlave.class)) { // If the slave is online then it is already counted by Jenkins // We only want to count potential additional Spot instance // slaves if (n.getComputer().isOffline() && label.matches(n.getAssignedLabels())) { DescribeSpotInstanceRequestsRequest dsir = new DescribeSpotInstanceRequestsRequest() .withSpotInstanceRequestIds(n.getSpotInstanceRequestId()); for (SpotInstanceRequest sir : connect().describeSpotInstanceRequests(dsir).getSpotInstanceRequests()) { // Count Spot requests that are open and still have a // chance to be active // A request can be active and not yet registered as a // slave. We check above // to ensure only unregistered slaves get counted if (sir.getState().equals("open") || sir.getState().equals("active")) { excessWorkload -= n.getNumExecutors(); } } } } LOGGER.log(Level.INFO, "Excess workload after pending Spot instances: " + excessWorkload); List<PlannedNode> r = new ArrayList<PlannedNode>(); final SlaveTemplate t = getTemplate(label); int amiCap = t.getInstanceCap(); while (excessWorkload > 0) { if (!addProvisionedSlave(t.ami, amiCap, t.description)) { break; } r.add( new PlannedNode( t.getDisplayName(), Computer.threadPoolForRemoting.submit( new Callable<Node>() { public Node call() throws Exception { // TODO: record the output somewhere try { EC2AbstractSlave s = t.provision(StreamTaskListener.fromStdout()); Hudson.getInstance().addNode(s); // EC2 instances may have a long init script. If we // declare // the provisioning complete by returning without // the connect // operation, NodeProvisioner may decide that it // still wants // one more instance, because it sees that (1) all // the slaves // are offline (because it's still being launched) // and // (2) there's no capacity provisioned yet. // // deferring the completion of provisioning until // the launch // goes successful prevents this problem. s.toComputer().connect(false).get(); return s; } finally { decrementAmiSlaveProvision(t.ami); } } }), t.getNumExecutors())); excessWorkload -= t.getNumExecutors(); } return r; } catch (AmazonClientException e) { LOGGER.log(Level.WARNING, "Failed to count the # of live instances on EC2", e); return Collections.emptyList(); } }