/**
* Counts the number of instances in EC2 currently running that are using the specified image and
* a template.
*
* @param ami If AMI is left null, then all instances are counted and template description is
* ignored.
* <p>This includes those instances that may be started outside Jenkins.
* @param templateDesc
*/
public int countCurrentEC2Slaves(String ami, String templateDesc) throws AmazonClientException {
int n = 0;
for (Reservation r : connect().describeInstances().getReservations()) {
for (Instance i : r.getInstances()) {
if (isEc2ProvisionedAmiSlave(i, ami, templateDesc)) {
InstanceStateName stateName = InstanceStateName.fromValue(i.getState().getName());
if (stateName == InstanceStateName.Pending || stateName == InstanceStateName.Running) {
EC2AbstractSlave foundSlave = null;
for (EC2AbstractSlave ec2Node : NodeIterator.nodes(EC2AbstractSlave.class)) {
if (ec2Node.getInstanceId().equals(i.getInstanceId())) {
foundSlave = ec2Node;
break;
}
}
// Don't count disconnected slaves as being used, we will connected them later is
// required
if (foundSlave != null && foundSlave.toComputer().isOffline()) continue;
n++;
}
}
}
}
// Count pending spot requests too
for (SpotInstanceRequest sir :
connect().describeSpotInstanceRequests().getSpotInstanceRequests()) {
// Count Spot requests that are open and still have a
// chance to be active.
if (sir.getState().equals("open")) {
n++;
}
}
return n;
}
@Override
public Collection<PlannedNode> provision(Label label, int excessWorkload) {
try {
// Count number of pending executors from spot requests
for (EC2SpotSlave n : NodeIterator.nodes(EC2SpotSlave.class)) {
// If the slave is online then it is already counted by Jenkins
// We only want to count potential additional Spot instance
// slaves
if (n.getComputer().isOffline() && label.matches(n.getAssignedLabels())) {
DescribeSpotInstanceRequestsRequest dsir =
new DescribeSpotInstanceRequestsRequest()
.withSpotInstanceRequestIds(n.getSpotInstanceRequestId());
for (SpotInstanceRequest sir :
connect().describeSpotInstanceRequests(dsir).getSpotInstanceRequests()) {
// Count Spot requests that are open and still have a
// chance to be active
// A request can be active and not yet registered as a
// slave. We check above
// to ensure only unregistered slaves get counted
if (sir.getState().equals("open") || sir.getState().equals("active")) {
excessWorkload -= n.getNumExecutors();
}
}
}
}
LOGGER.log(Level.INFO, "Excess workload after pending Spot instances: " + excessWorkload);
List<PlannedNode> r = new ArrayList<PlannedNode>();
final SlaveTemplate t = getTemplate(label);
int amiCap = t.getInstanceCap();
while (excessWorkload > 0) {
if (!addProvisionedSlave(t.ami, amiCap, t.description)) {
break;
}
r.add(
new PlannedNode(
t.getDisplayName(),
Computer.threadPoolForRemoting.submit(
new Callable<Node>() {
public Node call() throws Exception {
// TODO: record the output somewhere
try {
EC2AbstractSlave s = t.provision(StreamTaskListener.fromStdout());
Hudson.getInstance().addNode(s);
// EC2 instances may have a long init script. If we
// declare
// the provisioning complete by returning without
// the connect
// operation, NodeProvisioner may decide that it
// still wants
// one more instance, because it sees that (1) all
// the slaves
// are offline (because it's still being launched)
// and
// (2) there's no capacity provisioned yet.
//
// deferring the completion of provisioning until
// the launch
// goes successful prevents this problem.
s.toComputer().connect(false).get();
return s;
} finally {
decrementAmiSlaveProvision(t.ami);
}
}
}),
t.getNumExecutors()));
excessWorkload -= t.getNumExecutors();
}
return r;
} catch (AmazonClientException e) {
LOGGER.log(Level.WARNING, "Failed to count the # of live instances on EC2", e);
return Collections.emptyList();
}
}
