@Override
public Collection<PlannedNode> provision(Label label, int excessWorkload) {
try {
// Count number of pending executors from spot requests
for (EC2SpotSlave n : NodeIterator.nodes(EC2SpotSlave.class)) {
// If the slave is online then it is already counted by Jenkins
// We only want to count potential additional Spot instance
// slaves
if (n.getComputer().isOffline() && label.matches(n.getAssignedLabels())) {
DescribeSpotInstanceRequestsRequest dsir =
new DescribeSpotInstanceRequestsRequest()
.withSpotInstanceRequestIds(n.getSpotInstanceRequestId());
for (SpotInstanceRequest sir :
connect().describeSpotInstanceRequests(dsir).getSpotInstanceRequests()) {
// Count Spot requests that are open and still have a
// chance to be active
// A request can be active and not yet registered as a
// slave. We check above
// to ensure only unregistered slaves get counted
if (sir.getState().equals("open") || sir.getState().equals("active")) {
excessWorkload -= n.getNumExecutors();
}
}
}
}
LOGGER.log(Level.INFO, "Excess workload after pending Spot instances: " + excessWorkload);
List<PlannedNode> r = new ArrayList<PlannedNode>();
final SlaveTemplate t = getTemplate(label);
int amiCap = t.getInstanceCap();
while (excessWorkload > 0) {
if (!addProvisionedSlave(t.ami, amiCap, t.description)) {
break;
}
r.add(
new PlannedNode(
t.getDisplayName(),
Computer.threadPoolForRemoting.submit(
new Callable<Node>() {
public Node call() throws Exception {
// TODO: record the output somewhere
try {
EC2AbstractSlave s = t.provision(StreamTaskListener.fromStdout());
Hudson.getInstance().addNode(s);
// EC2 instances may have a long init script. If we
// declare
// the provisioning complete by returning without
// the connect
// operation, NodeProvisioner may decide that it
// still wants
// one more instance, because it sees that (1) all
// the slaves
// are offline (because it's still being launched)
// and
// (2) there's no capacity provisioned yet.
//
// deferring the completion of provisioning until
// the launch
// goes successful prevents this problem.
s.toComputer().connect(false).get();
return s;
} finally {
decrementAmiSlaveProvision(t.ami);
}
}
}),
t.getNumExecutors()));
excessWorkload -= t.getNumExecutors();
}
return r;
} catch (AmazonClientException e) {
LOGGER.log(Level.WARNING, "Failed to count the # of live instances on EC2", e);
return Collections.emptyList();
}
}
