private void blockUntilRunningAndAssignElasticIpsToInstancesOrPutIntoBadMap(
Set<RunningInstance> input, Map<NodeMetadata, Exception> badNodes) {
Map<RegionAndName, RunningInstance> instancesById =
Maps.uniqueIndex(input, instanceToRegionAndName);
for (Map.Entry<RegionAndName, RunningInstance> entry : instancesById.entrySet()) {
RegionAndName id = entry.getKey();
RunningInstance instance = entry.getValue();
try {
logger.debug("<< allocating elastic IP instance(%s)", id);
String ip = client.getElasticIPAddressServices().allocateAddressInRegion(id.getRegion());
// block until instance is running
logger.debug(">> awaiting status running instance(%s)", id);
AtomicReference<NodeMetadata> node =
newReference(runningInstanceToNodeMetadata.apply(instance));
nodeRunning.apply(node);
logger.trace("<< running instance(%s)", id);
logger.debug(">> associating elastic IP %s to instance %s", ip, id);
client
.getElasticIPAddressServices()
.associateAddressInRegion(id.getRegion(), ip, id.getName());
logger.trace("<< associated elastic IP %s to instance %s", ip, id);
// add mapping of instance to ip into the cache
elasticIpCache.put(id, ip);
} catch (RuntimeException e) {
badNodes.put(runningInstanceToNodeMetadata.apply(instancesById.get(id)), e);
}
}
}
public boolean apply(LoadBalancerRule rule) {
logger.trace("looking for state on rule %s", checkNotNull(rule, "rule"));
rule = refresh(rule);
if (rule == null) return false;
logger.trace(
"%s: looking for rule state %s: currently: %s",
rule.getId(), State.ACTIVE, rule.getState());
return rule.getState() == State.ACTIVE;
}
public ExecResponse runAction(String action) {
ExecResponse returnVal;
String command =
(runAsRoot && Predicates.in(ImmutableSet.of("start", "stop", "run")).apply(action))
? execScriptAsRoot(action)
: execScriptAsDefaultUser(action);
returnVal = runCommand(command);
if ("status".equals(action)) logger.trace("<< %s(%d)", action, returnVal.getExitCode());
else if (computeLogger.isTraceEnabled()) computeLogger.trace("<< %s[%s]", action, returnVal);
else computeLogger.debug("<< %s(%d)", action, returnVal.getExitCode());
return returnVal;
}
public boolean apply(String taskId) {
logger.trace("looking for status on task %s", checkNotNull(taskId, "taskId"));
Task task = refresh(taskId);
if (task == null) return false;
logger.trace(
"%s: looking for task status %s: currently: %s",
task.getId(), Task.Status.SUCCESS, task.getStatus());
if (task.getError() != null)
throw new IllegalStateException(
String.format(
"task %s failed with exception %s", task.getId(), task.getError().toString()));
return task.getStatus() == Task.Status.SUCCESS;
}
@Override
public Iterable<? extends Node> execute(
final ListeningExecutorService executor, String environmentName, Iterable<String> toGet) {
ListenableFuture<List<Node>> futures =
allAsList(
transform(
toGet,
new Function<String, ListenableFuture<Node>>() {
@Override
public ListenableFuture<Node> apply(final String input) {
return executor.submit(
new Callable<Node>() {
@Override
public Node call() throws Exception {
return api.getNode(input);
}
});
}
}));
logger.trace(
String.format(
"getting nodes in environment %s: %s", environmentName, Joiner.on(',').join(toGet)));
return getUnchecked(futures);
}
@Override
public ServerInDataCenter getNode(String id) {
DataCenterAndId datacenterAndId = DataCenterAndId.fromSlashEncoded(id);
logger.trace("<< searching for server with id=%s", id);
Server server =
api.serverApi()
.getServer(
datacenterAndId.getDataCenter(),
datacenterAndId.getId(),
new DepthOptions().depth(3));
if (server != null) {
logger.trace(">> found server [%s]", server.properties().name());
}
return server == null ? null : new ServerInDataCenter(server, datacenterAndId.getDataCenter());
}
@Override
public ExecResponse call() {
checkState(ssh != null, "please call init() before invoking call");
try {
ssh.connect();
ExecResponse returnVal;
eventBus.post(new StatementOnNodeSubmission(statement, node));
String command =
runAsRoot
? execAsRoot(statement.render(OsFamily.UNIX))
: execScriptAsDefaultUser(statement.render(OsFamily.UNIX));
try {
returnVal = runCommand(command);
} catch (Throwable e) {
eventBus.post(new StatementOnNodeFailure(statement, node, e));
throw Throwables.propagate(e);
}
eventBus.post(new StatementOnNodeCompletion(statement, node, returnVal));
if (logger.isTraceEnabled()) logger.trace("<< %s[%s]", statement, returnVal);
else logger.debug("<< %s(%d)", statement, returnVal.getExitStatus());
return returnVal;
} finally {
if (ssh != null) ssh.disconnect();
}
}
private Iterable<VirtualMachineWithNodeExtendedDto> listConcurrentVirtualMachines(
final ListeningExecutorService executor,
final Iterable<VirtualAppliance> vapps,
final VirtualMachineOptions options) {
ListenableFuture<List<VirtualMachinesWithNodeExtendedDto>> futures =
allAsList(
transform(
vapps,
new Function<
VirtualAppliance, ListenableFuture<VirtualMachinesWithNodeExtendedDto>>() {
@Override
public ListenableFuture<VirtualMachinesWithNodeExtendedDto> apply(
final VirtualAppliance input) {
return executor.submit(
new Callable<VirtualMachinesWithNodeExtendedDto>() {
@Override
public VirtualMachinesWithNodeExtendedDto call() throws Exception {
return context
.getApi()
.getCloudApi()
.listVirtualMachines(input.unwrap(), options);
}
});
}
}));
logger.trace("getting virtual machines");
return DomainWrapper.join(getUnchecked(futures));
}
private Iterable<RackDto> listConcurrentRacks(
final ListeningExecutorService executor, final Iterable<Datacenter> datacenters) {
ListenableFuture<List<RacksDto>> futures =
allAsList(
transform(
datacenters,
new Function<Datacenter, ListenableFuture<RacksDto>>() {
@Override
public ListenableFuture<RacksDto> apply(final Datacenter input) {
return executor.submit(
new Callable<RacksDto>() {
@Override
public RacksDto call() throws Exception {
return context
.getApi()
.getInfrastructureApi()
.listRacks(input.unwrap());
}
});
}
}));
logger.trace("getting racks");
return DomainWrapper.join(getUnchecked(futures));
}
@Override
public Iterable<ServerInDataCenter> listNodes() {
logger.trace("<< fetching servers..");
datacetners = api.dataCenterApi().list();
List<ServerInDataCenter> servers = new ArrayList<ServerInDataCenter>();
for (DataCenter dataCenter : datacetners) {
List<Server> serversInDataCenter =
api.serverApi().getList(dataCenter.id(), new DepthOptions().depth(4));
for (Server server : serversInDataCenter) {
servers.add(new ServerInDataCenter(server, dataCenter.id()));
}
}
logger.trace("<< fetching servers..");
return servers;
}
protected ExecResponse runAction(String action) {
ExecResponse returnVal;
String command = (runAsRoot) ? execScriptAsRoot(action) : execScriptAsDefaultUser(action);
returnVal = runCommand(command);
if (logger.isTraceEnabled()) logger.trace("<< %s[%s]", action, returnVal);
else logger.debug("<< %s(%d)", action, returnVal.getExitCode());
return returnVal;
}
public static <T> Map<T, Exception> awaitCompletion(
Map<T, ? extends Future<?>> responses,
ExecutorService exec,
@Nullable Long maxTime,
final Logger logger,
final String logPrefix) {
if (responses.size() == 0) return ImmutableMap.of();
final int total = responses.size();
final CountDownLatch doneSignal = new CountDownLatch(total);
final AtomicInteger complete = new AtomicInteger(0);
final AtomicInteger errors = new AtomicInteger(0);
final long start = System.currentTimeMillis();
final Map<T, Exception> errorMap = Maps.newHashMap();
for (final java.util.Map.Entry<T, ? extends Future<?>> future : responses.entrySet()) {
Futures.makeListenable(future.getValue(), exec)
.addListener(
new Runnable() {
@Override
public void run() {
try {
future.getValue().get();
complete.incrementAndGet();
} catch (Exception e) {
errors.incrementAndGet();
logException(logger, logPrefix, total, complete.get(), errors.get(), start, e);
errorMap.put(future.getKey(), e);
}
doneSignal.countDown();
}
@Override
public String toString() {
return "callGetOnFuture(" + future.getKey() + "," + future.getValue() + ")";
}
},
exec);
}
try {
if (maxTime != null) doneSignal.await(maxTime, TimeUnit.MILLISECONDS);
else doneSignal.await();
if (errors.get() > 0) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
RuntimeException exception = new RuntimeException(message);
logger.error(exception, message);
}
if (logger.isTraceEnabled()) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
logger.trace(message);
}
} catch (InterruptedException e) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
TimeoutException exception = new TimeoutException(message);
logger.error(exception, message);
Throwables.propagate(exception);
}
return errorMap;
}
@Override
public Provisionable getImage(String id) {
// try search images
logger.trace("<< searching for image with id=%s", id);
Image image = api.imageApi().getImage(id);
if (image != null) {
logger.trace(">> found image [%s].", image.properties().name());
return image;
}
// try search snapshots
logger.trace("<< not found from images. searching for snapshot with id=%s", id);
Snapshot snapshot = api.snapshotApi().get(id);
if (snapshot != null) {
logger.trace(">> found snapshot [%s]", snapshot.properties().name());
return snapshot;
}
throw new ResourceNotFoundException("No image/snapshot with id '" + id + "' was found");
}
private void destroyServer(final String serverId, final String dataCenterId) {
try {
logger.trace("<< deleting server with id=%s", serverId);
provisioningManager.provision(
jobFactory.create(
dataCenterId,
new Supplier<Object>() {
@Override
public Object get() {
URI requestStatusURI = api.serverApi().deleteServer(dataCenterId, serverId);
trackables.waitUntilRequestCompleted(requestStatusURI);
return serverId;
}
}));
logger.trace(">> server '%s' deleted.", serverId);
} catch (Exception ex) {
logger.warn(ex, ">> failed to delete server with id=%s", serverId);
}
}
ExecResponse runCommand(String command) {
String statement =
String.format(
">> running [%s] as %s@%s",
command.replace(
node.getCredentials().getPassword() != null
? node.getCredentials().getPassword()
: "XXXXX",
"XXXXX"),
ssh.getUsername(),
ssh.getHostAddress());
if (command.endsWith("status")) logger.trace(statement);
else computeLogger.debug(statement);
return ssh.exec(command);
}
/**
* attempts to start the specified count of instances. eventual consistency might cause a problem
* where instances aren't immediately visible to the api. This method will warn when that occurs.
*/
private Set<RunningInstance> runInstancesAndWarnOnInvisible(
String group, int count, Template mutableTemplate) {
Set<RunningInstance> started =
createKeyPairAndSecurityGroupsAsNeededThenRunInstances(group, count, mutableTemplate);
Set<RegionAndName> startedIds =
ImmutableSet.copyOf(transform(started, instanceToRegionAndName));
if (startedIds.size() == 0) {
return ImmutableSet.copyOf(started);
}
logger.debug("<< started instances(%s)", startedIds);
Set<RunningInstance> visible = presentInstances.apply(startedIds);
Set<RegionAndName> visibleIds =
ImmutableSet.copyOf(transform(visible, instanceToRegionAndName));
logger.trace("<< visible instances(%s)", visibleIds);
// add an exception for each of the nodes we cannot customize
Set<RegionAndName> invisibleIds = difference(startedIds, visibleIds);
if (invisibleIds.size() > 0) {
logger.warn("<< not api visible instances(%s)", invisibleIds);
}
return started;
}
private Iterable<MachineDto> listConcurrentMachines(
final ListeningExecutorService executor, final Iterable<RackDto> racks) {
ListenableFuture<List<MachinesDto>> futures =
allAsList(
transform(
racks,
new Function<RackDto, ListenableFuture<MachinesDto>>() {
@Override
public ListenableFuture<MachinesDto> apply(final RackDto input) {
return executor.submit(
new Callable<MachinesDto>() {
@Override
public MachinesDto call() throws Exception {
return context.getApi().getInfrastructureApi().listMachines(input);
}
});
}
}));
logger.trace("getting machines");
return DomainWrapper.join(getUnchecked(futures));
}
@Override
public Iterable<? extends Client> execute(
final ListeningExecutorService executor, Iterable<String> toGet) {
ListenableFuture<List<Client>> futures =
allAsList(
transform(
toGet,
new Function<String, ListenableFuture<Client>>() {
@Override
public ListenableFuture<Client> apply(final String input) {
return executor.submit(
new Callable<Client>() {
@Override
public Client call() throws Exception {
return api.getClient(input);
}
});
}
}));
logger.trace(String.format("getting clients: %s", Joiner.on(',').join(toGet)));
return getUnchecked(futures);
}
protected NodeAndInitialCredentials<ServerInDataCenter> createNodeWithGroupEncodedIntoName(
String group, String name, TemplateWithDataCenter template) {
checkArgument(
template.getLocation().getScope() == LocationScope.ZONE,
"Template must use a ZONE-scoped location");
final String dataCenterId = template.getDataCenter().id();
Hardware hardware = template.getHardware();
TemplateOptions options = template.getOptions();
final String loginUser =
isNullOrEmpty(options.getLoginUser()) ? "root" : options.getLoginUser();
final String password =
options.hasLoginPassword() ? options.getLoginPassword() : Passwords.generate();
final org.jclouds.compute.domain.Image image = template.getImage();
// provision all volumes based on hardware
List<? extends Volume> volumes = hardware.getVolumes();
List<String> volumeIds = Lists.newArrayListWithExpectedSize(volumes.size());
int i = 1;
for (final Volume volume : volumes) {
try {
logger.trace("<< provisioning volume '%s'", volume);
final org.apache.jclouds.profitbricks.rest.domain.Volume.Request.CreatePayload.Builder
request = org.apache.jclouds.profitbricks.rest.domain.Volume.Request.creatingBuilder();
if (i == 1) {
request.image(image.getId());
// we don't need to pass password to the API if we're using a snapshot
Provisionable.Type provisionableType =
Provisionable.Type.fromValue(
image.getUserMetadata().get(ProvisionableToImage.KEY_PROVISIONABLE_TYPE));
if (provisionableType == Provisionable.Type.IMAGE) {
request.imagePassword(password);
}
}
request
.dataCenterId(dataCenterId)
.name(format("%s-disk-%d", name, i++))
.size(volume.getSize().intValue())
.type(VolumeType.HDD);
org.apache.jclouds.profitbricks.rest.domain.Volume vol =
(org.apache.jclouds.profitbricks.rest.domain.Volume)
provisioningManager.provision(
jobFactory.create(
dataCenterId,
new Supplier<Object>() {
@Override
public Object get() {
return api.volumeApi().createVolume(request.build());
}
}));
volumeIds.add(vol.id());
logger.trace(">> provisioning complete for volume. returned id='%s'", vol.id());
} catch (Exception ex) {
if (i - 1 == 1) // if first volume (one with image) provisioning fails; stop method
{
throw Throwables.propagate(ex);
}
logger.warn(ex, ">> failed to provision volume. skipping..");
}
}
String volumeBootDeviceId = Iterables.get(volumeIds, 0); // must have atleast 1
waitVolumeUntilAvailable.apply(VolumeRef.create(dataCenterId, volumeBootDeviceId));
// provision server
final Server server;
Double cores = ComputeServiceUtils.getCores(hardware);
Server.BootVolume bootVolume = Server.BootVolume.create(volumeBootDeviceId);
try {
final Server.Request.CreatePayload serverRequest =
Server.Request.creatingBuilder()
.dataCenterId(dataCenterId)
.name(name)
.bootVolume(bootVolume)
.cores(cores.intValue())
.ram(hardware.getRam())
.build();
logger.trace("<< provisioning server '%s'", serverRequest);
server =
(Server)
provisioningManager.provision(
jobFactory.create(
dataCenterId,
new Supplier<Object>() {
@Override
public Object get() {
return api.serverApi().createServer(serverRequest);
}
}));
logger.trace(">> provisioning complete for server. returned id='%s'", server.id());
} catch (Exception ex) {
logger.error(ex, ">> failed to provision server. rollbacking..");
destroyVolumes(volumeIds, dataCenterId);
throw Throwables.propagate(ex);
}
waitServerUntilAvailable.apply(ServerRef.create(dataCenterId, server.id()));
waitDcUntilAvailable.apply(dataCenterId);
// attach bootVolume to Server
org.apache.jclouds.profitbricks.rest.domain.Volume volume =
api.serverApi()
.attachVolume(
Server.Request.attachVolumeBuilder()
.dataCenterId(dataCenterId)
.serverId(server.id())
.volumeId(bootVolume.id())
.build());
trackables.waitUntilRequestCompleted(volume);
waitServerUntilAvailable.apply(ServerRef.create(dataCenterId, server.id()));
waitDcUntilAvailable.apply(dataCenterId);
// fetch an existing lan and creat if non was found
Lan lan = null;
List<Lan> lans = api.lanApi().list(dataCenterId);
if (lans != null && !lans.isEmpty()) {
lan =
FluentIterable.from(lans)
.firstMatch(
new Predicate<Lan>() {
@Override
public boolean apply(Lan input) {
input =
api.lanApi().get(dataCenterId, input.id(), new DepthOptions().depth(3));
return input.properties().isPublic();
}
})
.orNull();
}
if (lan == null) {
logger.warn("Could not find an existing lan Creating one....");
lan =
api.lanApi()
.create(
Lan.Request.creatingBuilder()
.dataCenterId(dataCenterId)
.isPublic(Boolean.TRUE)
.name("lan " + name)
.build());
trackables.waitUntilRequestCompleted(lan);
}
// add a NIC to the server
int lanId = DEFAULT_LAN_ID;
if (options.getNetworks() != null) {
try {
String networkId = Iterables.get(options.getNetworks(), 0);
lanId = Integer.valueOf(networkId);
} catch (Exception ex) {
logger.warn(
"no valid network id found from options. using default id='%d'", DEFAULT_LAN_ID);
}
}
Nic nic =
api.nicApi()
.create(
Nic.Request.creatingBuilder()
.dataCenterId(dataCenterId)
.name("jclouds" + name)
.dhcp(Boolean.TRUE)
.lan(lanId)
.firewallActive(Boolean.FALSE)
.serverId(server.id())
.build());
trackables.waitUntilRequestCompleted(nic);
waitNICUntilAvailable.apply(NicRef.create(dataCenterId, server.id(), nic.id()));
waitDcUntilAvailable.apply(dataCenterId);
waitServerUntilAvailable.apply(ServerRef.create(dataCenterId, server.id()));
// connect the rest of volumes to server;delete if fails
final int volumeCount = volumeIds.size();
for (int j = 1; j < volumeCount; j++) { // skip first; already connected
final String volumeId = volumeIds.get(j);
try {
logger.trace("<< connecting volume '%s' to server '%s'", volumeId, server.id());
provisioningManager.provision(
jobFactory.create(
group,
new Supplier<Object>() {
@Override
public Object get() {
return api.serverApi()
.attachVolume(
Server.Request.attachVolumeBuilder()
.dataCenterId(dataCenterId)
.serverId(server.id())
.volumeId(volumeId)
.build());
}
}));
logger.trace(">> volume connected.");
} catch (Exception ex) {
try {
// delete unconnected volume
logger.warn(ex, ">> failed to connect volume '%s'. deleting..", volumeId);
destroyVolume(volumeId, dataCenterId);
logger.warn(ex, ">> rolling back server..", server.id());
destroyServer(server.id(), dataCenterId);
throw ex;
} catch (Exception ex1) {
logger.error(ex, ">> failed to rollback");
}
}
}
waitDcUntilAvailable.apply(dataCenterId);
waitServerUntilAvailable.apply(ServerRef.create(dataCenterId, server.id()));
LoginCredentials serverCredentials =
LoginCredentials.builder().user(loginUser).password(password).build();
String serverInDataCenterId =
DataCenterAndId.fromDataCenterAndId(dataCenterId, server.id()).slashEncode();
ServerInDataCenter serverInDatacenter = getNode(serverInDataCenterId);
return new NodeAndInitialCredentials<ServerInDataCenter>(
serverInDatacenter, serverInDataCenterId, serverCredentials);
}
