private Integer hashCodeIfResolved(ConfigKey<?>... keys) {
int hash = 0;
for (ConfigKey<?> k : keys) {
Maybe<?> value = ((ConfigurationSupportInternal) getEntity().config()).getNonBlocking(k);
if (value.isPresent()) {
hash = hash * 31 + (value.get() == null ? 0 : value.get().hashCode());
}
}
return hash;
}
protected Location getLocation(@Nullable Collection<? extends Location> locations) {
if (locations == null || locations.isEmpty()) locations = entity().getLocations();
if (locations.isEmpty()) {
MachineProvisioningLocation<?> provisioner =
entity().getAttribute(SoftwareProcess.PROVISIONING_LOCATION);
if (provisioner != null) locations = Arrays.<Location>asList(provisioner);
}
locations = Locations.getLocationsCheckingAncestors(locations, entity());
Maybe<MachineLocation> ml = Locations.findUniqueMachineLocation(locations);
if (ml.isPresent()) return ml.get();
if (locations.isEmpty())
throw new IllegalArgumentException("No locations specified when starting " + entity());
if (locations.size() != 1 || Iterables.getOnlyElement(locations) == null)
throw new IllegalArgumentException(
"Ambiguous locations detected when starting " + entity() + ": " + locations);
return Iterables.getOnlyElement(locations);
}
@Override
public String getSeeds() {
Set<Entity> seeds = getConfig(CassandraNode.INITIAL_SEEDS);
if (seeds == null) {
log.warn(
"No seeds available when requested for " + this,
new Throwable("source of no Cassandra seeds when requested"));
return null;
}
String snitchName = getConfig(CassandraNode.ENDPOINT_SNITCH_NAME);
MutableSet<String> seedsHostnames = MutableSet.of();
for (Entity entity : seeds) {
// tried removing ourselves if there are other nodes, but that is a BAD idea!
// blows up with a "java.lang.RuntimeException: No other nodes seen!"
if (snitchName.equals("Ec2MultiRegionSnitch") || snitchName.contains("MultiCloudSnitch")) {
// http://www.datastax.com/documentation/cassandra/2.0/mobile/cassandra/architecture/architectureSnitchEC2MultiRegion_c.html
// says the seeds should be public IPs.
seedsHostnames.add(entity.getAttribute(CassandraNode.ADDRESS));
} else {
String sensorName = getConfig(BROADCAST_ADDRESS_SENSOR);
if (Strings.isNonBlank(sensorName)) {
seedsHostnames.add(entity.getAttribute(Sensors.newStringSensor(sensorName)));
} else {
Maybe<String> optionalSeedHostname = Machines.findSubnetOrPublicHostname(entity);
if (optionalSeedHostname.isPresent()) {
String seedHostname = optionalSeedHostname.get();
seedsHostnames.add(seedHostname);
} else {
log.warn(
"In node {}, seed hostname missing for {}; not including in seeds list",
this,
entity);
}
}
}
}
String result = Strings.join(seedsHostnames, ",");
log.info("Seeds for {}: {}", this, result);
return result;
}
protected void configureResolver(Entity entity) {
Maybe<SshMachineLocation> machine =
Machines.findUniqueSshMachineLocation(entity.getLocations());
if (machine.isPresent()) {
if (getConfig(REPLACE_RESOLV_CONF)) {
machine.get().copyTo(new StringReader(getConfig(RESOLV_CONF_TEMPLATE)), "/etc/resolv.conf");
} else {
appendTemplate(
getConfig(INTERFACE_CONFIG_TEMPLATE),
"/etc/sysconfig/network-scripts/ifcfg-eth0",
machine.get());
machine
.get()
.execScript(
"reload network", ImmutableList.of(BashCommands.sudo("service network reload")));
}
LOG.info("configured resolver on {}", machine);
} else {
LOG.debug("{} can't configure resolver at {}: no SshMachineLocation", this, entity);
}
}
public void update() {
Lifecycle serverState = getAttribute(Attributes.SERVICE_STATE_ACTUAL);
if (Lifecycle.STOPPED.equals(serverState)
|| Lifecycle.STOPPING.equals(serverState)
|| Lifecycle.DESTROYED.equals(serverState)
|| !getAttribute(Attributes.SERVICE_UP)) {
LOG.debug(
"Skipped update of {} when service state is {} and running is {}",
new Object[] {
this, getAttribute(Attributes.SERVICE_STATE_ACTUAL), getAttribute(SERVICE_UP)
});
return;
}
synchronized (this) {
Iterable<Entity> availableEntities =
FluentIterable.from(getEntities().getMembers())
.filter(new HasHostnameAndValidLifecycle());
LOG.debug("{} updating with entities: {}", this, Iterables.toString(availableEntities));
ImmutableListMultimap<String, Entity> hostnameToEntity =
Multimaps.index(availableEntities, new HostnameTransformer());
Map<String, String> octetToName = Maps.newHashMap();
BiMap<String, String> ipToARecord = HashBiMap.create();
Multimap<String, String> aRecordToCnames = MultimapBuilder.hashKeys().hashSetValues().build();
Multimap<String, String> ipToAllNames = MultimapBuilder.hashKeys().hashSetValues().build();
for (Map.Entry<String, Entity> e : hostnameToEntity.entries()) {
String domainName = e.getKey();
Maybe<SshMachineLocation> location =
Machines.findUniqueSshMachineLocation(e.getValue().getLocations());
if (!location.isPresent()) {
LOG.debug(
"Member {} of {} does not have an SSH location so will not be configured",
e.getValue(),
this);
continue;
} else if (ipToARecord.inverse().containsKey(domainName)) {
continue;
}
String address = location.get().getAddress().getHostAddress();
ipToAllNames.put(address, domainName);
if (!ipToARecord.containsKey(address)) {
ipToARecord.put(address, domainName);
if (getReverseLookupNetwork().contains(new Cidr(address + "/32"))) {
String octet = Iterables.get(Splitter.on('.').split(address), 3);
if (!octetToName.containsKey(octet)) octetToName.put(octet, domainName);
}
} else {
aRecordToCnames.put(ipToARecord.get(address), domainName);
}
}
setAttribute(A_RECORDS, ImmutableMap.copyOf(ipToARecord.inverse()));
setAttribute(PTR_RECORDS, ImmutableMap.copyOf(octetToName));
setAttribute(CNAME_RECORDS, Multimaps.unmodifiableMultimap(aRecordToCnames));
setAttribute(ADDRESS_MAPPINGS, Multimaps.unmodifiableMultimap(ipToAllNames));
// Update Bind configuration files and restart the service
getDriver().updateBindConfiguration();
}
}
public static HostAndPort getBrooklynAccessibleAddress(Entity entity, int port) {
String host;
// look up port forwarding
PortForwardManager pfw = entity.getConfig(PORT_FORWARDING_MANAGER);
if (pfw != null) {
Collection<Location> ll = entity.getLocations();
synchronized (BrooklynAccessUtils.class) {
// TODO finer-grained synchronization
for (MachineLocation machine : Iterables.filter(ll, MachineLocation.class)) {
HostAndPort hp = pfw.lookup(machine, port);
if (hp != null) {
log.debug(
"BrooklynAccessUtils found port-forwarded address {} for entity {}, port {}, using machine {}",
new Object[] {hp, entity, port, machine});
return hp;
}
}
Maybe<SupportsPortForwarding> supportPortForwardingLoc =
Machines.findUniqueElement(ll, SupportsPortForwarding.class);
if (supportPortForwardingLoc.isPresent()) {
Cidr source = entity.getConfig(MANAGEMENT_ACCESS_CIDR);
SupportsPortForwarding loc = supportPortForwardingLoc.get();
if (source != null) {
log.debug(
"BrooklynAccessUtils requesting new port-forwarding rule to access "
+ port
+ " on "
+ entity
+ " (at "
+ loc
+ ", enabled for "
+ source
+ ")");
// TODO discuss, is this the best way to do it
// (will probably _create_ the port forwarding rule!)
HostAndPort hp = loc.getSocketEndpointFor(source, port);
if (hp != null) {
log.debug(
"BrooklynAccessUtils created port-forwarded address {} for entity {}, port {}, using {}",
new Object[] {hp, entity, port, loc});
return hp;
}
} else {
log.warn(
"No "
+ MANAGEMENT_ACCESS_CIDR.getName()
+ " configured for "
+ entity
+ ", so cannot forward "
+ "port "
+ port
+ " "
+ "even though "
+ PORT_FORWARDING_MANAGER.getName()
+ " was supplied, and "
+ "have location supporting port forwarding "
+ loc);
}
}
}
}
host = entity.getAttribute(Attributes.HOSTNAME);
if (host != null) return HostAndPort.fromParts(host, port);
throw new IllegalStateException(
"Cannot find way to access port "
+ port
+ " on "
+ entity
+ " from Brooklyn (no host.name)");
}
protected void doStop(ConfigBag parameters, Callable<StopMachineDetails<Integer>> stopTask) {
preStopConfirmCustom();
log.info("Stopping {} in {}", entity(), entity().getLocations());
StopMode stopMachineMode = getStopMachineMode(parameters);
StopMode stopProcessMode = parameters.get(StopSoftwareParameters.STOP_PROCESS_MODE);
DynamicTasks.queue("pre-stop", new PreStopCustomTask());
// BROOKLYN-263:
// With this change the stop effector will wait for Location to provision so it can terminate
// the machine, if a provisioning request is in-progress.
//
// The ProvisionMachineTask stores transient internal state in PROVISIONING_TASK_STATE and
// PROVISIONED_MACHINE: it records when the provisioning is running and when done; and it
// records the final machine. We record the machine in the internal sensor (rather than
// just relying on getLocations) because the latter is set much later in the start()
// process.
//
// This code is a big improvement (previously there was a several-minute window in some
// clouds where a call to stop() would leave the machine running).
//
// However, there are still races. If the start() code has not yet reached the call to
// location.obtain() then we won't wait, and the start() call won't know to abort. It's
// fiddly to get that right, because we need to cope with restart() - so we mustn't leave
// any state behind that will interfere with subsequent sequential calls to start().
// There is some attempt to handle it by ProvisionMachineTask checking if the expectedState
// is stopping/stopped.
Maybe<MachineLocation> machine = Machines.findUniqueMachineLocation(entity().getLocations());
ProvisioningTaskState provisioningState =
entity().sensors().get(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
if (machine.isAbsent() && provisioningState == ProvisioningTaskState.RUNNING) {
Duration maxWait = entity().config().get(STOP_WAIT_PROVISIONING_TIMEOUT);
log.info(
"When stopping {}, waiting for up to {} for the machine to finish provisioning, before terminating it",
entity(),
maxWait);
boolean success =
Repeater.create("Wait for a machine to appear")
.until(
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
ProvisioningTaskState state =
entity()
.sensors()
.get(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
return (state != ProvisioningTaskState.RUNNING);
}
})
.backoffTo(Duration.FIVE_SECONDS)
.limitTimeTo(maxWait)
.run();
if (!success) {
log.warn(
"When stopping {}, timed out after {} waiting for the machine to finish provisioning - machine may we left running",
entity(),
maxWait);
}
machine = Maybe.ofDisallowingNull(entity().sensors().get(INTERNAL_PROVISIONED_MACHINE));
}
entity().sensors().remove(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
entity().sensors().remove(INTERNAL_PROVISIONED_MACHINE);
Task<List<?>> stoppingProcess = null;
if (canStop(stopProcessMode, entity())) {
stoppingProcess =
Tasks.parallel(
"stopping",
Tasks.create("stopping (process)", new StopProcessesAtMachineTask()),
Tasks.create("stopping (feeds)", new StopFeedsAtMachineTask()));
DynamicTasks.queue(stoppingProcess);
}
Task<StopMachineDetails<Integer>> stoppingMachine = null;
if (canStop(stopMachineMode, machine.isAbsent())) {
// Release this machine (even if error trying to stop process - we rethrow that after)
Map<String, Object> stopMachineFlags = MutableMap.of();
if (Entitlements.getEntitlementContext() != null) {
stopMachineFlags.put(
"tags",
MutableSet.of(
BrooklynTaskTags.tagForEntitlement(Entitlements.getEntitlementContext())));
}
Task<StopMachineDetails<Integer>> stopMachineTask =
Tasks.<StopMachineDetails<Integer>>builder()
.displayName("stopping (machine)")
.body(stopTask)
.flags(stopMachineFlags)
.build();
stoppingMachine = DynamicTasks.queue(stopMachineTask);
DynamicTasks.drain(entity().getConfig(STOP_PROCESS_TIMEOUT), false);
// shutdown the machine if stopping process fails or takes too long
synchronized (stoppingMachine) {
// task also used as mutex by DST when it submits it; ensure it only submits once!
if (!stoppingMachine.isSubmitted()) {
// force the stoppingMachine task to run by submitting it here
StringBuilder msg =
new StringBuilder("Submitting machine stop early in background for ")
.append(entity());
if (stoppingProcess == null) {
msg.append(". Process stop skipped, pre-stop not finished?");
} else {
msg.append(" because process stop has ")
.append((stoppingProcess.isDone() ? "finished abnormally" : "not finished"));
}
log.warn(msg.toString());
Entities.submit(entity(), stoppingMachine);
}
}
}
try {
// This maintains previous behaviour of silently squashing any errors on the stoppingProcess
// task if the
// stoppingMachine exits with a nonzero value
boolean checkStopProcesses =
(stoppingProcess != null
&& (stoppingMachine == null || stoppingMachine.get().value == 0));
if (checkStopProcesses) {
// TODO we should test for destruction above, not merely successful "stop", as things like
// localhost and ssh won't be destroyed
DynamicTasks.waitForLast();
if (machine.isPresent()) {
// throw early errors *only if* there is a machine and we have not destroyed it
stoppingProcess.get();
}
}
} catch (Throwable e) {
ServiceStateLogic.setExpectedState(entity(), Lifecycle.ON_FIRE);
Exceptions.propagate(e);
}
entity().sensors().set(SoftwareProcess.SERVICE_UP, false);
ServiceStateLogic.setExpectedState(entity(), Lifecycle.STOPPED);
DynamicTasks.queue("post-stop", new PostStopCustomTask());
if (log.isDebugEnabled()) log.debug("Stopped software process entity " + entity());
}
public void run() {
log.info("Starting {} on machine {}", entity(), machine);
Collection<Location> oldLocs = entity().getLocations();
if (!oldLocs.isEmpty()) {
List<MachineLocation> oldSshLocs =
ImmutableList.copyOf(Iterables.filter(oldLocs, MachineLocation.class));
if (!oldSshLocs.isEmpty()) {
// check if existing locations are compatible
log.debug(
"Entity "
+ entity()
+ " had machine locations "
+ oldSshLocs
+ " when starting at "
+ machine
+ "; checking if they are compatible");
for (MachineLocation oldLoc : oldSshLocs) {
// machines are deemed compatible if hostname and address are the same, or they are
// localhost
// this allows a machine create by jclouds to then be defined with an ip-based spec
if (!"localhost".equals(machine.getConfig(AbstractLocation.ORIGINAL_SPEC))) {
checkLocationParametersCompatible(
machine,
oldLoc,
"hostname",
oldLoc.getAddress().getHostName(),
machine.getAddress().getHostName());
checkLocationParametersCompatible(
machine,
oldLoc,
"address",
oldLoc.getAddress().getHostAddress(),
machine.getAddress().getHostAddress());
}
}
log.debug(
"Entity "
+ entity()
+ " old machine locations "
+ oldSshLocs
+ " were compatible, removing them to start at "
+ machine);
entity().removeLocations(oldSshLocs);
}
}
entity().addLocations(ImmutableList.of((Location) machine));
// elsewhere we rely on (public) hostname being set _after_ subnet_hostname
// (to prevent the tiny possibility of races resulting in hostname being returned
// simply because subnet is still being looked up)
Maybe<String> lh = Machines.getSubnetHostname(machine);
Maybe<String> la = Machines.getSubnetIp(machine);
if (lh.isPresent()) entity().sensors().set(Attributes.SUBNET_HOSTNAME, lh.get());
if (la.isPresent()) entity().sensors().set(Attributes.SUBNET_ADDRESS, la.get());
entity().sensors().set(Attributes.HOSTNAME, machine.getAddress().getHostName());
entity().sensors().set(Attributes.ADDRESS, machine.getAddress().getHostAddress());
if (machine instanceof SshMachineLocation) {
SshMachineLocation sshMachine = (SshMachineLocation) machine;
UserAndHostAndPort sshAddress =
UserAndHostAndPort.fromParts(
sshMachine.getUser(), sshMachine.getAddress().getHostName(), sshMachine.getPort());
// FIXME: Who or what is SSH_ADDRESS intended for? It's not necessarily the address that
// the SshMachineLocation is using for ssh connections (because it accepts SSH_HOST as an
// override).
entity().sensors().set(Attributes.SSH_ADDRESS, sshAddress);
}
if (Boolean.TRUE.equals(entity().getConfig(SoftwareProcess.OPEN_IPTABLES))) {
if (machine instanceof SshMachineLocation) {
@SuppressWarnings("unchecked")
Iterable<Integer> inboundPorts =
(Iterable<Integer>) machine.config().get(CloudLocationConfig.INBOUND_PORTS);
machineInitTasks.openIptablesAsync(inboundPorts, (SshMachineLocation) machine);
} else {
log.warn("Ignoring flag OPEN_IPTABLES on non-ssh location {}", machine);
}
}
if (Boolean.TRUE.equals(entity().getConfig(SoftwareProcess.STOP_IPTABLES))) {
if (machine instanceof SshMachineLocation) {
machineInitTasks.stopIptablesAsync((SshMachineLocation) machine);
} else {
log.warn("Ignoring flag STOP_IPTABLES on non-ssh location {}", machine);
}
}
if (Boolean.TRUE.equals(entity().getConfig(SoftwareProcess.DONT_REQUIRE_TTY_FOR_SUDO))) {
if (machine instanceof SshMachineLocation) {
machineInitTasks.dontRequireTtyForSudoAsync((SshMachineLocation) machine);
} else {
log.warn("Ignoring flag DONT_REQUIRE_TTY_FOR_SUDO on non-ssh location {}", machine);
}
}
resolveOnBoxDir(entity(), machine);
preStartCustom(machine);
}
@SuppressWarnings({"rawtypes", "unchecked"})
@Override
public V call() {
T value = source.getAttribute(sensor);
// return immediately if either the ready predicate or the abort conditions hold
if (ready(value)) return postProcess(value);
final List<Exception> abortionExceptions = Lists.newCopyOnWriteArrayList();
long start = System.currentTimeMillis();
for (AttributeAndSensorCondition abortCondition : abortSensorConditions) {
Object abortValue = abortCondition.source.getAttribute(abortCondition.sensor);
if (abortCondition.predicate.apply(abortValue)) {
abortionExceptions.add(
new Exception(
"Abort due to " + abortCondition.source + " -> " + abortCondition.sensor));
}
}
if (abortionExceptions.size() > 0) {
throw new CompoundRuntimeException(
"Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
}
TaskInternal<?> current = (TaskInternal<?>) Tasks.current();
if (current == null)
throw new IllegalStateException("Should only be invoked in a running task");
Entity entity = BrooklynTaskTags.getTargetOrContextEntity(current);
if (entity == null)
throw new IllegalStateException(
"Should only be invoked in a running task with an entity tag; "
+ current
+ " has no entity tag ("
+ current.getStatusDetail(false)
+ ")");
final LinkedList<T> publishedValues = new LinkedList<T>();
final Semaphore semaphore = new Semaphore(0); // could use Exchanger
SubscriptionHandle subscription = null;
List<SubscriptionHandle> abortSubscriptions = Lists.newArrayList();
try {
subscription =
entity
.subscriptions()
.subscribe(
source,
sensor,
new SensorEventListener<T>() {
@Override
public void onEvent(SensorEvent<T> event) {
synchronized (publishedValues) {
publishedValues.add(event.getValue());
}
semaphore.release();
}
});
for (final AttributeAndSensorCondition abortCondition : abortSensorConditions) {
abortSubscriptions.add(
entity
.subscriptions()
.subscribe(
abortCondition.source,
abortCondition.sensor,
new SensorEventListener<Object>() {
@Override
public void onEvent(SensorEvent<Object> event) {
if (abortCondition.predicate.apply(event.getValue())) {
abortionExceptions.add(
new Exception(
"Abort due to "
+ abortCondition.source
+ " -> "
+ abortCondition.sensor));
semaphore.release();
}
}
}));
Object abortValue = abortCondition.source.getAttribute(abortCondition.sensor);
if (abortCondition.predicate.apply(abortValue)) {
abortionExceptions.add(
new Exception(
"Abort due to " + abortCondition.source + " -> " + abortCondition.sensor));
}
}
if (abortionExceptions.size() > 0) {
throw new CompoundRuntimeException(
"Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
}
CountdownTimer timer = timeout != null ? timeout.countdownTimer() : null;
Duration maxPeriod = ValueResolver.PRETTY_QUICK_WAIT;
Duration nextPeriod = ValueResolver.REAL_QUICK_PERIOD;
while (true) {
// check the source on initial run (could be done outside the loop)
// and also (optionally) on each iteration in case it is more recent
value = source.getAttribute(sensor);
if (ready(value)) break;
if (timer != null) {
if (timer.getDurationRemaining().isShorterThan(nextPeriod)) {
nextPeriod = timer.getDurationRemaining();
}
if (timer.isExpired()) {
if (onTimeout.isPresent()) return onTimeout.get();
throw new RuntimeTimeoutException("Unsatisfied after " + Duration.sinceUtc(start));
}
}
String prevBlockingDetails = current.setBlockingDetails(blockingDetails);
try {
if (semaphore.tryAcquire(nextPeriod.toMilliseconds(), TimeUnit.MILLISECONDS)) {
// immediately release so we are available for the next check
semaphore.release();
// if other permits have been made available (e.g. multiple notifications) drain them
// all as no point running multiple times
semaphore.drainPermits();
}
} finally {
current.setBlockingDetails(prevBlockingDetails);
}
// check any subscribed values which have come in first
while (true) {
synchronized (publishedValues) {
if (publishedValues.isEmpty()) break;
value = publishedValues.pop();
}
if (ready(value)) break;
}
// if unmanaged then ignore the other abort conditions
if (!ignoreUnmanaged && Entities.isNoLongerManaged(entity)) {
if (onUnmanaged.isPresent()) return onUnmanaged.get();
throw new NotManagedException(entity);
}
if (abortionExceptions.size() > 0) {
throw new CompoundRuntimeException(
"Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
}
nextPeriod = nextPeriod.times(2).upperBound(maxPeriod);
}
if (LOG.isDebugEnabled()) LOG.debug("Attribute-ready for {} in entity {}", sensor, source);
return postProcess(value);
} catch (InterruptedException e) {
throw Exceptions.propagate(e);
} finally {
if (subscription != null) {
entity.subscriptions().unsubscribe(subscription);
}
for (SubscriptionHandle handle : abortSubscriptions) {
entity.subscriptions().unsubscribe(handle);
}
}
}
@Override
@Deprecated
public Maybe<Object> getConfigRaw(ConfigKey<?> key, boolean includeInherited) {
Maybe<Object> result = delegate.getConfigRaw(key, includeInherited);
return (result.isPresent()) ? Maybe.of(transform(key, result.get())) : Maybe.absent();
}