@Override
public String execCommandTimeout(String command, Duration timeout) {
ProcessTaskWrapper<String> task =
SshEffectorTasks.ssh(command)
.environmentVariables(
((AbstractSoftwareProcessSshDriver) getDriver()).getShellEnvironment())
.configure(SshTool.PROP_ALLOCATE_PTY, true) // TODO configure globally
.requiringZeroAndReturningStdout()
.machine(getMachine())
.summary(command)
.newTask();
try {
String result =
DynamicTasks.queueIfPossible(task)
.executionContext(this)
.orSubmitAsync()
.asTask()
.get(timeout);
return result;
} catch (TimeoutException te) {
throw new IllegalStateException("Timed out running command: " + command);
} catch (Exception e) {
Integer exitCode = task.getExitCode();
LOG.warn(
"Command failed, return code {}: {}", exitCode == null ? -1 : exitCode, task.getStderr());
throw Exceptions.propagate(e);
}
}
private void copy(String src, File dst) {
try {
copy(new ReaderInputStream(new StringReader(src), "UTF-8"), dst);
} catch (Exception e) {
throw Exceptions.propagate(e);
}
}
/**
* as {@link #waitOnForExpiry(Object)} but catches and wraps InterruptedException as unchecked
* RuntimeInterruptedExcedption
*/
public boolean waitOnForExpiryUnchecked(Object waitTarget) {
try {
return waitOnForExpiry(waitTarget);
} catch (InterruptedException e) {
throw Exceptions.propagate(e);
}
}
private Response launch(String yaml, EntitySpec<? extends Application> spec) {
try {
Application app = EntityManagementUtils.createUnstarted(mgmt(), spec);
CreationResult<Application, Void> result = EntityManagementUtils.start(app);
boolean isEntitled =
Entitlements.isEntitled(
mgmt().getEntitlementManager(),
Entitlements.INVOKE_EFFECTOR,
EntityAndItem.of(app, StringAndArgument.of(Startable.START.getName(), null)));
if (!isEntitled) {
throw WebResourceUtils.unauthorized(
"User '%s' is not authorized to start application %s",
Entitlements.getEntitlementContext().user(), spec.getType());
}
log.info("Launched from YAML: " + yaml + " -> " + app + " (" + result.task() + ")");
URI ref = URI.create(app.getApplicationId());
ResponseBuilder response = created(ref);
if (result.task() != null) response.entity(TaskTransformer.FROM_TASK.apply(result.task()));
return response.build();
} catch (ConstraintViolationException e) {
throw new UserFacingException(e);
} catch (Exception e) {
throw Exceptions.propagate(e);
}
}
@Override
public EntitySpec<? extends Application> createApplicationSpec(String plan) {
try {
CampPlatform camp = CampCatalogUtils.getCampPlatform(mgmt);
BrooklynClassLoadingContext loader = JavaBrooklynClassLoadingContext.create(mgmt);
AssemblyTemplate at = CampUtils.registerDeploymentPlan(plan, loader, camp);
AssemblyTemplateInstantiator instantiator = CampUtils.getInstantiator(at);
if (instantiator instanceof AssemblyTemplateSpecInstantiator) {
return ((AssemblyTemplateSpecInstantiator) instantiator)
.createApplicationSpec(at, camp, loader);
} else {
// The unknown instantiator can create the app (Assembly), but not a spec.
// Currently, all brooklyn plans should produce the above.
if (at.getPlatformComponentTemplates() == null
|| at.getPlatformComponentTemplates().isEmpty()) {
if (at.getCustomAttributes().containsKey("brooklyn.catalog"))
throw new IllegalArgumentException(
"Unrecognized application blueprint format: expected an application, not a brooklyn.catalog");
throw new PlanNotRecognizedException(
"Unrecognized application blueprint format: no services defined");
}
// map this (expected) error to a nicer message
throw new PlanNotRecognizedException("Unrecognized application blueprint format");
}
} catch (Exception e) {
// TODO how do we figure out that the plan is not supported vs. invalid to wrap in a
// PlanNotRecognizedException?
if (log.isDebugEnabled()) log.debug("Failed to create entity from CAMP spec:\n" + plan, e);
throw Exceptions.propagate(e);
}
}
@Override
public final T get() {
if (log.isDebugEnabled())
log.debug("Queuing task to resolve " + dsl + ", called by " + Tasks.current());
EntityInternal entity =
(EntityInternal) BrooklynTaskTags.getTargetOrContextEntity(Tasks.current());
ExecutionContext exec =
(entity != null)
? entity.getExecutionContext()
: BasicExecutionContext.getCurrentExecutionContext();
if (exec == null) {
throw new IllegalStateException("No execution context available to resolve " + dsl);
}
Task<T> task = newTask();
T result;
try {
result = exec.submit(task).get();
} catch (InterruptedException | ExecutionException e) {
Task<?> currentTask = Tasks.current();
if (currentTask != null && currentTask.isCancelled()) {
task.cancel(true);
}
throw Exceptions.propagate(e);
}
if (log.isDebugEnabled()) log.debug("Resolved " + result + " from " + dsl);
return result;
}
/**
* Runs the given callable. Repeats until the operation succeeds or {@link #isExceptionRetryable}
* indicates that the request cannot be retried.
*/
protected <T> T runOperationWithRetry(Callable<T> operation) {
int backoff = 64;
Exception lastException = null;
for (int retries = 0; retries < 100; retries++) {
try {
return operation.call();
} catch (Exception e) {
lastException = e;
if (isExceptionRetryable.apply(e)) {
LOG.debug("Attempt #{} failed to add security group: {}", retries + 1, e.getMessage());
try {
Thread.sleep(backoff);
} catch (InterruptedException e1) {
throw Exceptions.propagate(e1);
}
backoff = backoff << 1;
} else {
break;
}
}
}
throw new RuntimeException(
"Unable to add security group rule; repeated errors from provider", lastException);
}
@Override
public void doStart(Collection<? extends Location> locs) {
List<Location> locations = MutableList.of();
sensors().set(SERVICE_UP, Boolean.FALSE);
ServiceStateLogic.setExpectedState(this, Lifecycle.STARTING);
LOG.info("Creating new MesosLocation");
createLocation(MutableMap.<String, Object>of());
// Start frameworks
try {
Group frameworks = sensors().get(MESOS_FRAMEWORKS);
Entities.invokeEffectorList(
this,
frameworks.getMembers(),
Startable.START,
ImmutableMap.of("locations", locations))
.getUnchecked();
} catch (Exception e) {
LOG.warn("Error starting frameworks", e);
ServiceStateLogic.setExpectedState(this, Lifecycle.ON_FIRE);
Exceptions.propagate(e);
}
super.doStart(locations);
connectSensors();
ServiceStateLogic.setExpectedState(this, Lifecycle.RUNNING);
sensors().set(SERVICE_UP, Boolean.TRUE);
}
protected void onUpdated() {
try {
emit(targetSensor, compute());
} catch (Throwable t) {
LOG.warn("Error calculating and setting aggregate for enricher " + this, t);
throw Exceptions.propagate(t);
}
}
public void waitForImage(String imageName) {
try {
CountDownLatch latch = images.get(imageName);
if (latch != null) latch.await(15, TimeUnit.MINUTES);
} catch (InterruptedException ie) {
throw Exceptions.propagate(ie);
}
}
@Override
public InetAddress getAddress() {
String address = getOwner().sensors().get(Attributes.ADDRESS);
try {
return InetAddress.getByName(address);
} catch (UnknownHostException e) {
throw Exceptions.propagate(e);
}
}
private void copy(InputStream in, File tmpJar) {
try {
OutputStream out = new FileOutputStream(tmpJar);
ByteStreams.copy(in, out);
Streams.closeQuietly(in);
Streams.closeQuietly(out);
} catch (Exception e) {
throw Exceptions.propagate(e);
}
}
/**
* runs the tasks needed to start, wrapped by setting {@link Attributes#SERVICE_STATE_EXPECTED}
* appropriately
*/
public void start(Collection<? extends Location> locations) {
ServiceStateLogic.setExpectedState(entity(), Lifecycle.STARTING);
try {
startInLocations(locations);
DynamicTasks.waitForLast();
ServiceStateLogic.setExpectedState(entity(), Lifecycle.RUNNING);
} catch (Throwable t) {
ServiceStateLogic.setExpectedState(entity(), Lifecycle.ON_FIRE);
throw Exceptions.propagate(t);
}
}
@Override
public InetAddress getNextAgentAddress(String agentId) {
Entity agent = getManagementContext().getEntityManager().getEntity(agentId);
String address =
agent.sensors().get(CalicoNode.DOCKER_HOST).sensors().get(Attributes.SUBNET_ADDRESS);
try {
return InetAddress.getByName(address);
} catch (UnknownHostException uhe) {
throw Exceptions.propagate(uhe);
}
}
@Override
public void close() throws IOException {
LOG.info("Close called on Docker host {}: {}", machine, this);
try {
machine.close();
} catch (Exception e) {
LOG.info("{}: Closing Docker host: {}", e.getMessage(), this);
throw Exceptions.propagate(e);
} finally {
LOG.info("Docker host closed: {}", this);
}
}
@Override
public void close() throws IOException {
LOG.debug("Close called on Marathon task: {}", this);
try {
if (marathonTask.sensors().get(MarathonTask.SERVICE_UP)) {
LOG.info("Stopping Marathon task entity for {}: {}", this, marathonTask);
marathonTask.stop();
}
LOG.info("Marathon task closed: {}", this);
} catch (Exception e) {
LOG.warn("Error closing Marathon task {}: {}", this, e.getMessage());
throw Exceptions.propagate(e);
}
}
@Override
public void onManagementStarting() {
super.onManagementStarting();
sensors().set(TOPIC_NAME, getName());
try {
String virtualHost = getParent().getVirtualHost();
exchange =
new ObjectName(
format(
"org.apache.qpid:type=VirtualHost.Exchange,VirtualHost=\"%s\",name=\"%s\",ExchangeType=topic",
virtualHost, getExchangeName()));
} catch (MalformedObjectNameException e) {
throw Exceptions.propagate(e);
}
}
/** builds remote keystores, stores config keys/certs, and copies necessary files across */
public void install() {
try {
// build truststore and keystore
FluentKeySigner signer = getBrooklynRootSigner();
KeyPair jmxAgentKey = SecureKeys.newKeyPair();
X509Certificate jmxAgentCert = signer.newCertificateFor("jmxmp-agent", jmxAgentKey);
agentKeyStore = SecureKeys.newKeyStore();
agentKeyStore.setKeyEntry(
"jmxmp-agent",
jmxAgentKey.getPrivate(),
// TODO jmx.ssl.agent.keyPassword
"".toCharArray(),
new Certificate[] {jmxAgentCert});
ByteArrayOutputStream agentKeyStoreBytes = new ByteArrayOutputStream();
agentKeyStore.store(
agentKeyStoreBytes,
// TODO jmx.ssl.agent.keyStorePassword
"".toCharArray());
agentTrustStore = SecureKeys.newKeyStore();
agentTrustStore.setCertificateEntry("brooklyn", getJmxAccessCert());
ByteArrayOutputStream agentTrustStoreBytes = new ByteArrayOutputStream();
agentTrustStore.store(agentTrustStoreBytes, "".toCharArray());
// install the truststore and keystore and rely on JmxSupport to install the agent
Tasks.setBlockingDetails("Copying keystore and truststore to the server.");
try {
jmxSupport
.getMachine()
.get()
.copyTo(
new ByteArrayInputStream(agentKeyStoreBytes.toByteArray()),
getJmxSslKeyStoreFilePath());
jmxSupport
.getMachine()
.get()
.copyTo(
new ByteArrayInputStream(agentTrustStoreBytes.toByteArray()),
getJmxSslTrustStoreFilePath());
} finally {
Tasks.resetBlockingDetails();
}
} catch (Exception e) {
throw Exceptions.propagate(e);
}
}
private EntitySpec<? extends Application> createEntitySpecForApplication(String potentialYaml) {
try {
return EntityManagementUtils.createEntitySpecForApplication(mgmt(), potentialYaml);
} catch (Exception e) {
// An IllegalArgumentException for creating the entity spec gets wrapped in a ISE, and
// possibly a Compound.
// But we want to return a 400 rather than 500, so ensure we throw IAE.
IllegalArgumentException iae =
(IllegalArgumentException)
Exceptions.getFirstThrowableOfType(e, IllegalArgumentException.class);
if (iae != null) {
throw new IllegalArgumentException("Cannot create spec for app: " + iae.getMessage(), e);
} else {
throw Exceptions.propagate(e);
}
}
}
@Override
public List<Application> rebind(
ClassLoader classLoaderO,
RebindExceptionHandler exceptionHandlerO,
ManagementNodeState modeO) {
final ClassLoader classLoader =
classLoaderO != null ? classLoaderO : managementContext.getCatalogClassLoader();
final RebindExceptionHandler exceptionHandler =
exceptionHandlerO != null
? exceptionHandlerO
: RebindExceptionHandlerImpl.builder()
.danglingRefFailureMode(danglingRefFailureMode)
.danglingRefQuorumRequiredHealthy(danglingRefsQuorumRequiredHealthy)
.rebindFailureMode(rebindFailureMode)
.addConfigFailureMode(addConfigFailureMode)
.addPolicyFailureMode(addPolicyFailureMode)
.loadPolicyFailureMode(loadPolicyFailureMode)
.build();
final ManagementNodeState mode = modeO != null ? modeO : getRebindMode();
if (mode != ManagementNodeState.MASTER
&& mode != ManagementNodeState.HOT_STANDBY
&& mode != ManagementNodeState.HOT_BACKUP)
throw new IllegalStateException(
"Must be either master or hot standby/backup to rebind (mode " + mode + ")");
ExecutionContext ec = BasicExecutionContext.getCurrentExecutionContext();
if (ec == null) {
ec = managementContext.getServerExecutionContext();
Task<List<Application>> task =
ec.submit(
new Callable<List<Application>>() {
@Override
public List<Application> call() throws Exception {
return rebindImpl(classLoader, exceptionHandler, mode);
}
});
try {
return task.get();
} catch (Exception e) {
throw Exceptions.propagate(e);
}
} else {
return rebindImpl(classLoader, exceptionHandler, mode);
}
}
private <T> T resolve(ConfigKey<T> key, Object value) {
Object transformed = transform(key, value);
Object result;
if (transformed instanceof DeferredSupplier) {
ExecutionContext exec = mgmt.getServerExecutionContext();
try {
result = Tasks.resolveValue(transformed, key.getType(), exec);
} catch (ExecutionException | InterruptedException e) {
throw Exceptions.propagate(e);
}
} else {
result = transformed;
}
return TypeCoercions.coerce(result, key.getTypeToken());
}
@SuppressWarnings({"unchecked", "rawtypes", "deprecation"})
protected void startWithKnifeAsync() {
// TODO prestart, ports (as above); also, note, some aspects of this are untested as we need a
// chef server
String primary = getPrimaryCookbook();
// put all config under brooklyn/cookbook/config
Navigator<MutableMap<Object, Object>> attrs = Jsonya.newInstancePrimitive().at("brooklyn");
if (Strings.isNonBlank(primary)) attrs.at(primary);
attrs.at("config");
attrs.put(entity().getAllConfigBag().getAllConfig());
// and put launch attrs at root
try {
attrs
.root()
.put(
(Map<?, ?>)
Tasks.resolveDeepValue(
entity().getConfig(CHEF_LAUNCH_ATTRIBUTES),
Object.class,
entity().getExecutionContext()));
} catch (Exception e) {
Exceptions.propagate(e);
}
Collection<? extends String> runList = entity().getConfig(CHEF_LAUNCH_RUN_LIST);
if (runList == null) runList = entity().getConfig(CHEF_RUN_LIST);
if (runList == null) {
if (Strings.isNonBlank(primary)) runList = ImmutableList.of(primary + "::" + "start");
else
throw new IllegalStateException(
"Require a primary cookbook or a run_list to effect " + "start" + " on " + entity());
}
DynamicTasks.queue(
ChefServerTasks.knifeConvergeTask()
.knifeNodeName(getNodeName())
.knifeRunList(Strings.join(runList, ","))
.knifeAddAttributes((Map<? extends Object, ? extends Object>) (Map) attrs.root().get())
.knifeRunTwice(entity().getConfig(CHEF_RUN_CONVERGE_TWICE)));
}
@SuppressWarnings({"unchecked", "rawtypes"})
protected static EffectorSummary.ParameterSummary<?> parameterSummary(
Entity entity, ParameterType<?> parameterType) {
try {
Maybe<?> defaultValue =
Tasks.resolving(parameterType.getDefaultValue())
.as(parameterType.getParameterClass())
.context(entity)
.timeout(ValueResolver.REAL_QUICK_WAIT)
.getMaybe();
return new ParameterSummary(
parameterType.getName(),
parameterType.getParameterClassName(),
parameterType.getDescription(),
WebResourceUtils.getValueForDisplay(defaultValue.orNull(), true, false),
Sanitizer.IS_SECRET_PREDICATE.apply(parameterType.getName()));
} catch (Exception e) {
throw Exceptions.propagate(e);
}
}
@Override
public int copyResource(
Map<Object, Object> sshFlags, String source, String target, boolean createParentDir) {
if (createParentDir) {
createDirectory(getDirectory(target), "Creating resource directory");
}
InputStream stream = null;
try {
Tasks.setBlockingDetails("retrieving resource " + source + " for copying across");
stream = resource.getResourceFromUrl(source);
Tasks.setBlockingDetails("copying resource " + source + " to server");
return copyTo(stream, target);
} catch (Exception e) {
throw Exceptions.propagate(e);
} finally {
Tasks.setBlockingDetails(null);
if (stream != null) Streams.closeQuietly(stream);
}
}
@Override
public void customize() {
// Some OSes start postgres during package installation
DynamicTasks.queue(
SshEffectorTasks.ssh(sudoAsUser("postgres", "/etc/init.d/postgresql stop"))
.allowingNonZeroExitCode())
.get();
newScript(CUSTOMIZING)
.body
.append(
sudo("mkdir -p " + getDataDir()),
sudo("chown postgres:postgres " + getDataDir()),
sudo("chmod 700 " + getDataDir()),
sudo("touch " + getLogFile()),
sudo("chown postgres:postgres " + getLogFile()),
sudo("touch " + getPidFile()),
sudo("chown postgres:postgres " + getPidFile()),
alternativesGroup(
chainGroup(
format("test -e %s", getInstallDir() + "/bin/initdb"),
sudoAsUser("postgres", getInstallDir() + "/bin/initdb -D " + getDataDir())),
callPgctl("initdb", true)))
.failOnNonZeroResultCode()
.execute();
String configUrl = getEntity().getConfig(PostgreSqlNode.CONFIGURATION_FILE_URL);
if (Strings.isBlank(configUrl)) {
// http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server
// If the same setting is listed multiple times, the last one wins.
DynamicTasks.queue(
SshEffectorTasks.ssh(
executeCommandThenAsUserTeeOutputToFile(
chainGroup(
"echo \"listen_addresses = '*'\"",
"echo \"port = " + getEntity().getPostgreSqlPort() + "\"",
"echo \"max_connections = " + getEntity().getMaxConnections() + "\"",
"echo \"shared_buffers = " + getEntity().getSharedMemory() + "\"",
"echo \"external_pid_file = '" + getPidFile() + "'\""),
"postgres",
getDataDir() + "/postgresql.conf")));
} else {
String contents = processTemplate(configUrl);
DynamicTasks.queue(
SshEffectorTasks.put("/tmp/postgresql.conf").contents(contents),
SshEffectorTasks.ssh(
sudoAsUser(
"postgres", "cp /tmp/postgresql.conf " + getDataDir() + "/postgresql.conf")));
}
String authConfigUrl =
getEntity().getConfig(PostgreSqlNode.AUTHENTICATION_CONFIGURATION_FILE_URL);
if (Strings.isBlank(authConfigUrl)) {
DynamicTasks.queue(
SshEffectorTasks.ssh(
// TODO give users control which hosts can connect and the authentication mechanism
executeCommandThenAsUserTeeOutputToFile(
"echo \"host all all 0.0.0.0/0 md5\"",
"postgres",
getDataDir() + "/pg_hba.conf")));
} else {
String contents = processTemplate(authConfigUrl);
DynamicTasks.queue(
SshEffectorTasks.put("/tmp/pg_hba.conf").contents(contents),
SshEffectorTasks.ssh(
sudoAsUser("postgres", "cp /tmp/pg_hba.conf " + getDataDir() + "/pg_hba.conf")));
}
// Wait for commands to complete before running the creation script
DynamicTasks.waitForLast();
if (Boolean.TRUE.equals(entity.getConfig(PostgreSqlNode.INITIALIZE_DB))) {
initializeNewDatabase();
}
// Capture log file contents if there is an error configuring the database
try {
executeDatabaseCreationScript();
} catch (RuntimeException r) {
logTailOfPostgresLog();
throw Exceptions.propagate(r);
}
// Try establishing an external connection. If you get a "Connection refused...accepting TCP/IP
// connections
// on port 5432?" error then the port is probably closed. Check that the firewall allows
// external TCP/IP
// connections (netstat -nap). You can open a port with lokkit or by configuring the iptables.
}
@SuppressWarnings({"unchecked", "deprecation"})
protected void startWithChefSoloAsync() {
String baseDir =
MachineLifecycleEffectorTasks.resolveOnBoxDir(
entity(),
Machines.findUniqueMachineLocation(entity().getLocations(), SshMachineLocation.class)
.get());
String installDir = Urls.mergePaths(baseDir, "installs/chef");
@SuppressWarnings("rawtypes")
Map<String, String> cookbooks =
(Map)
ConfigBag.newInstance(entity().getConfig(CHEF_COOKBOOK_URLS))
.putIfAbsent(entity().getConfig(CHEF_COOKBOOKS))
.getAllConfig();
if (cookbooks.isEmpty())
log.warn("No cookbook_urls set for " + entity() + "; launch will likely fail subsequently");
DynamicTasks.queue(
ChefSoloTasks.installChef(installDir, false),
ChefSoloTasks.installCookbooks(installDir, cookbooks, false));
// TODO chef for and run a prestart recipe if necessary
// TODO open ports
String primary = getPrimaryCookbook();
// put all config under brooklyn/cookbook/config
Navigator<MutableMap<Object, Object>> attrs = Jsonya.newInstancePrimitive().at("brooklyn");
if (Strings.isNonBlank(primary)) attrs.at(primary);
attrs.at("config");
attrs.put(entity().getAllConfigBag().getAllConfig());
// and put launch attrs at root
try {
attrs
.root()
.put(
(Map<?, ?>)
Tasks.resolveDeepValue(
entity().getConfig(CHEF_LAUNCH_ATTRIBUTES),
Object.class,
entity().getExecutionContext()));
} catch (Exception e) {
Exceptions.propagate(e);
}
Collection<? extends String> runList = entity().getConfig(CHEF_LAUNCH_RUN_LIST);
if (runList == null) runList = entity().getConfig(CHEF_RUN_LIST);
if (runList == null) {
if (Strings.isNonBlank(primary)) runList = ImmutableList.of(primary + "::" + "start");
else
throw new IllegalStateException(
"Require a primary cookbook or a run_list to effect " + "start" + " on " + entity());
}
String runDir =
Urls.mergePaths(
baseDir,
"apps/"
+ entity().getApplicationId()
+ "/chef/entities/"
+ entity().getEntityType().getSimpleName()
+ "_"
+ entity().getId());
DynamicTasks.queue(
ChefSoloTasks.buildChefFile(
runDir, installDir, "launch", runList, (Map<String, Object>) attrs.root().get()));
DynamicTasks.queue(
ChefSoloTasks.runChef(runDir, "launch", entity().getConfig(CHEF_RUN_CONVERGE_TWICE)));
}
protected void doStop(ConfigBag parameters, Callable<StopMachineDetails<Integer>> stopTask) {
preStopConfirmCustom();
log.info("Stopping {} in {}", entity(), entity().getLocations());
StopMode stopMachineMode = getStopMachineMode(parameters);
StopMode stopProcessMode = parameters.get(StopSoftwareParameters.STOP_PROCESS_MODE);
DynamicTasks.queue("pre-stop", new PreStopCustomTask());
// BROOKLYN-263:
// With this change the stop effector will wait for Location to provision so it can terminate
// the machine, if a provisioning request is in-progress.
//
// The ProvisionMachineTask stores transient internal state in PROVISIONING_TASK_STATE and
// PROVISIONED_MACHINE: it records when the provisioning is running and when done; and it
// records the final machine. We record the machine in the internal sensor (rather than
// just relying on getLocations) because the latter is set much later in the start()
// process.
//
// This code is a big improvement (previously there was a several-minute window in some
// clouds where a call to stop() would leave the machine running).
//
// However, there are still races. If the start() code has not yet reached the call to
// location.obtain() then we won't wait, and the start() call won't know to abort. It's
// fiddly to get that right, because we need to cope with restart() - so we mustn't leave
// any state behind that will interfere with subsequent sequential calls to start().
// There is some attempt to handle it by ProvisionMachineTask checking if the expectedState
// is stopping/stopped.
Maybe<MachineLocation> machine = Machines.findUniqueMachineLocation(entity().getLocations());
ProvisioningTaskState provisioningState =
entity().sensors().get(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
if (machine.isAbsent() && provisioningState == ProvisioningTaskState.RUNNING) {
Duration maxWait = entity().config().get(STOP_WAIT_PROVISIONING_TIMEOUT);
log.info(
"When stopping {}, waiting for up to {} for the machine to finish provisioning, before terminating it",
entity(),
maxWait);
boolean success =
Repeater.create("Wait for a machine to appear")
.until(
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
ProvisioningTaskState state =
entity()
.sensors()
.get(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
return (state != ProvisioningTaskState.RUNNING);
}
})
.backoffTo(Duration.FIVE_SECONDS)
.limitTimeTo(maxWait)
.run();
if (!success) {
log.warn(
"When stopping {}, timed out after {} waiting for the machine to finish provisioning - machine may we left running",
entity(),
maxWait);
}
machine = Maybe.ofDisallowingNull(entity().sensors().get(INTERNAL_PROVISIONED_MACHINE));
}
entity().sensors().remove(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
entity().sensors().remove(INTERNAL_PROVISIONED_MACHINE);
Task<List<?>> stoppingProcess = null;
if (canStop(stopProcessMode, entity())) {
stoppingProcess =
Tasks.parallel(
"stopping",
Tasks.create("stopping (process)", new StopProcessesAtMachineTask()),
Tasks.create("stopping (feeds)", new StopFeedsAtMachineTask()));
DynamicTasks.queue(stoppingProcess);
}
Task<StopMachineDetails<Integer>> stoppingMachine = null;
if (canStop(stopMachineMode, machine.isAbsent())) {
// Release this machine (even if error trying to stop process - we rethrow that after)
Map<String, Object> stopMachineFlags = MutableMap.of();
if (Entitlements.getEntitlementContext() != null) {
stopMachineFlags.put(
"tags",
MutableSet.of(
BrooklynTaskTags.tagForEntitlement(Entitlements.getEntitlementContext())));
}
Task<StopMachineDetails<Integer>> stopMachineTask =
Tasks.<StopMachineDetails<Integer>>builder()
.displayName("stopping (machine)")
.body(stopTask)
.flags(stopMachineFlags)
.build();
stoppingMachine = DynamicTasks.queue(stopMachineTask);
DynamicTasks.drain(entity().getConfig(STOP_PROCESS_TIMEOUT), false);
// shutdown the machine if stopping process fails or takes too long
synchronized (stoppingMachine) {
// task also used as mutex by DST when it submits it; ensure it only submits once!
if (!stoppingMachine.isSubmitted()) {
// force the stoppingMachine task to run by submitting it here
StringBuilder msg =
new StringBuilder("Submitting machine stop early in background for ")
.append(entity());
if (stoppingProcess == null) {
msg.append(". Process stop skipped, pre-stop not finished?");
} else {
msg.append(" because process stop has ")
.append((stoppingProcess.isDone() ? "finished abnormally" : "not finished"));
}
log.warn(msg.toString());
Entities.submit(entity(), stoppingMachine);
}
}
}
try {
// This maintains previous behaviour of silently squashing any errors on the stoppingProcess
// task if the
// stoppingMachine exits with a nonzero value
boolean checkStopProcesses =
(stoppingProcess != null
&& (stoppingMachine == null || stoppingMachine.get().value == 0));
if (checkStopProcesses) {
// TODO we should test for destruction above, not merely successful "stop", as things like
// localhost and ssh won't be destroyed
DynamicTasks.waitForLast();
if (machine.isPresent()) {
// throw early errors *only if* there is a machine and we have not destroyed it
stoppingProcess.get();
}
}
} catch (Throwable e) {
ServiceStateLogic.setExpectedState(entity(), Lifecycle.ON_FIRE);
Exceptions.propagate(e);
}
entity().sensors().set(SoftwareProcess.SERVICE_UP, false);
ServiceStateLogic.setExpectedState(entity(), Lifecycle.STOPPED);
DynamicTasks.queue("post-stop", new PostStopCustomTask());
if (log.isDebugEnabled()) log.debug("Stopped software process entity " + entity());
}
@SuppressWarnings("unchecked")
@Override
public void startReadOnly(final ManagementNodeState mode) {
if (!ManagementNodeState.isHotProxy(mode)) {
throw new IllegalStateException(
"Read-only rebind thread only permitted for hot proxy modes; not " + mode);
}
if (persistenceRunning) {
throw new IllegalStateException(
"Cannot start read-only when already running with persistence");
}
if (readOnlyRunning || readOnlyTask != null) {
LOG.warn(
"Cannot request read-only mode for "
+ this
+ " when already running - "
+ readOnlyTask
+ "; ignoring");
return;
}
LOG.debug(
"Starting read-only rebinding ("
+ this
+ "), mgmt "
+ managementContext.getManagementNodeId());
if (persistenceRealChangeListener != null) persistenceRealChangeListener.stop();
if (persistenceStoreAccess != null) persistenceStoreAccess.disableWriteAccess(true);
readOnlyRunning = true;
readOnlyRebindCount.set(0);
try {
rebind(null, null, mode);
} catch (Exception e) {
throw Exceptions.propagate(e);
}
Callable<Task<?>> taskFactory =
new Callable<Task<?>>() {
@Override
public Task<Void> call() {
return Tasks.<Void>builder()
.dynamic(false)
.displayName("rebind (periodic run")
.body(
new Callable<Void>() {
public Void call() {
try {
rebind(null, null, mode);
return null;
} catch (RuntimeInterruptedException e) {
LOG.debug("Interrupted rebinding (re-interrupting): " + e);
if (LOG.isTraceEnabled())
LOG.trace("Interrupted rebinding (re-interrupting), details: " + e, e);
Thread.currentThread().interrupt();
return null;
} catch (Exception e) {
// Don't rethrow: the behaviour of executionManager is different from a
// scheduledExecutorService,
// if we throw an exception, then our task will never get executed again
if (!readOnlyRunning) {
LOG.debug(
"Problem rebinding (read-only running has probably just been turned off): "
+ e);
if (LOG.isTraceEnabled()) {
LOG.trace(
"Problem rebinding (read-only running has probably just been turned off), details: "
+ e,
e);
}
} else {
LOG.error("Problem rebinding: " + Exceptions.collapseText(e), e);
}
return null;
} catch (Throwable t) {
LOG.warn("Problem rebinding (rethrowing)", t);
throw Exceptions.propagate(t);
}
}
})
.build();
}
};
readOnlyTask =
(ScheduledTask)
managementContext
.getServerExecutionContext()
.submit(
new ScheduledTask(
MutableMap.of("displayName", "Periodic read-only rebind"), taskFactory)
.period(periodicPersistPeriod));
}
@SuppressWarnings({"rawtypes", "unchecked"})
@Override
public V call() {
T value = source.getAttribute(sensor);
// return immediately if either the ready predicate or the abort conditions hold
if (ready(value)) return postProcess(value);
final List<Exception> abortionExceptions = Lists.newCopyOnWriteArrayList();
long start = System.currentTimeMillis();
for (AttributeAndSensorCondition abortCondition : abortSensorConditions) {
Object abortValue = abortCondition.source.getAttribute(abortCondition.sensor);
if (abortCondition.predicate.apply(abortValue)) {
abortionExceptions.add(
new Exception(
"Abort due to " + abortCondition.source + " -> " + abortCondition.sensor));
}
}
if (abortionExceptions.size() > 0) {
throw new CompoundRuntimeException(
"Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
}
TaskInternal<?> current = (TaskInternal<?>) Tasks.current();
if (current == null)
throw new IllegalStateException("Should only be invoked in a running task");
Entity entity = BrooklynTaskTags.getTargetOrContextEntity(current);
if (entity == null)
throw new IllegalStateException(
"Should only be invoked in a running task with an entity tag; "
+ current
+ " has no entity tag ("
+ current.getStatusDetail(false)
+ ")");
final LinkedList<T> publishedValues = new LinkedList<T>();
final Semaphore semaphore = new Semaphore(0); // could use Exchanger
SubscriptionHandle subscription = null;
List<SubscriptionHandle> abortSubscriptions = Lists.newArrayList();
try {
subscription =
entity
.subscriptions()
.subscribe(
source,
sensor,
new SensorEventListener<T>() {
@Override
public void onEvent(SensorEvent<T> event) {
synchronized (publishedValues) {
publishedValues.add(event.getValue());
}
semaphore.release();
}
});
for (final AttributeAndSensorCondition abortCondition : abortSensorConditions) {
abortSubscriptions.add(
entity
.subscriptions()
.subscribe(
abortCondition.source,
abortCondition.sensor,
new SensorEventListener<Object>() {
@Override
public void onEvent(SensorEvent<Object> event) {
if (abortCondition.predicate.apply(event.getValue())) {
abortionExceptions.add(
new Exception(
"Abort due to "
+ abortCondition.source
+ " -> "
+ abortCondition.sensor));
semaphore.release();
}
}
}));
Object abortValue = abortCondition.source.getAttribute(abortCondition.sensor);
if (abortCondition.predicate.apply(abortValue)) {
abortionExceptions.add(
new Exception(
"Abort due to " + abortCondition.source + " -> " + abortCondition.sensor));
}
}
if (abortionExceptions.size() > 0) {
throw new CompoundRuntimeException(
"Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
}
CountdownTimer timer = timeout != null ? timeout.countdownTimer() : null;
Duration maxPeriod = ValueResolver.PRETTY_QUICK_WAIT;
Duration nextPeriod = ValueResolver.REAL_QUICK_PERIOD;
while (true) {
// check the source on initial run (could be done outside the loop)
// and also (optionally) on each iteration in case it is more recent
value = source.getAttribute(sensor);
if (ready(value)) break;
if (timer != null) {
if (timer.getDurationRemaining().isShorterThan(nextPeriod)) {
nextPeriod = timer.getDurationRemaining();
}
if (timer.isExpired()) {
if (onTimeout.isPresent()) return onTimeout.get();
throw new RuntimeTimeoutException("Unsatisfied after " + Duration.sinceUtc(start));
}
}
String prevBlockingDetails = current.setBlockingDetails(blockingDetails);
try {
if (semaphore.tryAcquire(nextPeriod.toMilliseconds(), TimeUnit.MILLISECONDS)) {
// immediately release so we are available for the next check
semaphore.release();
// if other permits have been made available (e.g. multiple notifications) drain them
// all as no point running multiple times
semaphore.drainPermits();
}
} finally {
current.setBlockingDetails(prevBlockingDetails);
}
// check any subscribed values which have come in first
while (true) {
synchronized (publishedValues) {
if (publishedValues.isEmpty()) break;
value = publishedValues.pop();
}
if (ready(value)) break;
}
// if unmanaged then ignore the other abort conditions
if (!ignoreUnmanaged && Entities.isNoLongerManaged(entity)) {
if (onUnmanaged.isPresent()) return onUnmanaged.get();
throw new NotManagedException(entity);
}
if (abortionExceptions.size() > 0) {
throw new CompoundRuntimeException(
"Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
}
nextPeriod = nextPeriod.times(2).upperBound(maxPeriod);
}
if (LOG.isDebugEnabled()) LOG.debug("Attribute-ready for {} in entity {}", sensor, source);
return postProcess(value);
} catch (InterruptedException e) {
throw Exceptions.propagate(e);
} finally {
if (subscription != null) {
entity.subscriptions().unsubscribe(subscription);
}
for (SubscriptionHandle handle : abortSubscriptions) {
entity.subscriptions().unsubscribe(handle);
}
}
}
