Exemplo n.º 1
0
  @Override
  public String execCommandTimeout(String command, Duration timeout) {
    ProcessTaskWrapper<String> task =
        SshEffectorTasks.ssh(command)
            .environmentVariables(
                ((AbstractSoftwareProcessSshDriver) getDriver()).getShellEnvironment())
            .configure(SshTool.PROP_ALLOCATE_PTY, true) // TODO configure globally
            .requiringZeroAndReturningStdout()
            .machine(getMachine())
            .summary(command)
            .newTask();

    try {
      String result =
          DynamicTasks.queueIfPossible(task)
              .executionContext(this)
              .orSubmitAsync()
              .asTask()
              .get(timeout);
      return result;
    } catch (TimeoutException te) {
      throw new IllegalStateException("Timed out running command: " + command);
    } catch (Exception e) {
      Integer exitCode = task.getExitCode();
      LOG.warn(
          "Command failed, return code {}: {}", exitCode == null ? -1 : exitCode, task.getStderr());
      throw Exceptions.propagate(e);
    }
  }
 private void copy(String src, File dst) {
   try {
     copy(new ReaderInputStream(new StringReader(src), "UTF-8"), dst);
   } catch (Exception e) {
     throw Exceptions.propagate(e);
   }
 }
Exemplo n.º 3
0
 /**
  * as {@link #waitOnForExpiry(Object)} but catches and wraps InterruptedException as unchecked
  * RuntimeInterruptedExcedption
  */
 public boolean waitOnForExpiryUnchecked(Object waitTarget) {
   try {
     return waitOnForExpiry(waitTarget);
   } catch (InterruptedException e) {
     throw Exceptions.propagate(e);
   }
 }
  private Response launch(String yaml, EntitySpec<? extends Application> spec) {
    try {
      Application app = EntityManagementUtils.createUnstarted(mgmt(), spec);
      CreationResult<Application, Void> result = EntityManagementUtils.start(app);

      boolean isEntitled =
          Entitlements.isEntitled(
              mgmt().getEntitlementManager(),
              Entitlements.INVOKE_EFFECTOR,
              EntityAndItem.of(app, StringAndArgument.of(Startable.START.getName(), null)));

      if (!isEntitled) {
        throw WebResourceUtils.unauthorized(
            "User '%s' is not authorized to start application %s",
            Entitlements.getEntitlementContext().user(), spec.getType());
      }

      log.info("Launched from YAML: " + yaml + " -> " + app + " (" + result.task() + ")");

      URI ref = URI.create(app.getApplicationId());
      ResponseBuilder response = created(ref);
      if (result.task() != null) response.entity(TaskTransformer.FROM_TASK.apply(result.task()));
      return response.build();
    } catch (ConstraintViolationException e) {
      throw new UserFacingException(e);
    } catch (Exception e) {
      throw Exceptions.propagate(e);
    }
  }
 @Override
 public EntitySpec<? extends Application> createApplicationSpec(String plan) {
   try {
     CampPlatform camp = CampCatalogUtils.getCampPlatform(mgmt);
     BrooklynClassLoadingContext loader = JavaBrooklynClassLoadingContext.create(mgmt);
     AssemblyTemplate at = CampUtils.registerDeploymentPlan(plan, loader, camp);
     AssemblyTemplateInstantiator instantiator = CampUtils.getInstantiator(at);
     if (instantiator instanceof AssemblyTemplateSpecInstantiator) {
       return ((AssemblyTemplateSpecInstantiator) instantiator)
           .createApplicationSpec(at, camp, loader);
     } else {
       // The unknown instantiator can create the app (Assembly), but not a spec.
       // Currently, all brooklyn plans should produce the above.
       if (at.getPlatformComponentTemplates() == null
           || at.getPlatformComponentTemplates().isEmpty()) {
         if (at.getCustomAttributes().containsKey("brooklyn.catalog"))
           throw new IllegalArgumentException(
               "Unrecognized application blueprint format: expected an application, not a brooklyn.catalog");
         throw new PlanNotRecognizedException(
             "Unrecognized application blueprint format: no services defined");
       }
       // map this (expected) error to a nicer message
       throw new PlanNotRecognizedException("Unrecognized application blueprint format");
     }
   } catch (Exception e) {
     // TODO how do we figure out that the plan is not supported vs. invalid to wrap in a
     // PlanNotRecognizedException?
     if (log.isDebugEnabled()) log.debug("Failed to create entity from CAMP spec:\n" + plan, e);
     throw Exceptions.propagate(e);
   }
 }
  @Override
  public final T get() {
    if (log.isDebugEnabled())
      log.debug("Queuing task to resolve " + dsl + ", called by " + Tasks.current());

    EntityInternal entity =
        (EntityInternal) BrooklynTaskTags.getTargetOrContextEntity(Tasks.current());
    ExecutionContext exec =
        (entity != null)
            ? entity.getExecutionContext()
            : BasicExecutionContext.getCurrentExecutionContext();
    if (exec == null) {
      throw new IllegalStateException("No execution context available to resolve " + dsl);
    }

    Task<T> task = newTask();
    T result;
    try {
      result = exec.submit(task).get();
    } catch (InterruptedException | ExecutionException e) {
      Task<?> currentTask = Tasks.current();
      if (currentTask != null && currentTask.isCancelled()) {
        task.cancel(true);
      }
      throw Exceptions.propagate(e);
    }

    if (log.isDebugEnabled()) log.debug("Resolved " + result + " from " + dsl);
    return result;
  }
  /**
   * Runs the given callable. Repeats until the operation succeeds or {@link #isExceptionRetryable}
   * indicates that the request cannot be retried.
   */
  protected <T> T runOperationWithRetry(Callable<T> operation) {
    int backoff = 64;
    Exception lastException = null;
    for (int retries = 0; retries < 100; retries++) {
      try {
        return operation.call();
      } catch (Exception e) {
        lastException = e;
        if (isExceptionRetryable.apply(e)) {
          LOG.debug("Attempt #{} failed to add security group: {}", retries + 1, e.getMessage());
          try {
            Thread.sleep(backoff);
          } catch (InterruptedException e1) {
            throw Exceptions.propagate(e1);
          }
          backoff = backoff << 1;
        } else {
          break;
        }
      }
    }

    throw new RuntimeException(
        "Unable to add security group rule; repeated errors from provider", lastException);
  }
Exemplo n.º 8
0
  @Override
  public void doStart(Collection<? extends Location> locs) {
    List<Location> locations = MutableList.of();

    sensors().set(SERVICE_UP, Boolean.FALSE);
    ServiceStateLogic.setExpectedState(this, Lifecycle.STARTING);

    LOG.info("Creating new MesosLocation");
    createLocation(MutableMap.<String, Object>of());

    // Start frameworks
    try {
      Group frameworks = sensors().get(MESOS_FRAMEWORKS);
      Entities.invokeEffectorList(
              this,
              frameworks.getMembers(),
              Startable.START,
              ImmutableMap.of("locations", locations))
          .getUnchecked();
    } catch (Exception e) {
      LOG.warn("Error starting frameworks", e);
      ServiceStateLogic.setExpectedState(this, Lifecycle.ON_FIRE);
      Exceptions.propagate(e);
    }

    super.doStart(locations);

    connectSensors();

    ServiceStateLogic.setExpectedState(this, Lifecycle.RUNNING);
    sensors().set(SERVICE_UP, Boolean.TRUE);
  }
Exemplo n.º 9
0
 protected void onUpdated() {
   try {
     emit(targetSensor, compute());
   } catch (Throwable t) {
     LOG.warn("Error calculating and setting aggregate for enricher " + this, t);
     throw Exceptions.propagate(t);
   }
 }
Exemplo n.º 10
0
 public void waitForImage(String imageName) {
   try {
     CountDownLatch latch = images.get(imageName);
     if (latch != null) latch.await(15, TimeUnit.MINUTES);
   } catch (InterruptedException ie) {
     throw Exceptions.propagate(ie);
   }
 }
Exemplo n.º 11
0
 @Override
 public InetAddress getAddress() {
   String address = getOwner().sensors().get(Attributes.ADDRESS);
   try {
     return InetAddress.getByName(address);
   } catch (UnknownHostException e) {
     throw Exceptions.propagate(e);
   }
 }
 private void copy(InputStream in, File tmpJar) {
   try {
     OutputStream out = new FileOutputStream(tmpJar);
     ByteStreams.copy(in, out);
     Streams.closeQuietly(in);
     Streams.closeQuietly(out);
   } catch (Exception e) {
     throw Exceptions.propagate(e);
   }
 }
 /**
  * runs the tasks needed to start, wrapped by setting {@link Attributes#SERVICE_STATE_EXPECTED}
  * appropriately
  */
 public void start(Collection<? extends Location> locations) {
   ServiceStateLogic.setExpectedState(entity(), Lifecycle.STARTING);
   try {
     startInLocations(locations);
     DynamicTasks.waitForLast();
     ServiceStateLogic.setExpectedState(entity(), Lifecycle.RUNNING);
   } catch (Throwable t) {
     ServiceStateLogic.setExpectedState(entity(), Lifecycle.ON_FIRE);
     throw Exceptions.propagate(t);
   }
 }
Exemplo n.º 14
0
 @Override
 public InetAddress getNextAgentAddress(String agentId) {
   Entity agent = getManagementContext().getEntityManager().getEntity(agentId);
   String address =
       agent.sensors().get(CalicoNode.DOCKER_HOST).sensors().get(Attributes.SUBNET_ADDRESS);
   try {
     return InetAddress.getByName(address);
   } catch (UnknownHostException uhe) {
     throw Exceptions.propagate(uhe);
   }
 }
Exemplo n.º 15
0
 @Override
 public void close() throws IOException {
   LOG.info("Close called on Docker host {}: {}", machine, this);
   try {
     machine.close();
   } catch (Exception e) {
     LOG.info("{}: Closing Docker host: {}", e.getMessage(), this);
     throw Exceptions.propagate(e);
   } finally {
     LOG.info("Docker host closed: {}", this);
   }
 }
Exemplo n.º 16
0
 @Override
 public void close() throws IOException {
   LOG.debug("Close called on Marathon task: {}", this);
   try {
     if (marathonTask.sensors().get(MarathonTask.SERVICE_UP)) {
       LOG.info("Stopping Marathon task entity for {}: {}", this, marathonTask);
       marathonTask.stop();
     }
     LOG.info("Marathon task closed: {}", this);
   } catch (Exception e) {
     LOG.warn("Error closing Marathon task {}: {}", this, e.getMessage());
     throw Exceptions.propagate(e);
   }
 }
 @Override
 public void onManagementStarting() {
   super.onManagementStarting();
   sensors().set(TOPIC_NAME, getName());
   try {
     String virtualHost = getParent().getVirtualHost();
     exchange =
         new ObjectName(
             format(
                 "org.apache.qpid:type=VirtualHost.Exchange,VirtualHost=\"%s\",name=\"%s\",ExchangeType=topic",
                 virtualHost, getExchangeName()));
   } catch (MalformedObjectNameException e) {
     throw Exceptions.propagate(e);
   }
 }
  /** builds remote keystores, stores config keys/certs, and copies necessary files across */
  public void install() {
    try {
      // build truststore and keystore
      FluentKeySigner signer = getBrooklynRootSigner();
      KeyPair jmxAgentKey = SecureKeys.newKeyPair();
      X509Certificate jmxAgentCert = signer.newCertificateFor("jmxmp-agent", jmxAgentKey);

      agentKeyStore = SecureKeys.newKeyStore();
      agentKeyStore.setKeyEntry(
          "jmxmp-agent",
          jmxAgentKey.getPrivate(),
          // TODO jmx.ssl.agent.keyPassword
          "".toCharArray(),
          new Certificate[] {jmxAgentCert});
      ByteArrayOutputStream agentKeyStoreBytes = new ByteArrayOutputStream();
      agentKeyStore.store(
          agentKeyStoreBytes,
          // TODO jmx.ssl.agent.keyStorePassword
          "".toCharArray());

      agentTrustStore = SecureKeys.newKeyStore();
      agentTrustStore.setCertificateEntry("brooklyn", getJmxAccessCert());
      ByteArrayOutputStream agentTrustStoreBytes = new ByteArrayOutputStream();
      agentTrustStore.store(agentTrustStoreBytes, "".toCharArray());

      // install the truststore and keystore and rely on JmxSupport to install the agent
      Tasks.setBlockingDetails("Copying keystore and truststore to the server.");
      try {
        jmxSupport
            .getMachine()
            .get()
            .copyTo(
                new ByteArrayInputStream(agentKeyStoreBytes.toByteArray()),
                getJmxSslKeyStoreFilePath());
        jmxSupport
            .getMachine()
            .get()
            .copyTo(
                new ByteArrayInputStream(agentTrustStoreBytes.toByteArray()),
                getJmxSslTrustStoreFilePath());
      } finally {
        Tasks.resetBlockingDetails();
      }

    } catch (Exception e) {
      throw Exceptions.propagate(e);
    }
  }
 private EntitySpec<? extends Application> createEntitySpecForApplication(String potentialYaml) {
   try {
     return EntityManagementUtils.createEntitySpecForApplication(mgmt(), potentialYaml);
   } catch (Exception e) {
     // An IllegalArgumentException for creating the entity spec gets wrapped in a ISE, and
     // possibly a Compound.
     // But we want to return a 400 rather than 500, so ensure we throw IAE.
     IllegalArgumentException iae =
         (IllegalArgumentException)
             Exceptions.getFirstThrowableOfType(e, IllegalArgumentException.class);
     if (iae != null) {
       throw new IllegalArgumentException("Cannot create spec for app: " + iae.getMessage(), e);
     } else {
       throw Exceptions.propagate(e);
     }
   }
 }
  @Override
  public List<Application> rebind(
      ClassLoader classLoaderO,
      RebindExceptionHandler exceptionHandlerO,
      ManagementNodeState modeO) {
    final ClassLoader classLoader =
        classLoaderO != null ? classLoaderO : managementContext.getCatalogClassLoader();
    final RebindExceptionHandler exceptionHandler =
        exceptionHandlerO != null
            ? exceptionHandlerO
            : RebindExceptionHandlerImpl.builder()
                .danglingRefFailureMode(danglingRefFailureMode)
                .danglingRefQuorumRequiredHealthy(danglingRefsQuorumRequiredHealthy)
                .rebindFailureMode(rebindFailureMode)
                .addConfigFailureMode(addConfigFailureMode)
                .addPolicyFailureMode(addPolicyFailureMode)
                .loadPolicyFailureMode(loadPolicyFailureMode)
                .build();
    final ManagementNodeState mode = modeO != null ? modeO : getRebindMode();

    if (mode != ManagementNodeState.MASTER
        && mode != ManagementNodeState.HOT_STANDBY
        && mode != ManagementNodeState.HOT_BACKUP)
      throw new IllegalStateException(
          "Must be either master or hot standby/backup to rebind (mode " + mode + ")");

    ExecutionContext ec = BasicExecutionContext.getCurrentExecutionContext();
    if (ec == null) {
      ec = managementContext.getServerExecutionContext();
      Task<List<Application>> task =
          ec.submit(
              new Callable<List<Application>>() {
                @Override
                public List<Application> call() throws Exception {
                  return rebindImpl(classLoader, exceptionHandler, mode);
                }
              });
      try {
        return task.get();
      } catch (Exception e) {
        throw Exceptions.propagate(e);
      }
    } else {
      return rebindImpl(classLoader, exceptionHandler, mode);
    }
  }
  private <T> T resolve(ConfigKey<T> key, Object value) {
    Object transformed = transform(key, value);

    Object result;
    if (transformed instanceof DeferredSupplier) {
      ExecutionContext exec = mgmt.getServerExecutionContext();
      try {
        result = Tasks.resolveValue(transformed, key.getType(), exec);
      } catch (ExecutionException | InterruptedException e) {
        throw Exceptions.propagate(e);
      }
    } else {
      result = transformed;
    }

    return TypeCoercions.coerce(result, key.getTypeToken());
  }
  @SuppressWarnings({"unchecked", "rawtypes", "deprecation"})
  protected void startWithKnifeAsync() {
    // TODO prestart, ports (as above); also, note, some aspects of this are untested as we need a
    // chef server

    String primary = getPrimaryCookbook();

    // put all config under brooklyn/cookbook/config
    Navigator<MutableMap<Object, Object>> attrs = Jsonya.newInstancePrimitive().at("brooklyn");
    if (Strings.isNonBlank(primary)) attrs.at(primary);
    attrs.at("config");
    attrs.put(entity().getAllConfigBag().getAllConfig());
    // and put launch attrs at root
    try {
      attrs
          .root()
          .put(
              (Map<?, ?>)
                  Tasks.resolveDeepValue(
                      entity().getConfig(CHEF_LAUNCH_ATTRIBUTES),
                      Object.class,
                      entity().getExecutionContext()));
    } catch (Exception e) {
      Exceptions.propagate(e);
    }

    Collection<? extends String> runList = entity().getConfig(CHEF_LAUNCH_RUN_LIST);
    if (runList == null) runList = entity().getConfig(CHEF_RUN_LIST);
    if (runList == null) {
      if (Strings.isNonBlank(primary)) runList = ImmutableList.of(primary + "::" + "start");
      else
        throw new IllegalStateException(
            "Require a primary cookbook or a run_list to effect " + "start" + " on " + entity());
    }

    DynamicTasks.queue(
        ChefServerTasks.knifeConvergeTask()
            .knifeNodeName(getNodeName())
            .knifeRunList(Strings.join(runList, ","))
            .knifeAddAttributes((Map<? extends Object, ? extends Object>) (Map) attrs.root().get())
            .knifeRunTwice(entity().getConfig(CHEF_RUN_CONVERGE_TWICE)));
  }
Exemplo n.º 23
0
 @SuppressWarnings({"unchecked", "rawtypes"})
 protected static EffectorSummary.ParameterSummary<?> parameterSummary(
     Entity entity, ParameterType<?> parameterType) {
   try {
     Maybe<?> defaultValue =
         Tasks.resolving(parameterType.getDefaultValue())
             .as(parameterType.getParameterClass())
             .context(entity)
             .timeout(ValueResolver.REAL_QUICK_WAIT)
             .getMaybe();
     return new ParameterSummary(
         parameterType.getName(),
         parameterType.getParameterClassName(),
         parameterType.getDescription(),
         WebResourceUtils.getValueForDisplay(defaultValue.orNull(), true, false),
         Sanitizer.IS_SECRET_PREDICATE.apply(parameterType.getName()));
   } catch (Exception e) {
     throw Exceptions.propagate(e);
   }
 }
  @Override
  public int copyResource(
      Map<Object, Object> sshFlags, String source, String target, boolean createParentDir) {
    if (createParentDir) {
      createDirectory(getDirectory(target), "Creating resource directory");
    }

    InputStream stream = null;
    try {
      Tasks.setBlockingDetails("retrieving resource " + source + " for copying across");
      stream = resource.getResourceFromUrl(source);
      Tasks.setBlockingDetails("copying resource " + source + " to server");
      return copyTo(stream, target);
    } catch (Exception e) {
      throw Exceptions.propagate(e);
    } finally {
      Tasks.setBlockingDetails(null);
      if (stream != null) Streams.closeQuietly(stream);
    }
  }
  @Override
  public void customize() {
    // Some OSes start postgres during package installation
    DynamicTasks.queue(
            SshEffectorTasks.ssh(sudoAsUser("postgres", "/etc/init.d/postgresql stop"))
                .allowingNonZeroExitCode())
        .get();

    newScript(CUSTOMIZING)
        .body
        .append(
            sudo("mkdir -p " + getDataDir()),
            sudo("chown postgres:postgres " + getDataDir()),
            sudo("chmod 700 " + getDataDir()),
            sudo("touch " + getLogFile()),
            sudo("chown postgres:postgres " + getLogFile()),
            sudo("touch " + getPidFile()),
            sudo("chown postgres:postgres " + getPidFile()),
            alternativesGroup(
                chainGroup(
                    format("test -e %s", getInstallDir() + "/bin/initdb"),
                    sudoAsUser("postgres", getInstallDir() + "/bin/initdb -D " + getDataDir())),
                callPgctl("initdb", true)))
        .failOnNonZeroResultCode()
        .execute();

    String configUrl = getEntity().getConfig(PostgreSqlNode.CONFIGURATION_FILE_URL);
    if (Strings.isBlank(configUrl)) {
      // http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server
      // If the same setting is listed multiple times, the last one wins.
      DynamicTasks.queue(
          SshEffectorTasks.ssh(
              executeCommandThenAsUserTeeOutputToFile(
                  chainGroup(
                      "echo \"listen_addresses = '*'\"",
                      "echo \"port = " + getEntity().getPostgreSqlPort() + "\"",
                      "echo \"max_connections = " + getEntity().getMaxConnections() + "\"",
                      "echo \"shared_buffers = " + getEntity().getSharedMemory() + "\"",
                      "echo \"external_pid_file = '" + getPidFile() + "'\""),
                  "postgres",
                  getDataDir() + "/postgresql.conf")));
    } else {
      String contents = processTemplate(configUrl);
      DynamicTasks.queue(
          SshEffectorTasks.put("/tmp/postgresql.conf").contents(contents),
          SshEffectorTasks.ssh(
              sudoAsUser(
                  "postgres", "cp /tmp/postgresql.conf " + getDataDir() + "/postgresql.conf")));
    }

    String authConfigUrl =
        getEntity().getConfig(PostgreSqlNode.AUTHENTICATION_CONFIGURATION_FILE_URL);
    if (Strings.isBlank(authConfigUrl)) {
      DynamicTasks.queue(
          SshEffectorTasks.ssh(
              // TODO give users control which hosts can connect and the authentication mechanism
              executeCommandThenAsUserTeeOutputToFile(
                  "echo \"host all all 0.0.0.0/0 md5\"",
                  "postgres",
                  getDataDir() + "/pg_hba.conf")));
    } else {
      String contents = processTemplate(authConfigUrl);
      DynamicTasks.queue(
          SshEffectorTasks.put("/tmp/pg_hba.conf").contents(contents),
          SshEffectorTasks.ssh(
              sudoAsUser("postgres", "cp /tmp/pg_hba.conf " + getDataDir() + "/pg_hba.conf")));
    }

    // Wait for commands to complete before running the creation script
    DynamicTasks.waitForLast();
    if (Boolean.TRUE.equals(entity.getConfig(PostgreSqlNode.INITIALIZE_DB))) {
      initializeNewDatabase();
    }
    // Capture log file contents if there is an error configuring the database
    try {
      executeDatabaseCreationScript();
    } catch (RuntimeException r) {
      logTailOfPostgresLog();
      throw Exceptions.propagate(r);
    }

    // Try establishing an external connection. If you get a "Connection refused...accepting TCP/IP
    // connections
    // on port 5432?" error then the port is probably closed. Check that the firewall allows
    // external TCP/IP
    // connections (netstat -nap). You can open a port with lokkit or by configuring the iptables.
  }
  @SuppressWarnings({"unchecked", "deprecation"})
  protected void startWithChefSoloAsync() {
    String baseDir =
        MachineLifecycleEffectorTasks.resolveOnBoxDir(
            entity(),
            Machines.findUniqueMachineLocation(entity().getLocations(), SshMachineLocation.class)
                .get());
    String installDir = Urls.mergePaths(baseDir, "installs/chef");

    @SuppressWarnings("rawtypes")
    Map<String, String> cookbooks =
        (Map)
            ConfigBag.newInstance(entity().getConfig(CHEF_COOKBOOK_URLS))
                .putIfAbsent(entity().getConfig(CHEF_COOKBOOKS))
                .getAllConfig();
    if (cookbooks.isEmpty())
      log.warn("No cookbook_urls set for " + entity() + "; launch will likely fail subsequently");
    DynamicTasks.queue(
        ChefSoloTasks.installChef(installDir, false),
        ChefSoloTasks.installCookbooks(installDir, cookbooks, false));

    // TODO chef for and run a prestart recipe if necessary
    // TODO open ports

    String primary = getPrimaryCookbook();

    // put all config under brooklyn/cookbook/config
    Navigator<MutableMap<Object, Object>> attrs = Jsonya.newInstancePrimitive().at("brooklyn");
    if (Strings.isNonBlank(primary)) attrs.at(primary);
    attrs.at("config");
    attrs.put(entity().getAllConfigBag().getAllConfig());
    // and put launch attrs at root
    try {
      attrs
          .root()
          .put(
              (Map<?, ?>)
                  Tasks.resolveDeepValue(
                      entity().getConfig(CHEF_LAUNCH_ATTRIBUTES),
                      Object.class,
                      entity().getExecutionContext()));
    } catch (Exception e) {
      Exceptions.propagate(e);
    }

    Collection<? extends String> runList = entity().getConfig(CHEF_LAUNCH_RUN_LIST);
    if (runList == null) runList = entity().getConfig(CHEF_RUN_LIST);
    if (runList == null) {
      if (Strings.isNonBlank(primary)) runList = ImmutableList.of(primary + "::" + "start");
      else
        throw new IllegalStateException(
            "Require a primary cookbook or a run_list to effect " + "start" + " on " + entity());
    }

    String runDir =
        Urls.mergePaths(
            baseDir,
            "apps/"
                + entity().getApplicationId()
                + "/chef/entities/"
                + entity().getEntityType().getSimpleName()
                + "_"
                + entity().getId());

    DynamicTasks.queue(
        ChefSoloTasks.buildChefFile(
            runDir, installDir, "launch", runList, (Map<String, Object>) attrs.root().get()));

    DynamicTasks.queue(
        ChefSoloTasks.runChef(runDir, "launch", entity().getConfig(CHEF_RUN_CONVERGE_TWICE)));
  }
  protected void doStop(ConfigBag parameters, Callable<StopMachineDetails<Integer>> stopTask) {
    preStopConfirmCustom();

    log.info("Stopping {} in {}", entity(), entity().getLocations());

    StopMode stopMachineMode = getStopMachineMode(parameters);
    StopMode stopProcessMode = parameters.get(StopSoftwareParameters.STOP_PROCESS_MODE);

    DynamicTasks.queue("pre-stop", new PreStopCustomTask());

    // BROOKLYN-263:
    // With this change the stop effector will wait for Location to provision so it can terminate
    // the machine, if a provisioning request is in-progress.
    //
    // The ProvisionMachineTask stores transient internal state in PROVISIONING_TASK_STATE and
    // PROVISIONED_MACHINE: it records when the provisioning is running and when done; and it
    // records the final machine. We record the machine in the internal sensor (rather than
    // just relying on getLocations) because the latter is set much later in the start()
    // process.
    //
    // This code is a big improvement (previously there was a several-minute window in some
    // clouds where a call to stop() would leave the machine running).
    //
    // However, there are still races. If the start() code has not yet reached the call to
    // location.obtain() then we won't wait, and the start() call won't know to abort. It's
    // fiddly to get that right, because we need to cope with restart() - so we mustn't leave
    // any state behind that will interfere with subsequent sequential calls to start().
    // There is some attempt to handle it by ProvisionMachineTask checking if the expectedState
    // is stopping/stopped.
    Maybe<MachineLocation> machine = Machines.findUniqueMachineLocation(entity().getLocations());
    ProvisioningTaskState provisioningState =
        entity().sensors().get(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);

    if (machine.isAbsent() && provisioningState == ProvisioningTaskState.RUNNING) {
      Duration maxWait = entity().config().get(STOP_WAIT_PROVISIONING_TIMEOUT);
      log.info(
          "When stopping {}, waiting for up to {} for the machine to finish provisioning, before terminating it",
          entity(),
          maxWait);
      boolean success =
          Repeater.create("Wait for a machine to appear")
              .until(
                  new Callable<Boolean>() {
                    @Override
                    public Boolean call() throws Exception {
                      ProvisioningTaskState state =
                          entity()
                              .sensors()
                              .get(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
                      return (state != ProvisioningTaskState.RUNNING);
                    }
                  })
              .backoffTo(Duration.FIVE_SECONDS)
              .limitTimeTo(maxWait)
              .run();
      if (!success) {
        log.warn(
            "When stopping {}, timed out after {} waiting for the machine to finish provisioning - machine may we left running",
            entity(),
            maxWait);
      }
      machine = Maybe.ofDisallowingNull(entity().sensors().get(INTERNAL_PROVISIONED_MACHINE));
    }
    entity().sensors().remove(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
    entity().sensors().remove(INTERNAL_PROVISIONED_MACHINE);

    Task<List<?>> stoppingProcess = null;
    if (canStop(stopProcessMode, entity())) {
      stoppingProcess =
          Tasks.parallel(
              "stopping",
              Tasks.create("stopping (process)", new StopProcessesAtMachineTask()),
              Tasks.create("stopping (feeds)", new StopFeedsAtMachineTask()));
      DynamicTasks.queue(stoppingProcess);
    }

    Task<StopMachineDetails<Integer>> stoppingMachine = null;
    if (canStop(stopMachineMode, machine.isAbsent())) {
      // Release this machine (even if error trying to stop process - we rethrow that after)
      Map<String, Object> stopMachineFlags = MutableMap.of();
      if (Entitlements.getEntitlementContext() != null) {
        stopMachineFlags.put(
            "tags",
            MutableSet.of(
                BrooklynTaskTags.tagForEntitlement(Entitlements.getEntitlementContext())));
      }
      Task<StopMachineDetails<Integer>> stopMachineTask =
          Tasks.<StopMachineDetails<Integer>>builder()
              .displayName("stopping (machine)")
              .body(stopTask)
              .flags(stopMachineFlags)
              .build();
      stoppingMachine = DynamicTasks.queue(stopMachineTask);

      DynamicTasks.drain(entity().getConfig(STOP_PROCESS_TIMEOUT), false);

      // shutdown the machine if stopping process fails or takes too long
      synchronized (stoppingMachine) {
        // task also used as mutex by DST when it submits it; ensure it only submits once!
        if (!stoppingMachine.isSubmitted()) {
          // force the stoppingMachine task to run by submitting it here
          StringBuilder msg =
              new StringBuilder("Submitting machine stop early in background for ")
                  .append(entity());
          if (stoppingProcess == null) {
            msg.append(". Process stop skipped, pre-stop not finished?");
          } else {
            msg.append(" because process stop has ")
                .append((stoppingProcess.isDone() ? "finished abnormally" : "not finished"));
          }
          log.warn(msg.toString());
          Entities.submit(entity(), stoppingMachine);
        }
      }
    }

    try {
      // This maintains previous behaviour of silently squashing any errors on the stoppingProcess
      // task if the
      // stoppingMachine exits with a nonzero value
      boolean checkStopProcesses =
          (stoppingProcess != null
              && (stoppingMachine == null || stoppingMachine.get().value == 0));

      if (checkStopProcesses) {
        // TODO we should test for destruction above, not merely successful "stop", as things like
        // localhost and ssh won't be destroyed
        DynamicTasks.waitForLast();
        if (machine.isPresent()) {
          // throw early errors *only if* there is a machine and we have not destroyed it
          stoppingProcess.get();
        }
      }
    } catch (Throwable e) {
      ServiceStateLogic.setExpectedState(entity(), Lifecycle.ON_FIRE);
      Exceptions.propagate(e);
    }
    entity().sensors().set(SoftwareProcess.SERVICE_UP, false);
    ServiceStateLogic.setExpectedState(entity(), Lifecycle.STOPPED);

    DynamicTasks.queue("post-stop", new PostStopCustomTask());

    if (log.isDebugEnabled()) log.debug("Stopped software process entity " + entity());
  }
  @SuppressWarnings("unchecked")
  @Override
  public void startReadOnly(final ManagementNodeState mode) {
    if (!ManagementNodeState.isHotProxy(mode)) {
      throw new IllegalStateException(
          "Read-only rebind thread only permitted for hot proxy modes; not " + mode);
    }

    if (persistenceRunning) {
      throw new IllegalStateException(
          "Cannot start read-only when already running with persistence");
    }
    if (readOnlyRunning || readOnlyTask != null) {
      LOG.warn(
          "Cannot request read-only mode for "
              + this
              + " when already running - "
              + readOnlyTask
              + "; ignoring");
      return;
    }
    LOG.debug(
        "Starting read-only rebinding ("
            + this
            + "), mgmt "
            + managementContext.getManagementNodeId());

    if (persistenceRealChangeListener != null) persistenceRealChangeListener.stop();
    if (persistenceStoreAccess != null) persistenceStoreAccess.disableWriteAccess(true);

    readOnlyRunning = true;
    readOnlyRebindCount.set(0);

    try {
      rebind(null, null, mode);
    } catch (Exception e) {
      throw Exceptions.propagate(e);
    }

    Callable<Task<?>> taskFactory =
        new Callable<Task<?>>() {
          @Override
          public Task<Void> call() {
            return Tasks.<Void>builder()
                .dynamic(false)
                .displayName("rebind (periodic run")
                .body(
                    new Callable<Void>() {
                      public Void call() {
                        try {
                          rebind(null, null, mode);
                          return null;
                        } catch (RuntimeInterruptedException e) {
                          LOG.debug("Interrupted rebinding (re-interrupting): " + e);
                          if (LOG.isTraceEnabled())
                            LOG.trace("Interrupted rebinding (re-interrupting), details: " + e, e);
                          Thread.currentThread().interrupt();
                          return null;
                        } catch (Exception e) {
                          // Don't rethrow: the behaviour of executionManager is different from a
                          // scheduledExecutorService,
                          // if we throw an exception, then our task will never get executed again
                          if (!readOnlyRunning) {
                            LOG.debug(
                                "Problem rebinding (read-only running has probably just been turned off): "
                                    + e);
                            if (LOG.isTraceEnabled()) {
                              LOG.trace(
                                  "Problem rebinding (read-only running has probably just been turned off), details: "
                                      + e,
                                  e);
                            }
                          } else {
                            LOG.error("Problem rebinding: " + Exceptions.collapseText(e), e);
                          }
                          return null;
                        } catch (Throwable t) {
                          LOG.warn("Problem rebinding (rethrowing)", t);
                          throw Exceptions.propagate(t);
                        }
                      }
                    })
                .build();
          }
        };
    readOnlyTask =
        (ScheduledTask)
            managementContext
                .getServerExecutionContext()
                .submit(
                    new ScheduledTask(
                            MutableMap.of("displayName", "Periodic read-only rebind"), taskFactory)
                        .period(periodicPersistPeriod));
  }
    @SuppressWarnings({"rawtypes", "unchecked"})
    @Override
    public V call() {
      T value = source.getAttribute(sensor);

      // return immediately if either the ready predicate or the abort conditions hold
      if (ready(value)) return postProcess(value);

      final List<Exception> abortionExceptions = Lists.newCopyOnWriteArrayList();
      long start = System.currentTimeMillis();

      for (AttributeAndSensorCondition abortCondition : abortSensorConditions) {
        Object abortValue = abortCondition.source.getAttribute(abortCondition.sensor);
        if (abortCondition.predicate.apply(abortValue)) {
          abortionExceptions.add(
              new Exception(
                  "Abort due to " + abortCondition.source + " -> " + abortCondition.sensor));
        }
      }
      if (abortionExceptions.size() > 0) {
        throw new CompoundRuntimeException(
            "Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
      }

      TaskInternal<?> current = (TaskInternal<?>) Tasks.current();
      if (current == null)
        throw new IllegalStateException("Should only be invoked in a running task");
      Entity entity = BrooklynTaskTags.getTargetOrContextEntity(current);
      if (entity == null)
        throw new IllegalStateException(
            "Should only be invoked in a running task with an entity tag; "
                + current
                + " has no entity tag ("
                + current.getStatusDetail(false)
                + ")");

      final LinkedList<T> publishedValues = new LinkedList<T>();
      final Semaphore semaphore = new Semaphore(0); // could use Exchanger
      SubscriptionHandle subscription = null;
      List<SubscriptionHandle> abortSubscriptions = Lists.newArrayList();

      try {
        subscription =
            entity
                .subscriptions()
                .subscribe(
                    source,
                    sensor,
                    new SensorEventListener<T>() {
                      @Override
                      public void onEvent(SensorEvent<T> event) {
                        synchronized (publishedValues) {
                          publishedValues.add(event.getValue());
                        }
                        semaphore.release();
                      }
                    });
        for (final AttributeAndSensorCondition abortCondition : abortSensorConditions) {
          abortSubscriptions.add(
              entity
                  .subscriptions()
                  .subscribe(
                      abortCondition.source,
                      abortCondition.sensor,
                      new SensorEventListener<Object>() {
                        @Override
                        public void onEvent(SensorEvent<Object> event) {
                          if (abortCondition.predicate.apply(event.getValue())) {
                            abortionExceptions.add(
                                new Exception(
                                    "Abort due to "
                                        + abortCondition.source
                                        + " -> "
                                        + abortCondition.sensor));
                            semaphore.release();
                          }
                        }
                      }));
          Object abortValue = abortCondition.source.getAttribute(abortCondition.sensor);
          if (abortCondition.predicate.apply(abortValue)) {
            abortionExceptions.add(
                new Exception(
                    "Abort due to " + abortCondition.source + " -> " + abortCondition.sensor));
          }
        }
        if (abortionExceptions.size() > 0) {
          throw new CompoundRuntimeException(
              "Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
        }

        CountdownTimer timer = timeout != null ? timeout.countdownTimer() : null;
        Duration maxPeriod = ValueResolver.PRETTY_QUICK_WAIT;
        Duration nextPeriod = ValueResolver.REAL_QUICK_PERIOD;
        while (true) {
          // check the source on initial run (could be done outside the loop)
          // and also (optionally) on each iteration in case it is more recent
          value = source.getAttribute(sensor);
          if (ready(value)) break;

          if (timer != null) {
            if (timer.getDurationRemaining().isShorterThan(nextPeriod)) {
              nextPeriod = timer.getDurationRemaining();
            }
            if (timer.isExpired()) {
              if (onTimeout.isPresent()) return onTimeout.get();
              throw new RuntimeTimeoutException("Unsatisfied after " + Duration.sinceUtc(start));
            }
          }

          String prevBlockingDetails = current.setBlockingDetails(blockingDetails);
          try {
            if (semaphore.tryAcquire(nextPeriod.toMilliseconds(), TimeUnit.MILLISECONDS)) {
              // immediately release so we are available for the next check
              semaphore.release();
              // if other permits have been made available (e.g. multiple notifications) drain them
              // all as no point running multiple times
              semaphore.drainPermits();
            }
          } finally {
            current.setBlockingDetails(prevBlockingDetails);
          }

          // check any subscribed values which have come in first
          while (true) {
            synchronized (publishedValues) {
              if (publishedValues.isEmpty()) break;
              value = publishedValues.pop();
            }
            if (ready(value)) break;
          }

          // if unmanaged then ignore the other abort conditions
          if (!ignoreUnmanaged && Entities.isNoLongerManaged(entity)) {
            if (onUnmanaged.isPresent()) return onUnmanaged.get();
            throw new NotManagedException(entity);
          }

          if (abortionExceptions.size() > 0) {
            throw new CompoundRuntimeException(
                "Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
          }

          nextPeriod = nextPeriod.times(2).upperBound(maxPeriod);
        }
        if (LOG.isDebugEnabled()) LOG.debug("Attribute-ready for {} in entity {}", sensor, source);
        return postProcess(value);
      } catch (InterruptedException e) {
        throw Exceptions.propagate(e);
      } finally {
        if (subscription != null) {
          entity.subscriptions().unsubscribe(subscription);
        }
        for (SubscriptionHandle handle : abortSubscriptions) {
          entity.subscriptions().unsubscribe(handle);
        }
      }
    }