@Test(groups = "Integration") // Because slow
  public void testRecoversThenDownUpResetsStabilisationCount() throws Exception {
    final long stabilisationDelay = 1000;

    e1.enrichers()
        .add(
            EnricherSpec.create(ServiceFailureDetector.class)
                .configure(
                    ServiceFailureDetector.ENTITY_RECOVERED_STABILIZATION_DELAY,
                    Duration.of(stabilisationDelay)));

    e1.sensors().set(TestEntity.SERVICE_UP, false);
    assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
    events.clear();

    e1.sensors().set(TestEntity.SERVICE_UP, true);
    assertNoEventsContinually(Duration.of(stabilisationDelay - OVERHEAD));

    e1.sensors().set(TestEntity.SERVICE_UP, false);
    Thread.sleep(OVERHEAD);
    e1.sensors().set(TestEntity.SERVICE_UP, true);
    assertNoEventsContinually(Duration.of(stabilisationDelay - OVERHEAD));

    assertHasEventEventually(HASensors.ENTITY_RECOVERED, Predicates.<Object>equalTo(e1), null);
  }
Пример #2
0
  @Override
  public void connectSensors() {
    super.connectSensors();
    connectServiceUpIsRunning();

    HostAndPort hostAndPort =
        BrooklynAccessUtils.getBrooklynAccessibleAddress(this, sensors().get(DOCKER_REGISTRY_PORT));
    sensors().set(Attributes.MAIN_URI, URI.create("https://" + hostAndPort + "/v2"));

    httpFeed =
        HttpFeed.builder()
            .entity(this)
            .period(Duration.seconds(3))
            .baseUri(getAttribute(Attributes.MAIN_URI))
            .poll(
                new HttpPollConfig<Boolean>(Attributes.SERVICE_UP)
                    .onSuccess(Functions.constant(true))
                    .onFailureOrException(Functions.constant(false)))
            .poll(
                new HttpPollConfig<List<String>>(DOCKER_REGISTRY_CATALOG)
                    .suburl("/_catalog")
                    .onSuccess(
                        Functionals.chain(
                            HttpValueFunctions.jsonContents(),
                            JsonFunctions.walk("repositories"),
                            JsonFunctions.forEach(JsonFunctions.cast(String.class))))
                    .onFailureOrException(Functions.constant(Collections.<String>emptyList())))
            .build();
  }
  @Test(groups = "Integration") // Because slow
  public void testNotifiedOfFailureAfterStabilisationDelay() throws Exception {
    final int stabilisationDelay = 1000;

    e1.enrichers()
        .add(
            EnricherSpec.create(ServiceFailureDetector.class)
                .configure(
                    ServiceFailureDetector.ENTITY_FAILED_STABILIZATION_DELAY,
                    Duration.of(stabilisationDelay)));

    e1.sensors().set(TestEntity.SERVICE_UP, false);

    assertNoEventsContinually(Duration.of(stabilisationDelay - OVERHEAD));
    assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
  }
 @Override
 public Cidr get() {
   Cidr local = cidr;
   if (local == null) {
     synchronized (this) {
       local = cidr;
       if (local == null) {
         String externalIp = LocalhostExternalIpLoader.getLocalhostIpWithin(Duration.seconds(5));
         cidr = local = new Cidr(externalIp + "/32");
       }
     }
   }
   return local;
 }
  @Override
  public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
    String provider = checkNotNull(conf.get(CLOUD_PROVIDER), "provider must not be null");
    String identity =
        checkNotNull(conf.get(CloudLocationConfig.ACCESS_IDENTITY), "identity must not be null");
    String credential =
        checkNotNull(
            conf.get(CloudLocationConfig.ACCESS_CREDENTIAL), "credential must not be null");

    Properties properties = new Properties();
    properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
    properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
    properties.setProperty(
        "jclouds.ssh.max-retries",
        conf.getStringKey("jclouds.ssh.max-retries") != null
            ? conf.getStringKey("jclouds.ssh.max-retries").toString()
            : "50");
    // Enable aws-ec2 lazy image fetching, if given a specific imageId; otherwise customize for
    // specific owners; or all as a last resort
    // See https://issues.apache.org/jira/browse/WHIRR-416
    if ("aws-ec2".equals(provider)) {
      // TODO convert AWS-only flags to config keys
      if (groovyTruth(conf.get(IMAGE_ID))) {
        properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
        properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
      } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
        properties.setProperty(
            PROPERTY_EC2_AMI_QUERY,
            "owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
      } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
        // set `anyOwner: true` to override the default query (which is restricted to certain owners
        // as per below),
        // allowing the AMI query to bind to any machine
        // (note however, we sometimes pick defaults in JcloudsLocationFactory);
        // (and be careful, this can give a LOT of data back, taking several minutes,
        // and requiring extra memory allocated on the command-line)
        properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
        /*
         * by default the following filters are applied:
         * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
         * Filter.1.Value.2=063491364108&
         * Filter.1.Value.3=099720109477&
         * Filter.1.Value.4=411009282317&
         * Filter.2.Name=state&Filter.2.Value.1=available&
         * Filter.3.Name=image-type&Filter.3.Value.1=machine&
         */
      }

      // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException:
      // java.lang.RuntimeException:
      //     security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm
      // is not available after creating
      // the default timeout was 500ms so let's raise it in case that helps
      properties.setProperty(
          EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT,
          "" + Duration.seconds(30).toMilliseconds());
    }

    // FIXME Deprecated mechanism, should have a ConfigKey for overrides
    Map<String, Object> extra =
        Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
    if (extra.size() > 0) {
      LOG.warn("Jclouds using deprecated property overrides: " + Sanitizer.sanitize(extra));
    }
    properties.putAll(extra);

    String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
    if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
    if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);

    Map<?, ?> cacheKey =
        MutableMap.builder()
            .putAll(properties)
            .put("provider", provider)
            .put("identity", identity)
            .put("credential", credential)
            .putIfNotNull("endpoint", endpoint)
            .build()
            .asUnmodifiable();

    if (allowReuse) {
      ComputeService result = cachedComputeServices.get(cacheKey);
      if (result != null) {
        LOG.trace(
            "jclouds ComputeService cache hit for compute service, for "
                + Sanitizer.sanitize(properties));
        return result;
      }
      LOG.debug(
          "jclouds ComputeService cache miss for compute service, creating, for "
              + Sanitizer.sanitize(properties));
    }

    Iterable<Module> modules = getCommonModules();

    // Synchronizing to avoid deadlock from sun.reflect.annotation.AnnotationType.
    // See https://github.com/brooklyncentral/brooklyn/issues/974
    ComputeServiceContext computeServiceContext;
    synchronized (createComputeServicesMutex) {
      computeServiceContext =
          ContextBuilder.newBuilder(provider)
              .modules(modules)
              .credentials(identity, credential)
              .overrides(properties)
              .build(ComputeServiceContext.class);
    }
    final ComputeService computeService = computeServiceContext.getComputeService();
    if (allowReuse) {
      synchronized (cachedComputeServices) {
        ComputeService result = cachedComputeServices.get(cacheKey);
        if (result != null) {
          LOG.debug(
              "jclouds ComputeService cache recovery for compute service, for "
                  + Sanitizer.sanitize(cacheKey));
          // keep the old one, discard the new one
          computeService.getContext().close();
          return result;
        }
        LOG.debug(
            "jclouds ComputeService created "
                + computeService
                + ", adding to cache, for "
                + Sanitizer.sanitize(properties));
        cachedComputeServices.put(cacheKey, computeService);
      }
    }
    return computeService;
  }
/**
 * Default skeleton for start/stop/restart tasks on machines.
 *
 * <p>Knows how to provision machines, making use of {@link
 * ProvidesProvisioningFlags#obtainProvisioningFlags(MachineProvisioningLocation)}, and provides
 * hooks for injecting behaviour at common places.
 *
 * <p>Methods are designed for overriding, with the convention that *Async methods should queue (and
 * not block). The following methods are commonly overridden (and you can safely queue tasks, block,
 * or return immediately in them):
 *
 * <ul>
 *   <li>{@link #startProcessesAtMachine(Supplier)} (required)
 *   <li>{@link #stopProcessesAtMachine()} (required, but can be left blank if you assume the VM
 *       will be destroyed)
 *   <li>{@link #preStartCustom(MachineLocation)}
 *   <li>{@link #postStartCustom()}
 *   <li>{@link #preStopCustom()}
 *   <li>{@link #postStopCustom()}
 * </ul>
 *
 * Note methods at this level typically look after the {@link Attributes#SERVICE_STATE_ACTUAL}
 * sensor.
 *
 * @since 0.6.0
 */
@Beta
public abstract class MachineLifecycleEffectorTasks {

  private static final Logger log = LoggerFactory.getLogger(MachineLifecycleEffectorTasks.class);

  public static final ConfigKey<Boolean> ON_BOX_BASE_DIR_RESOLVED =
      ConfigKeys.newBooleanConfigKey(
          "onbox.base.dir.resolved",
          "Whether the on-box base directory has been resolved (for internal use)");

  public static final ConfigKey<Collection<? extends Location>> LOCATIONS =
      StartParameters.LOCATIONS;

  public static final ConfigKey<Duration> STOP_PROCESS_TIMEOUT =
      ConfigKeys.newDurationConfigKey(
          "process.stop.timeout",
          "How long to wait for the processes to be stopped; use null to mean forever",
          Duration.TWO_MINUTES);

  @Beta
  public static final ConfigKey<Duration> STOP_WAIT_PROVISIONING_TIMEOUT =
      ConfigKeys.newDurationConfigKey(
          "stop.wait.provisioning.timeout",
          "If stop is called on an entity while it is still provisioning the machine (such that "
              + "the provisioning cannot be safely interrupted), this is the length of time "
              + "to wait for the machine instance to become available so that it can be terminated. "
              + "If stop aborts before this point, the machine may be left running.",
          Duration.minutes(10));

  @Beta
  public static final AttributeSensor<MachineLocation> INTERNAL_PROVISIONED_MACHINE =
      new BasicAttributeSensor<MachineLocation>(
          TypeToken.of(MachineLocation.class),
          "internal.provisioning.task.machine",
          "Internal transient sensor (do not use) for tracking the machine being provisioned (to better handle aborting)",
          AttributeSensor.SensorPersistenceMode.NONE);

  protected final MachineInitTasks machineInitTasks = new MachineInitTasks();

  /** Attaches lifecycle effectors (start, restart, stop) to the given entity post-creation. */
  public void attachLifecycleEffectors(Entity entity) {
    ((EntityInternal) entity).getMutableEntityType().addEffector(newStartEffector());
    ((EntityInternal) entity).getMutableEntityType().addEffector(newRestartEffector());
    ((EntityInternal) entity).getMutableEntityType().addEffector(newStopEffector());
  }

  /**
   * Return an effector suitable for setting in a {@code public static final} or attaching
   * dynamically.
   *
   * <p>The effector overrides the corresponding effector from {@link Startable} with the behaviour
   * in this lifecycle class instance.
   */
  public Effector<Void> newStartEffector() {
    return Effectors.effector(Startable.START).impl(newStartEffectorTask()).build();
  }

  /** @see {@link #newStartEffector()} */
  public Effector<Void> newRestartEffector() {
    return Effectors.effector(Startable.RESTART)
        .parameter(RestartSoftwareParameters.RESTART_CHILDREN)
        .parameter(RestartSoftwareParameters.RESTART_MACHINE)
        .impl(newRestartEffectorTask())
        .build();
  }

  /** @see {@link #newStartEffector()} */
  public Effector<Void> newStopEffector() {
    return Effectors.effector(Startable.STOP)
        .parameter(StopSoftwareParameters.STOP_PROCESS_MODE)
        .parameter(StopSoftwareParameters.STOP_MACHINE_MODE)
        .impl(newStopEffectorTask())
        .build();
  }

  /** @see {@link #newStartEffector()} */
  public Effector<Void> newSuspendEffector() {
    return Effectors.effector(Void.class, "suspend")
        .description("Suspend the process/service represented by an entity")
        .parameter(StopSoftwareParameters.STOP_PROCESS_MODE)
        .parameter(StopSoftwareParameters.STOP_MACHINE_MODE)
        .impl(newSuspendEffectorTask())
        .build();
  }

  /**
   * Returns the {@link EffectorBody} which supplies the implementation for the start effector.
   *
   * <p>Calls {@link #start(Collection)} in this class.
   */
  public EffectorBody<Void> newStartEffectorTask() {
    // TODO included anonymous inner class for backwards compatibility with persisted state.
    new EffectorBody<Void>() {
      @Override
      public Void call(ConfigBag parameters) {
        Collection<? extends Location> locations = null;

        Object locationsRaw = parameters.getStringKey(LOCATIONS.getName());
        locations =
            Locations.coerceToCollectionOfLocationsManaged(
                entity().getManagementContext(), locationsRaw);

        if (locations == null) {
          // null/empty will mean to inherit from parent
          locations = Collections.emptyList();
        }

        start(locations);
        return null;
      }
    };
    return new StartEffectorBody();
  }

  private class StartEffectorBody extends EffectorBody<Void> {
    @Override
    public Void call(ConfigBag parameters) {
      Collection<? extends Location> locations = null;

      Object locationsRaw = parameters.getStringKey(LOCATIONS.getName());
      locations =
          Locations.coerceToCollectionOfLocationsManaged(
              entity().getManagementContext(), locationsRaw);

      if (locations == null) {
        // null/empty will mean to inherit from parent
        locations = Collections.emptyList();
      }

      start(locations);
      return null;
    }
  }

  /**
   * Calls {@link #restart(ConfigBag)}.
   *
   * @see {@link #newStartEffectorTask()}
   */
  public EffectorBody<Void> newRestartEffectorTask() {
    // TODO included anonymous inner class for backwards compatibility with persisted state.
    new EffectorBody<Void>() {
      @Override
      public Void call(ConfigBag parameters) {
        restart(parameters);
        return null;
      }
    };
    return new RestartEffectorBody();
  }

  private class RestartEffectorBody extends EffectorBody<Void> {
    @Override
    public Void call(ConfigBag parameters) {
      restart(parameters);
      return null;
    }
  }

  /**
   * Calls {@link #stop(ConfigBag)}.
   *
   * @see {@link #newStartEffectorTask()}
   */
  public EffectorBody<Void> newStopEffectorTask() {
    // TODO included anonymous inner class for backwards compatibility with persisted state.
    new EffectorBody<Void>() {
      @Override
      public Void call(ConfigBag parameters) {
        stop(parameters);
        return null;
      }
    };
    return new StopEffectorBody();
  }

  private class StopEffectorBody extends EffectorBody<Void> {
    @Override
    public Void call(ConfigBag parameters) {
      stop(parameters);
      return null;
    }
  }

  /**
   * Calls {@link #suspend(ConfigBag)}.
   *
   * @see {@link #newStartEffectorTask()}
   */
  public EffectorBody<Void> newSuspendEffectorTask() {
    return new SuspendEffectorBody();
  }

  private class SuspendEffectorBody extends EffectorBody<Void> {
    @Override
    public Void call(ConfigBag parameters) {
      suspend(parameters);
      return null;
    }
  }

  protected EntityInternal entity() {
    return (EntityInternal) BrooklynTaskTags.getTargetOrContextEntity(Tasks.current());
  }

  protected Location getLocation(@Nullable Collection<? extends Location> locations) {
    if (locations == null || locations.isEmpty()) locations = entity().getLocations();
    if (locations.isEmpty()) {
      MachineProvisioningLocation<?> provisioner =
          entity().getAttribute(SoftwareProcess.PROVISIONING_LOCATION);
      if (provisioner != null) locations = Arrays.<Location>asList(provisioner);
    }
    locations = Locations.getLocationsCheckingAncestors(locations, entity());

    Maybe<MachineLocation> ml = Locations.findUniqueMachineLocation(locations);
    if (ml.isPresent()) return ml.get();

    if (locations.isEmpty())
      throw new IllegalArgumentException("No locations specified when starting " + entity());
    if (locations.size() != 1 || Iterables.getOnlyElement(locations) == null)
      throw new IllegalArgumentException(
          "Ambiguous locations detected when starting " + entity() + ": " + locations);
    return Iterables.getOnlyElement(locations);
  }

  /**
   * runs the tasks needed to start, wrapped by setting {@link Attributes#SERVICE_STATE_EXPECTED}
   * appropriately
   */
  public void start(Collection<? extends Location> locations) {
    ServiceStateLogic.setExpectedState(entity(), Lifecycle.STARTING);
    try {
      startInLocations(locations);
      DynamicTasks.waitForLast();
      ServiceStateLogic.setExpectedState(entity(), Lifecycle.RUNNING);
    } catch (Throwable t) {
      ServiceStateLogic.setExpectedState(entity(), Lifecycle.ON_FIRE);
      throw Exceptions.propagate(t);
    }
  }

  /**
   * Asserts there is a single location and calls {@link #startInLocation(Location)} with that
   * location.
   */
  protected void startInLocations(Collection<? extends Location> locations) {
    startInLocation(getLocation(locations));
  }

  /** Dispatches to the appropriate method(s) to start in the given location. */
  protected void startInLocation(final Location location) {
    Supplier<MachineLocation> locationS = null;
    if (location instanceof MachineProvisioningLocation) {
      Task<MachineLocation> machineTask = provisionAsync((MachineProvisioningLocation<?>) location);
      locationS = Tasks.supplier(machineTask);
    } else if (location instanceof MachineLocation) {
      locationS = Suppliers.ofInstance((MachineLocation) location);
    }
    Preconditions.checkState(
        locationS != null, "Unsupported location " + location + ", when starting " + entity());

    final Supplier<MachineLocation> locationSF = locationS;
    preStartAtMachineAsync(locationSF);
    DynamicTasks.queue("start (processes)", new StartProcessesAtMachineTask(locationSF));
    postStartAtMachineAsync();
  }

  private class StartProcessesAtMachineTask implements Runnable {
    private final Supplier<MachineLocation> machineSupplier;

    private StartProcessesAtMachineTask(Supplier<MachineLocation> machineSupplier) {
      this.machineSupplier = machineSupplier;
    }

    @Override
    public void run() {
      startProcessesAtMachine(machineSupplier);
    }
  }

  /**
   * Returns a queued {@link Task} which provisions a machine in the given location and returns that
   * machine. The task can be used as a supplier to subsequent methods.
   */
  protected Task<MachineLocation> provisionAsync(final MachineProvisioningLocation<?> location) {
    return DynamicTasks.queue(
        Tasks.<MachineLocation>builder()
            .displayName("provisioning (" + location.getDisplayName() + ")")
            .body(new ProvisionMachineTask(location))
            .build());
  }

  private class ProvisionMachineTask implements Callable<MachineLocation> {
    final MachineProvisioningLocation<?> location;

    private ProvisionMachineTask(MachineProvisioningLocation<?> location) {
      this.location = location;
    }

    public MachineLocation call() throws Exception {
      // Blocks if a latch was configured.
      entity().getConfig(BrooklynConfigKeys.PROVISION_LATCH);
      final Map<String, Object> flags = obtainProvisioningFlags(location);
      if (!(location instanceof LocalhostMachineProvisioningLocation))
        log.info(
            "Starting {}, obtaining a new location instance in {} with ports {}",
            new Object[] {entity(), location, flags.get("inboundPorts")});
      entity().sensors().set(SoftwareProcess.PROVISIONING_LOCATION, location);
      Transition expectedState = entity().sensors().get(Attributes.SERVICE_STATE_EXPECTED);

      // BROOKLYN-263: see corresponding code in doStop()
      if (expectedState != null
          && (expectedState.getState() == Lifecycle.STOPPING
              || expectedState.getState() == Lifecycle.STOPPED)) {
        throw new IllegalStateException(
            "Provisioning aborted before even begun for "
                + entity()
                + " in "
                + location
                + " (presumably by a concurrent call to stop");
      }
      entity()
          .sensors()
          .set(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE, ProvisioningTaskState.RUNNING);

      MachineLocation machine;
      try {
        machine =
            Tasks.withBlockingDetails(
                "Provisioning machine in " + location, new ObtainLocationTask(location, flags));
        entity().sensors().set(INTERNAL_PROVISIONED_MACHINE, machine);
      } finally {
        entity().sensors().remove(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
      }

      if (machine == null) {
        throw new NoMachinesAvailableException(
            "Failed to obtain machine in " + location.toString());
      }
      if (log.isDebugEnabled()) {
        log.debug(
            "While starting {}, obtained new location instance {}",
            entity(),
            (machine instanceof SshMachineLocation
                ? machine
                    + ", details "
                    + ((SshMachineLocation) machine).getUser()
                    + ":"
                    + Sanitizer.sanitize(((SshMachineLocation) machine).config().getLocalBag())
                : machine));
      }
      return machine;
    }
  }

  private static class ObtainLocationTask implements Callable<MachineLocation> {
    final MachineProvisioningLocation<?> location;
    final Map<String, Object> flags;

    private ObtainLocationTask(MachineProvisioningLocation<?> location, Map<String, Object> flags) {
      this.flags = flags;
      this.location = location;
    }

    public MachineLocation call() throws NoMachinesAvailableException {
      return location.obtain(flags);
    }
  }

  /**
   * Wraps a call to {@link #preStartCustom(MachineLocation)}, after setting the hostname and
   * address.
   */
  protected void preStartAtMachineAsync(final Supplier<MachineLocation> machineS) {
    DynamicTasks.queue("pre-start", new PreStartTask(machineS.get()));
  }

  private class PreStartTask implements Runnable {
    final MachineLocation machine;

    private PreStartTask(MachineLocation machine) {
      this.machine = machine;
    }

    public void run() {
      log.info("Starting {} on machine {}", entity(), machine);
      Collection<Location> oldLocs = entity().getLocations();
      if (!oldLocs.isEmpty()) {
        List<MachineLocation> oldSshLocs =
            ImmutableList.copyOf(Iterables.filter(oldLocs, MachineLocation.class));
        if (!oldSshLocs.isEmpty()) {
          // check if existing locations are compatible
          log.debug(
              "Entity "
                  + entity()
                  + " had machine locations "
                  + oldSshLocs
                  + " when starting at "
                  + machine
                  + "; checking if they are compatible");
          for (MachineLocation oldLoc : oldSshLocs) {
            // machines are deemed compatible if hostname and address are the same, or they are
            // localhost
            // this allows a machine create by jclouds to then be defined with an ip-based spec
            if (!"localhost".equals(machine.getConfig(AbstractLocation.ORIGINAL_SPEC))) {
              checkLocationParametersCompatible(
                  machine,
                  oldLoc,
                  "hostname",
                  oldLoc.getAddress().getHostName(),
                  machine.getAddress().getHostName());
              checkLocationParametersCompatible(
                  machine,
                  oldLoc,
                  "address",
                  oldLoc.getAddress().getHostAddress(),
                  machine.getAddress().getHostAddress());
            }
          }
          log.debug(
              "Entity "
                  + entity()
                  + " old machine locations "
                  + oldSshLocs
                  + " were compatible, removing them to start at "
                  + machine);
          entity().removeLocations(oldSshLocs);
        }
      }
      entity().addLocations(ImmutableList.of((Location) machine));

      // elsewhere we rely on (public) hostname being set _after_ subnet_hostname
      // (to prevent the tiny possibility of races resulting in hostname being returned
      // simply because subnet is still being looked up)
      Maybe<String> lh = Machines.getSubnetHostname(machine);
      Maybe<String> la = Machines.getSubnetIp(machine);
      if (lh.isPresent()) entity().sensors().set(Attributes.SUBNET_HOSTNAME, lh.get());
      if (la.isPresent()) entity().sensors().set(Attributes.SUBNET_ADDRESS, la.get());
      entity().sensors().set(Attributes.HOSTNAME, machine.getAddress().getHostName());
      entity().sensors().set(Attributes.ADDRESS, machine.getAddress().getHostAddress());
      if (machine instanceof SshMachineLocation) {
        SshMachineLocation sshMachine = (SshMachineLocation) machine;
        UserAndHostAndPort sshAddress =
            UserAndHostAndPort.fromParts(
                sshMachine.getUser(), sshMachine.getAddress().getHostName(), sshMachine.getPort());
        // FIXME: Who or what is SSH_ADDRESS intended for? It's not necessarily the address that
        // the SshMachineLocation is using for ssh connections (because it accepts SSH_HOST as an
        // override).
        entity().sensors().set(Attributes.SSH_ADDRESS, sshAddress);
      }

      if (Boolean.TRUE.equals(entity().getConfig(SoftwareProcess.OPEN_IPTABLES))) {
        if (machine instanceof SshMachineLocation) {
          @SuppressWarnings("unchecked")
          Iterable<Integer> inboundPorts =
              (Iterable<Integer>) machine.config().get(CloudLocationConfig.INBOUND_PORTS);
          machineInitTasks.openIptablesAsync(inboundPorts, (SshMachineLocation) machine);
        } else {
          log.warn("Ignoring flag OPEN_IPTABLES on non-ssh location {}", machine);
        }
      }
      if (Boolean.TRUE.equals(entity().getConfig(SoftwareProcess.STOP_IPTABLES))) {
        if (machine instanceof SshMachineLocation) {
          machineInitTasks.stopIptablesAsync((SshMachineLocation) machine);
        } else {
          log.warn("Ignoring flag STOP_IPTABLES on non-ssh location {}", machine);
        }
      }
      if (Boolean.TRUE.equals(entity().getConfig(SoftwareProcess.DONT_REQUIRE_TTY_FOR_SUDO))) {
        if (machine instanceof SshMachineLocation) {
          machineInitTasks.dontRequireTtyForSudoAsync((SshMachineLocation) machine);
        } else {
          log.warn("Ignoring flag DONT_REQUIRE_TTY_FOR_SUDO on non-ssh location {}", machine);
        }
      }
      resolveOnBoxDir(entity(), machine);
      preStartCustom(machine);
    }
  }

  /**
   * Resolves the on-box dir.
   *
   * <p>Initialize and pre-create the right onbox working dir, if an ssh machine location. Logs a
   * warning if not.
   */
  @SuppressWarnings("deprecation")
  public static String resolveOnBoxDir(EntityInternal entity, MachineLocation machine) {
    String base = entity.getConfig(BrooklynConfigKeys.ONBOX_BASE_DIR);
    if (base == null) base = machine.getConfig(BrooklynConfigKeys.ONBOX_BASE_DIR);
    if (base != null && Boolean.TRUE.equals(entity.getConfig(ON_BOX_BASE_DIR_RESOLVED)))
      return base;
    if (base == null)
      base = entity.getManagementContext().getConfig().getConfig(BrooklynConfigKeys.ONBOX_BASE_DIR);
    if (base == null) base = entity.getConfig(BrooklynConfigKeys.BROOKLYN_DATA_DIR);
    if (base == null) base = machine.getConfig(BrooklynConfigKeys.BROOKLYN_DATA_DIR);
    if (base == null)
      base =
          entity.getManagementContext().getConfig().getConfig(BrooklynConfigKeys.BROOKLYN_DATA_DIR);
    if (base == null) base = "~/brooklyn-managed-processes";
    if (base.equals("~")) base = ".";
    if (base.startsWith("~/")) base = "." + base.substring(1);

    String resolvedBase = null;
    if (entity.getConfig(BrooklynConfigKeys.SKIP_ON_BOX_BASE_DIR_RESOLUTION)
        || machine.getConfig(BrooklynConfigKeys.SKIP_ON_BOX_BASE_DIR_RESOLUTION)) {
      if (log.isDebugEnabled())
        log.debug("Skipping on-box base dir resolution for " + entity + " at " + machine);
      if (!Os.isAbsolutish(base)) base = "~/" + base;
      resolvedBase = Os.tidyPath(base);
    } else if (machine instanceof SshMachineLocation) {
      SshMachineLocation ms = (SshMachineLocation) machine;
      ProcessTaskWrapper<Integer> baseTask =
          SshEffectorTasks.ssh(
                  BashCommands.alternatives(
                      "mkdir -p \"${BASE_DIR}\"",
                      BashCommands.chain(
                          BashCommands.sudo("mkdir -p \"${BASE_DIR}\""),
                          BashCommands.sudo("chown " + ms.getUser() + " \"${BASE_DIR}\""))),
                  "cd ~",
                  "cd ${BASE_DIR}",
                  "echo BASE_DIR_RESULT':'`pwd`:BASE_DIR_RESULT")
              .environmentVariable("BASE_DIR", base)
              .requiringExitCodeZero()
              .summary("initializing on-box base dir " + base)
              .newTask();
      DynamicTasks.queueIfPossible(baseTask).orSubmitAsync(entity);
      resolvedBase =
          Strings.getFragmentBetween(
              baseTask.block().getStdout(), "BASE_DIR_RESULT:", ":BASE_DIR_RESULT");
    }
    if (resolvedBase == null) {
      if (!Os.isAbsolutish(base)) base = "~/" + base;
      resolvedBase = Os.tidyPath(base);
      log.warn(
          "Could not resolve on-box directory for "
              + entity
              + " at "
              + machine
              + "; using "
              + resolvedBase
              + ", though this may not be accurate at the target (and may fail shortly)");
    }
    entity.config().set(BrooklynConfigKeys.ONBOX_BASE_DIR, resolvedBase);
    entity.config().set(ON_BOX_BASE_DIR_RESOLVED, true);
    return resolvedBase;
  }

  protected void checkLocationParametersCompatible(
      MachineLocation oldLoc,
      MachineLocation newLoc,
      String paramSummary,
      Object oldParam,
      Object newParam) {
    if (oldParam == null || newParam == null || !oldParam.equals(newParam))
      throw new IllegalStateException(
          "Cannot start "
              + entity()
              + " in "
              + newLoc
              + " as it has already been started with incompatible location "
              + oldLoc
              + " "
              + "("
              + paramSummary
              + " not compatible: "
              + oldParam
              + " / "
              + newParam
              + "); "
              + newLoc
              + " may require manual removal.");
  }

  /**
   * Default pre-start hooks.
   *
   * <p>Can be extended by subclasses if needed.
   */
  protected void preStartCustom(MachineLocation machine) {
    ConfigToAttributes.apply(entity());

    // Opportunity to block startup until other dependent components are available
    Object val = entity().getConfig(SoftwareProcess.START_LATCH);
    if (val != null)
      log.debug("{} finished waiting for start-latch {}; continuing...", entity(), val);
  }

  protected Map<String, Object> obtainProvisioningFlags(
      final MachineProvisioningLocation<?> location) {
    if (entity() instanceof ProvidesProvisioningFlags) {
      return ((ProvidesProvisioningFlags) entity())
          .obtainProvisioningFlags(location)
          .getAllConfig();
    }
    return MutableMap.<String, Object>of();
  }

  protected abstract String startProcessesAtMachine(final Supplier<MachineLocation> machineS);

  protected void postStartAtMachineAsync() {
    DynamicTasks.queue("post-start", new PostStartTask());
  }

  private class PostStartTask implements Runnable {
    public void run() {
      postStartCustom();
    }
  }

  /**
   * Default post-start hooks.
   *
   * <p>Can be extended by subclasses, and typically will wait for confirmation of start. The
   * service not set to running until after this. Also invoked following a restart.
   */
  protected void postStartCustom() {
    // nothing by default
  }

  /**
   * whether when 'auto' mode is specified, the machine should be stopped when the restart effector
   * is called
   *
   * <p>with {@link MachineLifecycleEffectorTasks}, a machine will always get created on restart if
   * there wasn't one already (unlike certain subclasses which might attempt a shortcut
   * process-level restart) so there is no reason for default behaviour of restart to throw away a
   * provisioned machine, hence default impl returns <code>false</code>.
   *
   * <p>if it is possible to tell that a machine is unhealthy, or if {@link #restart(ConfigBag)} is
   * overridden, then it might be appropriate to return <code>true</code> here.
   */
  protected boolean getDefaultRestartStopsMachine() {
    return false;
  }

  /**
   * Default restart implementation for an entity.
   *
   * <p>Stops processes if possible, then starts the entity again.
   */
  public void restart(ConfigBag parameters) {
    ServiceStateLogic.setExpectedState(entity(), Lifecycle.STOPPING);

    RestartMachineMode isRestartMachine =
        parameters.get(RestartSoftwareParameters.RESTART_MACHINE_TYPED);
    if (isRestartMachine == null) isRestartMachine = RestartMachineMode.AUTO;
    if (isRestartMachine == RestartMachineMode.AUTO)
      isRestartMachine =
          getDefaultRestartStopsMachine() ? RestartMachineMode.TRUE : RestartMachineMode.FALSE;

    // Calling preStopCustom without a corresponding postStopCustom invocation
    // doesn't look right so use a separate callback pair; Also depending on the arguments
    // stop() could be called which will call the {pre,post}StopCustom on its own.
    DynamicTasks.queue("pre-restart", new PreRestartTask());

    if (isRestartMachine == RestartMachineMode.FALSE) {
      DynamicTasks.queue("stopping (process)", new StopProcessesAtMachineTask());
    } else {
      Map<String, Object> stopMachineFlags = MutableMap.of();
      if (Entitlements.getEntitlementContext() != null) {
        stopMachineFlags.put(
            "tags",
            MutableSet.of(
                BrooklynTaskTags.tagForEntitlement(Entitlements.getEntitlementContext())));
      }
      Task<String> stopTask =
          Tasks.<String>builder()
              .displayName("stopping (machine)")
              .body(new StopMachineTask())
              .flags(stopMachineFlags)
              .build();
      DynamicTasks.queue(stopTask);
    }

    DynamicTasks.queue("starting", new StartInLocationsTask());
    restartChildren(parameters);
    DynamicTasks.queue("post-restart", new PostRestartTask());

    DynamicTasks.waitForLast();
    ServiceStateLogic.setExpectedState(entity(), Lifecycle.RUNNING);
  }

  private class PreRestartTask implements Runnable {
    @Override
    public void run() {
      preRestartCustom();
    }
  }

  private class PostRestartTask implements Runnable {
    @Override
    public void run() {
      postRestartCustom();
    }
  }

  private class StartInLocationsTask implements Runnable {
    @Override
    public void run() {
      // startInLocations will look up the location, and provision a machine if necessary
      // (if it remembered the provisioning location)
      ServiceStateLogic.setExpectedState(entity(), Lifecycle.STARTING);
      startInLocations(null);
    }
  }

  protected void restartChildren(ConfigBag parameters) {
    // TODO should we consult ChildStartableMode?

    Boolean isRestartChildren = parameters.get(RestartSoftwareParameters.RESTART_CHILDREN);
    if (isRestartChildren == null || !isRestartChildren) {
      return;
    }

    if (isRestartChildren) {
      DynamicTasks.queue(StartableMethods.restartingChildren(entity(), parameters));
      return;
    }

    throw new IllegalArgumentException(
        "Invalid value '"
            + isRestartChildren
            + "' for "
            + RestartSoftwareParameters.RESTART_CHILDREN.getName());
  }

  /**
   * Default stop implementation for an entity.
   *
   * <p>Aborts if already stopped, otherwise sets state {@link Lifecycle#STOPPING} then invokes
   * {@link #preStopCustom()}, {@link #stopProcessesAtMachine()}, then finally {@link
   * #stopAnyProvisionedMachines()} and sets state {@link Lifecycle#STOPPED}. If no errors were
   * encountered call {@link #postStopCustom()} at the end.
   */
  public void stop(ConfigBag parameters) {
    doStop(parameters, new StopAnyProvisionedMachinesTask());
  }

  /**
   * As {@link #stop} but calling {@link #suspendAnyProvisionedMachines} rather than {@link
   * #stopAnyProvisionedMachines}.
   */
  public void suspend(ConfigBag parameters) {
    doStop(parameters, new SuspendAnyProvisionedMachinesTask());
  }

  protected void doStop(ConfigBag parameters, Callable<StopMachineDetails<Integer>> stopTask) {
    preStopConfirmCustom();

    log.info("Stopping {} in {}", entity(), entity().getLocations());

    StopMode stopMachineMode = getStopMachineMode(parameters);
    StopMode stopProcessMode = parameters.get(StopSoftwareParameters.STOP_PROCESS_MODE);

    DynamicTasks.queue("pre-stop", new PreStopCustomTask());

    // BROOKLYN-263:
    // With this change the stop effector will wait for Location to provision so it can terminate
    // the machine, if a provisioning request is in-progress.
    //
    // The ProvisionMachineTask stores transient internal state in PROVISIONING_TASK_STATE and
    // PROVISIONED_MACHINE: it records when the provisioning is running and when done; and it
    // records the final machine. We record the machine in the internal sensor (rather than
    // just relying on getLocations) because the latter is set much later in the start()
    // process.
    //
    // This code is a big improvement (previously there was a several-minute window in some
    // clouds where a call to stop() would leave the machine running).
    //
    // However, there are still races. If the start() code has not yet reached the call to
    // location.obtain() then we won't wait, and the start() call won't know to abort. It's
    // fiddly to get that right, because we need to cope with restart() - so we mustn't leave
    // any state behind that will interfere with subsequent sequential calls to start().
    // There is some attempt to handle it by ProvisionMachineTask checking if the expectedState
    // is stopping/stopped.
    Maybe<MachineLocation> machine = Machines.findUniqueMachineLocation(entity().getLocations());
    ProvisioningTaskState provisioningState =
        entity().sensors().get(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);

    if (machine.isAbsent() && provisioningState == ProvisioningTaskState.RUNNING) {
      Duration maxWait = entity().config().get(STOP_WAIT_PROVISIONING_TIMEOUT);
      log.info(
          "When stopping {}, waiting for up to {} for the machine to finish provisioning, before terminating it",
          entity(),
          maxWait);
      boolean success =
          Repeater.create("Wait for a machine to appear")
              .until(
                  new Callable<Boolean>() {
                    @Override
                    public Boolean call() throws Exception {
                      ProvisioningTaskState state =
                          entity()
                              .sensors()
                              .get(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
                      return (state != ProvisioningTaskState.RUNNING);
                    }
                  })
              .backoffTo(Duration.FIVE_SECONDS)
              .limitTimeTo(maxWait)
              .run();
      if (!success) {
        log.warn(
            "When stopping {}, timed out after {} waiting for the machine to finish provisioning - machine may we left running",
            entity(),
            maxWait);
      }
      machine = Maybe.ofDisallowingNull(entity().sensors().get(INTERNAL_PROVISIONED_MACHINE));
    }
    entity().sensors().remove(AttributesInternal.INTERNAL_PROVISIONING_TASK_STATE);
    entity().sensors().remove(INTERNAL_PROVISIONED_MACHINE);

    Task<List<?>> stoppingProcess = null;
    if (canStop(stopProcessMode, entity())) {
      stoppingProcess =
          Tasks.parallel(
              "stopping",
              Tasks.create("stopping (process)", new StopProcessesAtMachineTask()),
              Tasks.create("stopping (feeds)", new StopFeedsAtMachineTask()));
      DynamicTasks.queue(stoppingProcess);
    }

    Task<StopMachineDetails<Integer>> stoppingMachine = null;
    if (canStop(stopMachineMode, machine.isAbsent())) {
      // Release this machine (even if error trying to stop process - we rethrow that after)
      Map<String, Object> stopMachineFlags = MutableMap.of();
      if (Entitlements.getEntitlementContext() != null) {
        stopMachineFlags.put(
            "tags",
            MutableSet.of(
                BrooklynTaskTags.tagForEntitlement(Entitlements.getEntitlementContext())));
      }
      Task<StopMachineDetails<Integer>> stopMachineTask =
          Tasks.<StopMachineDetails<Integer>>builder()
              .displayName("stopping (machine)")
              .body(stopTask)
              .flags(stopMachineFlags)
              .build();
      stoppingMachine = DynamicTasks.queue(stopMachineTask);

      DynamicTasks.drain(entity().getConfig(STOP_PROCESS_TIMEOUT), false);

      // shutdown the machine if stopping process fails or takes too long
      synchronized (stoppingMachine) {
        // task also used as mutex by DST when it submits it; ensure it only submits once!
        if (!stoppingMachine.isSubmitted()) {
          // force the stoppingMachine task to run by submitting it here
          StringBuilder msg =
              new StringBuilder("Submitting machine stop early in background for ")
                  .append(entity());
          if (stoppingProcess == null) {
            msg.append(". Process stop skipped, pre-stop not finished?");
          } else {
            msg.append(" because process stop has ")
                .append((stoppingProcess.isDone() ? "finished abnormally" : "not finished"));
          }
          log.warn(msg.toString());
          Entities.submit(entity(), stoppingMachine);
        }
      }
    }

    try {
      // This maintains previous behaviour of silently squashing any errors on the stoppingProcess
      // task if the
      // stoppingMachine exits with a nonzero value
      boolean checkStopProcesses =
          (stoppingProcess != null
              && (stoppingMachine == null || stoppingMachine.get().value == 0));

      if (checkStopProcesses) {
        // TODO we should test for destruction above, not merely successful "stop", as things like
        // localhost and ssh won't be destroyed
        DynamicTasks.waitForLast();
        if (machine.isPresent()) {
          // throw early errors *only if* there is a machine and we have not destroyed it
          stoppingProcess.get();
        }
      }
    } catch (Throwable e) {
      ServiceStateLogic.setExpectedState(entity(), Lifecycle.ON_FIRE);
      Exceptions.propagate(e);
    }
    entity().sensors().set(SoftwareProcess.SERVICE_UP, false);
    ServiceStateLogic.setExpectedState(entity(), Lifecycle.STOPPED);

    DynamicTasks.queue("post-stop", new PostStopCustomTask());

    if (log.isDebugEnabled()) log.debug("Stopped software process entity " + entity());
  }

  private class StopAnyProvisionedMachinesTask implements Callable<StopMachineDetails<Integer>> {
    public StopMachineDetails<Integer> call() {
      return stopAnyProvisionedMachines();
    }
  }

  private class SuspendAnyProvisionedMachinesTask implements Callable<StopMachineDetails<Integer>> {
    public StopMachineDetails<Integer> call() {
      return suspendAnyProvisionedMachines();
    }
  }

  private class StopProcessesAtMachineTask implements Callable<String> {
    public String call() {
      DynamicTasks.markInessential();
      stopProcessesAtMachine();
      DynamicTasks.waitForLast();
      return "Stop processes completed with no errors.";
    }
  }

  private class StopFeedsAtMachineTask implements Callable<String> {
    public String call() {
      DynamicTasks.markInessential();
      for (Feed feed : entity().feeds().getFeeds()) {
        if (feed.isActivated()) feed.stop();
      }
      DynamicTasks.waitForLast();
      return "Stop feeds completed with no errors.";
    }
  }

  private class StopMachineTask implements Callable<String> {
    public String call() {
      DynamicTasks.markInessential();
      stop(
          ConfigBag.newInstance()
              .configure(StopSoftwareParameters.STOP_MACHINE_MODE, StopMode.IF_NOT_STOPPED));
      DynamicTasks.waitForLast();
      return "Stop of machine completed with no errors.";
    }
  }

  private class PreStopCustomTask implements Callable<String> {
    public String call() {
      if (entity().getAttribute(SoftwareProcess.SERVICE_STATE_ACTUAL) == Lifecycle.STOPPED) {
        log.debug("Skipping stop of entity " + entity() + " when already stopped");
        return "Already stopped";
      }
      ServiceStateLogic.setExpectedState(entity(), Lifecycle.STOPPING);
      entity().sensors().set(SoftwareProcess.SERVICE_UP, false);
      preStopCustom();
      return null;
    }
  }

  private class PostStopCustomTask implements Callable<Void> {
    public Void call() {
      postStopCustom();
      return null;
    }
  }

  public static StopMode getStopMachineMode(ConfigBag parameters) {
    final StopMode stopMachineMode = parameters.get(StopSoftwareParameters.STOP_MACHINE_MODE);
    return stopMachineMode;
  }

  public static boolean canStop(StopMode stopMode, Entity entity) {
    boolean isEntityStopped =
        entity.getAttribute(SoftwareProcess.SERVICE_STATE_ACTUAL) == Lifecycle.STOPPED;
    return canStop(stopMode, isEntityStopped);
  }

  protected static boolean canStop(StopMode stopMode, boolean isStopped) {
    return stopMode == StopMode.ALWAYS || stopMode == StopMode.IF_NOT_STOPPED && !isStopped;
  }

  /** Override to check whether stop can be executed. Throw if stop should be aborted. */
  protected void preStopConfirmCustom() {
    // Opportunity to block stop() until other dependent components are ready for it
    Object val = entity().getConfig(SoftwareProcess.STOP_LATCH);
    if (val != null)
      log.debug("{} finished waiting for stop-latch {}; continuing...", entity(), val);
  }

  protected void preStopCustom() {
    // nothing needed here
  }

  protected void postStopCustom() {
    // nothing needed here
  }

  protected void preRestartCustom() {
    // nothing needed here
  }

  protected void postRestartCustom() {
    // nothing needed here
  }

  public static class StopMachineDetails<T> implements Serializable {
    private static final long serialVersionUID = 3256747214315895431L;
    final String message;
    final T value;

    protected StopMachineDetails(String message, T value) {
      this.message = message;
      this.value = value;
    }

    @Override
    public String toString() {
      return message;
    }
  }

  /**
   * Return string message of result.
   *
   * <p>Can run synchronously or not, caller will submit/queue as needed, and will block on any
   * submitted tasks.
   */
  protected abstract String stopProcessesAtMachine();

  /**
   * Stop and release the {@link MachineLocation} the entity is provisioned at.
   *
   * <p>Can run synchronously or not, caller will submit/queue as needed, and will block on any
   * submitted tasks.
   */
  protected StopMachineDetails<Integer> stopAnyProvisionedMachines() {
    @SuppressWarnings("unchecked")
    MachineProvisioningLocation<MachineLocation> provisioner =
        entity().getAttribute(SoftwareProcess.PROVISIONING_LOCATION);

    if (Iterables.isEmpty(entity().getLocations())) {
      log.debug("No machine decommissioning necessary for " + entity() + " - no locations");
      return new StopMachineDetails<Integer>(
          "No machine decommissioning necessary - no locations", 0);
    }

    // Only release this machine if we ourselves provisioned it (e.g. it might be running other
    // services)
    if (provisioner == null) {
      log.debug("No machine decommissioning necessary for " + entity() + " - did not provision");
      return new StopMachineDetails<Integer>(
          "No machine decommissioning necessary - did not provision", 0);
    }

    Location machine = getLocation(null);
    if (!(machine instanceof MachineLocation)) {
      log.debug(
          "No decommissioning necessary for "
              + entity()
              + " - not a machine location ("
              + machine
              + ")");
      return new StopMachineDetails<Integer>(
          "No machine decommissioning necessary - not a machine (" + machine + ")", 0);
    }

    entity()
        .sensors()
        .set(AttributesInternal.INTERNAL_TERMINATION_TASK_STATE, ProvisioningTaskState.RUNNING);
    try {
      clearEntityLocationAttributes(machine);
      provisioner.release((MachineLocation) machine);
    } finally {
      // TODO On exception, should we add the machine back to the entity (because it might not
      // really be terminated)?
      //      Do we need a better exception hierarchy for that?
      entity().sensors().remove(AttributesInternal.INTERNAL_TERMINATION_TASK_STATE);
    }

    return new StopMachineDetails<Integer>("Decommissioned " + machine, 1);
  }

  /**
   * Suspend the {@link MachineLocation} the entity is provisioned at.
   *
   * <p>Expects the entity's {@link SoftwareProcess#PROVISIONING_LOCATION provisioner} to be capable
   * of {@link SuspendsMachines suspending machines}.
   *
   * @throws java.lang.UnsupportedOperationException if the entity's provisioner cannot suspend
   *     machines.
   * @see MachineManagementMixins.SuspendsMachines
   */
  protected StopMachineDetails<Integer> suspendAnyProvisionedMachines() {
    @SuppressWarnings("unchecked")
    MachineProvisioningLocation<MachineLocation> provisioner =
        entity().getAttribute(SoftwareProcess.PROVISIONING_LOCATION);

    if (Iterables.isEmpty(entity().getLocations())) {
      log.debug("No machine decommissioning necessary for " + entity() + " - no locations");
      return new StopMachineDetails<>("No machine suspend necessary - no locations", 0);
    }

    // Only release this machine if we ourselves provisioned it (e.g. it might be running other
    // services)
    if (provisioner == null) {
      log.debug("No machine decommissioning necessary for " + entity() + " - did not provision");
      return new StopMachineDetails<>("No machine suspend necessary - did not provision", 0);
    }

    Location machine = getLocation(null);
    if (!(machine instanceof MachineLocation)) {
      log.debug(
          "No decommissioning necessary for "
              + entity()
              + " - not a machine location ("
              + machine
              + ")");
      return new StopMachineDetails<>(
          "No machine suspend necessary - not a machine (" + machine + ")", 0);
    }

    if (!(provisioner instanceof SuspendsMachines)) {
      log.debug("Location provisioner ({}) cannot suspend machines", provisioner);
      throw new UnsupportedOperationException(
          "Location provisioner cannot suspend machines: " + provisioner);
    }

    clearEntityLocationAttributes(machine);
    SuspendsMachines.class.cast(provisioner).suspendMachine(MachineLocation.class.cast(machine));

    return new StopMachineDetails<>("Suspended " + machine, 1);
  }

  /**
   * Nulls the attached entity's hostname, address, subnet hostname and subnet address sensors and
   * removes the given machine from its locations.
   */
  protected void clearEntityLocationAttributes(Location machine) {
    entity().removeLocations(ImmutableList.of(machine));
    entity().sensors().set(Attributes.HOSTNAME, null);
    entity().sensors().set(Attributes.ADDRESS, null);
    entity().sensors().set(Attributes.SUBNET_HOSTNAME, null);
    entity().sensors().set(Attributes.SUBNET_ADDRESS, null);
  }
}
Пример #7
0
public class RebindTestUtils {

  private static final Logger LOG = LoggerFactory.getLogger(RebindTestUtils.class);

  private static final Duration TIMEOUT = Duration.seconds(20);

  public static <T> T serializeAndDeserialize(T memento) throws Exception {
    ObjectReplacer replacer =
        new ObjectReplacer() {
          private final Map<Pointer, Object> replaced = Maps.newLinkedHashMap();

          @Override
          public Object replace(Object toserialize) {
            if (toserialize instanceof Location || toserialize instanceof Entity) {
              Pointer pointer = new Pointer(((Identifiable) toserialize).getId());
              replaced.put(pointer, toserialize);
              return pointer;
            }
            return toserialize;
          }

          @Override
          public Object resolve(Object todeserialize) {
            if (todeserialize instanceof Pointer) {
              return checkNotNull(replaced.get(todeserialize), todeserialize);
            }
            return todeserialize;
          }
        };

    try {
      return Serializers.reconstitute(memento, replacer);
    } catch (Exception e) {
      try {
        Dumpers.logUnserializableChains(memento, replacer);
        // Dumpers.deepDumpSerializableness(memento);
      } catch (Throwable t) {
        LOG.warn(
            "Error logging unserializable chains for memento "
                + memento
                + " (propagating original exception)",
            t);
      }
      throw e;
    }
  }

  public static void deleteMementoDir(String path) {
    deleteMementoDir(new File(path));
  }

  public static void deleteMementoDir(File f) {
    FileBasedObjectStore.deleteCompletely(f);
  }

  public static void checkMementoSerializable(Application app) throws Exception {
    BrooklynMemento memento = MementosGenerators.newBrooklynMemento(app.getManagementContext());
    checkMementoSerializable(memento);
  }

  public static void checkMementoSerializable(BrooklynMemento memento) throws Exception {
    serializeAndDeserialize(memento);
  }

  public static LocalManagementContext newPersistingManagementContext(
      File mementoDir, ClassLoader classLoader) {
    return managementContextBuilder(mementoDir, classLoader).buildStarted();
  }

  public static LocalManagementContext newPersistingManagementContext(
      File mementoDir, ClassLoader classLoader, long persistPeriodMillis) {
    return managementContextBuilder(mementoDir, classLoader)
        .persistPeriodMillis(persistPeriodMillis)
        .buildStarted();
  }

  public static LocalManagementContext newPersistingManagementContextUnstarted(
      File mementoDir, ClassLoader classLoader) {
    return managementContextBuilder(mementoDir, classLoader).buildUnstarted();
  }

  public static ManagementContextBuilder managementContextBuilder(
      File mementoDir, ClassLoader classLoader) {
    return new ManagementContextBuilder(classLoader, mementoDir);
  }

  public static ManagementContextBuilder managementContextBuilder(
      ClassLoader classLoader, File mementoDir) {
    return new ManagementContextBuilder(classLoader, mementoDir);
  }

  public static ManagementContextBuilder managementContextBuilder(
      ClassLoader classLoader, PersistenceObjectStore objectStore) {
    return new ManagementContextBuilder(classLoader, objectStore);
  }

  public static class ManagementContextBuilder {
    final ClassLoader classLoader;
    BrooklynProperties properties;
    PersistenceObjectStore objectStore;
    Duration persistPeriod = Duration.millis(100);
    HighAvailabilityMode haMode;
    boolean forLive;
    boolean enableOsgi = false;
    boolean emptyCatalog;
    private boolean enablePersistenceBackups = true;

    ManagementContextBuilder(File mementoDir, ClassLoader classLoader) {
      this(classLoader, new FileBasedObjectStore(mementoDir));
    }

    ManagementContextBuilder(ClassLoader classLoader, File mementoDir) {
      this(classLoader, new FileBasedObjectStore(mementoDir));
    }

    ManagementContextBuilder(ClassLoader classLoader, PersistenceObjectStore objStore) {
      this.classLoader = checkNotNull(classLoader, "classLoader");
      this.objectStore = checkNotNull(objStore, "objStore");
    }

    public ManagementContextBuilder persistPeriodMillis(long persistPeriodMillis) {
      checkArgument(
          persistPeriodMillis > 0,
          "persistPeriodMillis must be greater than 0; was " + persistPeriodMillis);
      return persistPeriod(Duration.millis(persistPeriodMillis));
    }

    public ManagementContextBuilder persistPeriod(Duration persistPeriod) {
      checkNotNull(persistPeriod);
      this.persistPeriod = persistPeriod;
      return this;
    }

    public ManagementContextBuilder properties(BrooklynProperties properties) {
      this.properties = checkNotNull(properties, "properties");
      return this;
    }

    public ManagementContextBuilder forLive(boolean val) {
      this.forLive = val;
      return this;
    }

    public ManagementContextBuilder enablePersistenceBackups(boolean val) {
      this.enablePersistenceBackups = val;
      return this;
    }

    public ManagementContextBuilder enableOsgi(boolean val) {
      this.enableOsgi = val;
      return this;
    }

    public ManagementContextBuilder emptyCatalog() {
      this.emptyCatalog = true;
      return this;
    }

    public ManagementContextBuilder emptyCatalog(boolean val) {
      this.emptyCatalog = val;
      return this;
    }

    public ManagementContextBuilder haMode(HighAvailabilityMode val) {
      this.haMode = val;
      return this;
    }

    public LocalManagementContext buildUnstarted() {
      LocalManagementContext unstarted;
      BrooklynProperties properties =
          this.properties != null ? this.properties : BrooklynProperties.Factory.newDefault();
      if (this.emptyCatalog) {
        properties.putIfAbsent(
            BrooklynServerConfig.BROOKLYN_CATALOG_URL, ManagementContextInternal.EMPTY_CATALOG_URL);
      }
      if (!enablePersistenceBackups) {
        properties.putIfAbsent(
            BrooklynServerConfig.PERSISTENCE_BACKUPS_REQUIRED_ON_DEMOTION, false);
        properties.putIfAbsent(
            BrooklynServerConfig.PERSISTENCE_BACKUPS_REQUIRED_ON_PROMOTION, false);
        properties.putIfAbsent(BrooklynServerConfig.PERSISTENCE_BACKUPS_REQUIRED, false);
      }
      if (forLive) {
        unstarted = new LocalManagementContext(properties);
      } else {
        unstarted =
            LocalManagementContextForTests.builder(true)
                .useProperties(properties)
                .disableOsgi(!enableOsgi)
                .build();
      }

      objectStore.injectManagementContext(unstarted);
      objectStore.prepareForSharedUse(
          PersistMode.AUTO, (haMode == null ? HighAvailabilityMode.DISABLED : haMode));
      BrooklynMementoPersisterToObjectStore newPersister =
          new BrooklynMementoPersisterToObjectStore(
              objectStore, unstarted.getBrooklynProperties(), classLoader);
      ((RebindManagerImpl) unstarted.getRebindManager()).setPeriodicPersistPeriod(persistPeriod);
      unstarted
          .getRebindManager()
          .setPersister(newPersister, PersistenceExceptionHandlerImpl.builder().build());
      // set the HA persister, in case any children want to use HA
      unstarted
          .getHighAvailabilityManager()
          .setPersister(
              new ManagementPlaneSyncRecordPersisterToObjectStore(
                  unstarted, objectStore, classLoader));
      return unstarted;
    }

    public LocalManagementContext buildStarted() {
      LocalManagementContext unstarted = buildUnstarted();
      unstarted.getHighAvailabilityManager().disabled();
      unstarted.getRebindManager().startPersistence();
      return unstarted;
    }
  }

  /** Convenience for common call; delegates to {@link #rebind(RebindOptions)} */
  public static Application rebind(File mementoDir, ClassLoader classLoader) throws Exception {
    return rebind(RebindOptions.create().mementoDir(mementoDir).classLoader(classLoader));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Application rebind(
      File mementoDir, ClassLoader classLoader, RebindExceptionHandler exceptionHandler)
      throws Exception {
    return rebind(
        RebindOptions.create()
            .mementoDir(mementoDir)
            .classLoader(classLoader)
            .exceptionHandler(exceptionHandler));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Application rebind(ManagementContext newManagementContext, ClassLoader classLoader)
      throws Exception {
    return rebind(
        RebindOptions.create().newManagementContext(newManagementContext).classLoader(classLoader));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Application rebind(
      ManagementContext newManagementContext,
      ClassLoader classLoader,
      RebindExceptionHandler exceptionHandler)
      throws Exception {
    return rebind(
        RebindOptions.create()
            .newManagementContext(newManagementContext)
            .classLoader(classLoader)
            .exceptionHandler(exceptionHandler));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Application rebind(
      ManagementContext newManagementContext, File mementoDir, ClassLoader classLoader)
      throws Exception {
    return rebind(
        RebindOptions.create()
            .newManagementContext(newManagementContext)
            .mementoDir(mementoDir)
            .classLoader(classLoader));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Application rebind(
      ManagementContext newManagementContext,
      File mementoDir,
      ClassLoader classLoader,
      RebindExceptionHandler exceptionHandler)
      throws Exception {
    return rebind(
        RebindOptions.create()
            .newManagementContext(newManagementContext)
            .mementoDir(mementoDir)
            .classLoader(classLoader)
            .exceptionHandler(exceptionHandler));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Application rebind(
      ManagementContext newManagementContext,
      File mementoDir,
      ClassLoader classLoader,
      RebindExceptionHandler exceptionHandler,
      PersistenceObjectStore objectStore)
      throws Exception {
    return rebind(
        RebindOptions.create()
            .newManagementContext(newManagementContext)
            .mementoDir(mementoDir)
            .classLoader(classLoader)
            .exceptionHandler(exceptionHandler)
            .objectStore(objectStore));
  }

  public static Application rebind(RebindOptions options) throws Exception {
    Collection<Application> newApps = rebindAll(options);
    if (newApps.isEmpty())
      throw new IllegalStateException(
          "Application could not be rebinded; serialization probably failed");
    return Iterables.getFirst(newApps, null);
  }

  /** @deprecated since 0.7.0; use {@link #rebindAll(RebindOptions)} */
  @Deprecated
  public static Collection<Application> rebindAll(File mementoDir, ClassLoader classLoader)
      throws Exception {
    return rebindAll(RebindOptions.create().mementoDir(mementoDir).classLoader(classLoader));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Collection<Application> rebindAll(
      File mementoDir, ClassLoader classLoader, RebindExceptionHandler exceptionHandler)
      throws Exception {
    return rebindAll(
        RebindOptions.create()
            .mementoDir(mementoDir)
            .classLoader(classLoader)
            .exceptionHandler(exceptionHandler));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Collection<Application> rebindAll(
      LocalManagementContext newManagementContext,
      ClassLoader classLoader,
      RebindExceptionHandler exceptionHandler)
      throws Exception {
    return rebindAll(
        RebindOptions.create()
            .newManagementContext(newManagementContext)
            .classLoader(classLoader)
            .exceptionHandler(exceptionHandler));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Collection<Application> rebindAll(
      ManagementContext newManagementContext, File mementoDir, ClassLoader classLoader)
      throws Exception {
    return rebindAll(
        RebindOptions.create()
            .newManagementContext(newManagementContext)
            .mementoDir(mementoDir)
            .classLoader(classLoader));
  }

  /** @deprecated since 0.7.0; use {@link #rebind(RebindOptions)} */
  @Deprecated
  public static Collection<Application> rebindAll(
      ManagementContext newManagementContext,
      File mementoDir,
      ClassLoader classLoader,
      RebindExceptionHandler exceptionHandler,
      PersistenceObjectStore objectStore)
      throws Exception {
    return rebindAll(
        RebindOptions.create()
            .newManagementContext(newManagementContext)
            .mementoDir(mementoDir)
            .classLoader(classLoader)
            .exceptionHandler(exceptionHandler)
            .objectStore(objectStore));
  }

  public static Collection<Application> rebindAll(RebindOptions options) throws Exception {
    File mementoDir = options.mementoDir;
    File mementoDirBackup = options.mementoDirBackup;
    ClassLoader classLoader = checkNotNull(options.classLoader, "classLoader");
    ManagementContextInternal origManagementContext =
        (ManagementContextInternal) options.origManagementContext;
    ManagementContextInternal newManagementContext =
        (ManagementContextInternal) options.newManagementContext;
    PersistenceObjectStore objectStore = options.objectStore;
    HighAvailabilityMode haMode =
        (options.haMode == null ? HighAvailabilityMode.DISABLED : options.haMode);
    RebindExceptionHandler exceptionHandler = options.exceptionHandler;
    boolean hasPersister =
        newManagementContext != null
            && newManagementContext.getRebindManager().getPersister() != null;
    boolean checkSerializable = options.checkSerializable;
    boolean terminateOrigManagementContext = options.terminateOrigManagementContext;
    Function<BrooklynMementoPersister, Void> stateTransformer = options.stateTransformer;

    LOG.info("Rebinding app, using mementoDir " + mementoDir + "; object store " + objectStore);

    if (newManagementContext == null) {
      // TODO Could use empty properties, to save reading brooklyn.properties file.
      // Would that affect any tests?
      newManagementContext =
          new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault());
    }
    if (!hasPersister) {
      if (objectStore == null) {
        objectStore =
            new FileBasedObjectStore(
                checkNotNull(mementoDir, "mementoDir and objectStore must not both be null"));
      }
      objectStore.injectManagementContext(newManagementContext);
      objectStore.prepareForSharedUse(PersistMode.AUTO, haMode);

      BrooklynMementoPersisterToObjectStore newPersister =
          new BrooklynMementoPersisterToObjectStore(
              objectStore, newManagementContext.getBrooklynProperties(), classLoader);
      newManagementContext
          .getRebindManager()
          .setPersister(newPersister, PersistenceExceptionHandlerImpl.builder().build());
    } else {
      if (objectStore != null)
        throw new IllegalStateException(
            "Must not supply ManagementContext with persister and an object store");
    }

    if (checkSerializable) {
      checkNotNull(
          origManagementContext, "must supply origManagementContext with checkSerializable");
      RebindTestUtils.checkCurrentMementoSerializable(origManagementContext);
    }

    if (terminateOrigManagementContext) {
      checkNotNull(
          origManagementContext,
          "must supply origManagementContext with terminateOrigManagementContext");
      origManagementContext.terminate();
    }

    if (mementoDirBackup != null) {
      FileUtil.copyDir(mementoDir, mementoDirBackup);
      FileUtil.setFilePermissionsTo700(mementoDirBackup);
    }

    if (stateTransformer != null) {
      BrooklynMementoPersister persister = newManagementContext.getRebindManager().getPersister();
      stateTransformer.apply(persister);
    }

    List<Application> newApps =
        newManagementContext
            .getRebindManager()
            .rebind(
                classLoader,
                exceptionHandler,
                (haMode == HighAvailabilityMode.DISABLED)
                    ? ManagementNodeState.MASTER
                    : ManagementNodeState.of(haMode).get());
    newManagementContext.getRebindManager().startPersistence();
    return newApps;
  }

  public static void waitForPersisted(Application origApp)
      throws InterruptedException, TimeoutException {
    waitForPersisted(origApp.getManagementContext());
  }

  public static void waitForPersisted(ManagementContext managementContext)
      throws InterruptedException, TimeoutException {
    managementContext.getRebindManager().waitForPendingComplete(TIMEOUT, true);
  }

  public static void checkCurrentMementoSerializable(Application app) throws Exception {
    checkCurrentMementoSerializable(app.getManagementContext());
  }

  public static void checkCurrentMementoSerializable(ManagementContext mgmt) throws Exception {
    BrooklynMemento memento = MementosGenerators.newBrooklynMemento(mgmt);
    serializeAndDeserialize(memento);
  }

  /**
   * Dumps out the persisted mementos that are at the given directory.
   *
   * <p>Binds to the persisted state (as a "hot standby") to load the raw data (as strings), and to
   * write out the entity, location, policy, enricher, feed and catalog-item data.
   *
   * @param dir The directory containing the persisted state
   */
  public static void dumpMementoDir(File dir) {
    LocalManagementContextForTests mgmt =
        new LocalManagementContextForTests(BrooklynProperties.Factory.newEmpty());
    FileBasedObjectStore store = null;
    BrooklynMementoPersisterToObjectStore persister = null;
    try {
      store = new FileBasedObjectStore(dir);
      store.injectManagementContext(mgmt);
      store.prepareForSharedUse(PersistMode.AUTO, HighAvailabilityMode.HOT_STANDBY);
      persister =
          new BrooklynMementoPersisterToObjectStore(
              store, BrooklynProperties.Factory.newEmpty(), RebindTestUtils.class.getClassLoader());
      BrooklynMementoRawData data =
          persister.loadMementoRawData(RebindExceptionHandlerImpl.builder().build());
      List<BrooklynObjectType> types =
          ImmutableList.of(
              BrooklynObjectType.ENTITY,
              BrooklynObjectType.LOCATION,
              BrooklynObjectType.POLICY,
              BrooklynObjectType.ENRICHER,
              BrooklynObjectType.FEED,
              BrooklynObjectType.CATALOG_ITEM);
      for (BrooklynObjectType type : types) {
        LOG.info(type + " (" + data.getObjectsOfType(type).keySet() + "):");
        for (Map.Entry<String, String> entry : data.getObjectsOfType(type).entrySet()) {
          LOG.info("\t" + type + " " + entry.getKey() + ": " + entry.getValue());
        }
      }
    } finally {
      if (persister != null) persister.stop(false);
      if (store != null) store.close();
      mgmt.terminate();
    }
  }
}
Пример #8
0
 public ManagementContextBuilder persistPeriodMillis(long persistPeriodMillis) {
   checkArgument(
       persistPeriodMillis > 0,
       "persistPeriodMillis must be greater than 0; was " + persistPeriodMillis);
   return persistPeriod(Duration.millis(persistPeriodMillis));
 }
Пример #9
0
  public static class ManagementContextBuilder {
    final ClassLoader classLoader;
    BrooklynProperties properties;
    PersistenceObjectStore objectStore;
    Duration persistPeriod = Duration.millis(100);
    HighAvailabilityMode haMode;
    boolean forLive;
    boolean enableOsgi = false;
    boolean emptyCatalog;
    private boolean enablePersistenceBackups = true;

    ManagementContextBuilder(File mementoDir, ClassLoader classLoader) {
      this(classLoader, new FileBasedObjectStore(mementoDir));
    }

    ManagementContextBuilder(ClassLoader classLoader, File mementoDir) {
      this(classLoader, new FileBasedObjectStore(mementoDir));
    }

    ManagementContextBuilder(ClassLoader classLoader, PersistenceObjectStore objStore) {
      this.classLoader = checkNotNull(classLoader, "classLoader");
      this.objectStore = checkNotNull(objStore, "objStore");
    }

    public ManagementContextBuilder persistPeriodMillis(long persistPeriodMillis) {
      checkArgument(
          persistPeriodMillis > 0,
          "persistPeriodMillis must be greater than 0; was " + persistPeriodMillis);
      return persistPeriod(Duration.millis(persistPeriodMillis));
    }

    public ManagementContextBuilder persistPeriod(Duration persistPeriod) {
      checkNotNull(persistPeriod);
      this.persistPeriod = persistPeriod;
      return this;
    }

    public ManagementContextBuilder properties(BrooklynProperties properties) {
      this.properties = checkNotNull(properties, "properties");
      return this;
    }

    public ManagementContextBuilder forLive(boolean val) {
      this.forLive = val;
      return this;
    }

    public ManagementContextBuilder enablePersistenceBackups(boolean val) {
      this.enablePersistenceBackups = val;
      return this;
    }

    public ManagementContextBuilder enableOsgi(boolean val) {
      this.enableOsgi = val;
      return this;
    }

    public ManagementContextBuilder emptyCatalog() {
      this.emptyCatalog = true;
      return this;
    }

    public ManagementContextBuilder emptyCatalog(boolean val) {
      this.emptyCatalog = val;
      return this;
    }

    public ManagementContextBuilder haMode(HighAvailabilityMode val) {
      this.haMode = val;
      return this;
    }

    public LocalManagementContext buildUnstarted() {
      LocalManagementContext unstarted;
      BrooklynProperties properties =
          this.properties != null ? this.properties : BrooklynProperties.Factory.newDefault();
      if (this.emptyCatalog) {
        properties.putIfAbsent(
            BrooklynServerConfig.BROOKLYN_CATALOG_URL, ManagementContextInternal.EMPTY_CATALOG_URL);
      }
      if (!enablePersistenceBackups) {
        properties.putIfAbsent(
            BrooklynServerConfig.PERSISTENCE_BACKUPS_REQUIRED_ON_DEMOTION, false);
        properties.putIfAbsent(
            BrooklynServerConfig.PERSISTENCE_BACKUPS_REQUIRED_ON_PROMOTION, false);
        properties.putIfAbsent(BrooklynServerConfig.PERSISTENCE_BACKUPS_REQUIRED, false);
      }
      if (forLive) {
        unstarted = new LocalManagementContext(properties);
      } else {
        unstarted =
            LocalManagementContextForTests.builder(true)
                .useProperties(properties)
                .disableOsgi(!enableOsgi)
                .build();
      }

      objectStore.injectManagementContext(unstarted);
      objectStore.prepareForSharedUse(
          PersistMode.AUTO, (haMode == null ? HighAvailabilityMode.DISABLED : haMode));
      BrooklynMementoPersisterToObjectStore newPersister =
          new BrooklynMementoPersisterToObjectStore(
              objectStore, unstarted.getBrooklynProperties(), classLoader);
      ((RebindManagerImpl) unstarted.getRebindManager()).setPeriodicPersistPeriod(persistPeriod);
      unstarted
          .getRebindManager()
          .setPersister(newPersister, PersistenceExceptionHandlerImpl.builder().build());
      // set the HA persister, in case any children want to use HA
      unstarted
          .getHighAvailabilityManager()
          .setPersister(
              new ManagementPlaneSyncRecordPersisterToObjectStore(
                  unstarted, objectStore, classLoader));
      return unstarted;
    }

    public LocalManagementContext buildStarted() {
      LocalManagementContext unstarted = buildUnstarted();
      unstarted.getHighAvailabilityManager().disabled();
      unstarted.getRebindManager().startPersistence();
      return unstarted;
    }
  }
    @SuppressWarnings({"rawtypes", "unchecked"})
    @Override
    public V call() {
      T value = source.getAttribute(sensor);

      // return immediately if either the ready predicate or the abort conditions hold
      if (ready(value)) return postProcess(value);

      final List<Exception> abortionExceptions = Lists.newCopyOnWriteArrayList();
      long start = System.currentTimeMillis();

      for (AttributeAndSensorCondition abortCondition : abortSensorConditions) {
        Object abortValue = abortCondition.source.getAttribute(abortCondition.sensor);
        if (abortCondition.predicate.apply(abortValue)) {
          abortionExceptions.add(
              new Exception(
                  "Abort due to " + abortCondition.source + " -> " + abortCondition.sensor));
        }
      }
      if (abortionExceptions.size() > 0) {
        throw new CompoundRuntimeException(
            "Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
      }

      TaskInternal<?> current = (TaskInternal<?>) Tasks.current();
      if (current == null)
        throw new IllegalStateException("Should only be invoked in a running task");
      Entity entity = BrooklynTaskTags.getTargetOrContextEntity(current);
      if (entity == null)
        throw new IllegalStateException(
            "Should only be invoked in a running task with an entity tag; "
                + current
                + " has no entity tag ("
                + current.getStatusDetail(false)
                + ")");

      final LinkedList<T> publishedValues = new LinkedList<T>();
      final Semaphore semaphore = new Semaphore(0); // could use Exchanger
      SubscriptionHandle subscription = null;
      List<SubscriptionHandle> abortSubscriptions = Lists.newArrayList();

      try {
        subscription =
            entity
                .subscriptions()
                .subscribe(
                    source,
                    sensor,
                    new SensorEventListener<T>() {
                      @Override
                      public void onEvent(SensorEvent<T> event) {
                        synchronized (publishedValues) {
                          publishedValues.add(event.getValue());
                        }
                        semaphore.release();
                      }
                    });
        for (final AttributeAndSensorCondition abortCondition : abortSensorConditions) {
          abortSubscriptions.add(
              entity
                  .subscriptions()
                  .subscribe(
                      abortCondition.source,
                      abortCondition.sensor,
                      new SensorEventListener<Object>() {
                        @Override
                        public void onEvent(SensorEvent<Object> event) {
                          if (abortCondition.predicate.apply(event.getValue())) {
                            abortionExceptions.add(
                                new Exception(
                                    "Abort due to "
                                        + abortCondition.source
                                        + " -> "
                                        + abortCondition.sensor));
                            semaphore.release();
                          }
                        }
                      }));
          Object abortValue = abortCondition.source.getAttribute(abortCondition.sensor);
          if (abortCondition.predicate.apply(abortValue)) {
            abortionExceptions.add(
                new Exception(
                    "Abort due to " + abortCondition.source + " -> " + abortCondition.sensor));
          }
        }
        if (abortionExceptions.size() > 0) {
          throw new CompoundRuntimeException(
              "Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
        }

        CountdownTimer timer = timeout != null ? timeout.countdownTimer() : null;
        Duration maxPeriod = ValueResolver.PRETTY_QUICK_WAIT;
        Duration nextPeriod = ValueResolver.REAL_QUICK_PERIOD;
        while (true) {
          // check the source on initial run (could be done outside the loop)
          // and also (optionally) on each iteration in case it is more recent
          value = source.getAttribute(sensor);
          if (ready(value)) break;

          if (timer != null) {
            if (timer.getDurationRemaining().isShorterThan(nextPeriod)) {
              nextPeriod = timer.getDurationRemaining();
            }
            if (timer.isExpired()) {
              if (onTimeout.isPresent()) return onTimeout.get();
              throw new RuntimeTimeoutException("Unsatisfied after " + Duration.sinceUtc(start));
            }
          }

          String prevBlockingDetails = current.setBlockingDetails(blockingDetails);
          try {
            if (semaphore.tryAcquire(nextPeriod.toMilliseconds(), TimeUnit.MILLISECONDS)) {
              // immediately release so we are available for the next check
              semaphore.release();
              // if other permits have been made available (e.g. multiple notifications) drain them
              // all as no point running multiple times
              semaphore.drainPermits();
            }
          } finally {
            current.setBlockingDetails(prevBlockingDetails);
          }

          // check any subscribed values which have come in first
          while (true) {
            synchronized (publishedValues) {
              if (publishedValues.isEmpty()) break;
              value = publishedValues.pop();
            }
            if (ready(value)) break;
          }

          // if unmanaged then ignore the other abort conditions
          if (!ignoreUnmanaged && Entities.isNoLongerManaged(entity)) {
            if (onUnmanaged.isPresent()) return onUnmanaged.get();
            throw new NotManagedException(entity);
          }

          if (abortionExceptions.size() > 0) {
            throw new CompoundRuntimeException(
                "Aborted waiting for ready from " + source + " " + sensor, abortionExceptions);
          }

          nextPeriod = nextPeriod.times(2).upperBound(maxPeriod);
        }
        if (LOG.isDebugEnabled()) LOG.debug("Attribute-ready for {} in entity {}", sensor, source);
        return postProcess(value);
      } catch (InterruptedException e) {
        throw Exceptions.propagate(e);
      } finally {
        if (subscription != null) {
          entity.subscriptions().unsubscribe(subscription);
        }
        for (SubscriptionHandle handle : abortSubscriptions) {
          entity.subscriptions().unsubscribe(handle);
        }
      }
    }
 public void noteFailure(Duration duration) {
   count++;
   failureCount++;
   lastFailureTime = System.currentTimeMillis();
   lastDuration = duration != null ? duration.toMilliseconds() : -1;
 }
 public void noteSuccess(Duration duration) {
   count++;
   lastSuccessTime = System.currentTimeMillis();
   lastDuration = duration.toMilliseconds();
 }
 /** @deprecated since 0.7.0; use {@link #setPeriodicPersistPeriod(Duration)} */
 public void setPeriodicPersistPeriod(long periodMillis) {
   setPeriodicPersistPeriod(Duration.of(periodMillis, TimeUnit.MILLISECONDS));
 }