@Test
  public void testPolicyUpdatesModel() {
    final MockContainerEntity containerA = newContainer(app, "A", 10, 20);
    final MockContainerEntity containerB = newContainer(app, "B", 11, 21);
    final MockItemEntity item1 = newItem(app, containerA, "1", 12);
    final MockItemEntity item2 = newItem(app, containerB, "2", 13);

    Asserts.succeedsEventually(
        MutableMap.of("timeout", TIMEOUT_MS),
        new Runnable() {
          public void run() {
            assertEquals(model.getPoolSize(), 2);
            assertEquals(model.getPoolContents(), ImmutableSet.of(containerA, containerB));
            assertEquals(model.getItemWorkrate(item1), 12d);
            assertEquals(model.getItemWorkrate(item2), 13d);

            assertEquals(model.getParentContainer(item1), containerA);
            assertEquals(model.getParentContainer(item2), containerB);
            assertEquals(
                model.getContainerWorkrates(), ImmutableMap.of(containerA, 12d, containerB, 13d));

            assertEquals(model.getPoolLowThreshold(), 10 + 11d);
            assertEquals(model.getPoolHighThreshold(), 20 + 21d);
            assertEquals(model.getCurrentPoolWorkrate(), 12 + 13d);
            assertFalse(model.isHot());
            assertFalse(model.isCold());
          }
        });
  }
  @Test(
      dependsOnMethods = {
        "testListEffectors",
        "testFetchApplicationsAndEntity",
        "testTriggerSampleEffector",
        "testListApplications",
        "testReadEachSensor",
        "testPolicyWhichCapitalizes",
        "testLocatedLocation"
      })
  public void testDeleteApplication() throws TimeoutException, InterruptedException {
    waitForPageFoundResponse("/v1/applications/simple-app", ApplicationSummary.class);
    Collection<Application> apps = getManagementContext().getApplications();
    log.info("Deleting simple-app from " + apps);
    int size = apps.size();

    ClientResponse response =
        client().resource("/v1/applications/simple-app").delete(ClientResponse.class);

    assertEquals(response.getStatus(), Response.Status.ACCEPTED.getStatusCode());
    TaskSummary task = response.getEntity(TaskSummary.class);
    assertTrue(task.getDescription().toLowerCase().contains("destroy"), task.getDescription());
    assertTrue(task.getDescription().toLowerCase().contains("simple-app"), task.getDescription());

    waitForPageNotFoundResponse("/v1/applications/simple-app", ApplicationSummary.class);

    log.info("App appears gone, apps are: " + getManagementContext().getApplications());
    // more logging above, for failure in the check below

    Asserts.eventually(
        EntityFunctions.applications(getManagementContext()),
        Predicates.compose(Predicates.equalTo(size - 1), CollectionFunctionals.sizeFunction()));
  }
  @Test(groups = "Integration", dependsOnMethods = "testTriggerRedisStopEffector")
  public void testDeleteRedisApplication() throws Exception {
    int size = getManagementContext().getApplications().size();
    Response response = api.getApplicationApi().delete("redis-app");
    Assert.assertNotNull(response);
    try {
      Asserts.succeedsEventually(
          ImmutableMap.of("timeout", Duration.minutes(1)),
          new Runnable() {
            public void run() {
              try {
                ApplicationSummary summary = api.getApplicationApi().get("redis-app");
                fail("Redis app failed to disappear: summary=" + summary);
              } catch (Exception failure) {
                // expected -- it will be a ClientResponseFailure but that class is deprecated so
                // catching all
                // and asserting contains the word 404
                Assert.assertTrue(failure.toString().indexOf("404") >= 0);
              }
            }
          });
    } catch (Exception failure) {
      // expected -- it will be a ClientResponseFailure but that class is deprecated so catching all
      // and asserting contains the word 404
      Assert.assertTrue(failure.toString().indexOf("404") >= 0);
    }

    assertEquals(getManagementContext().getApplications().size(), size - 1);
  }
  protected void runTest(UsesJmx.JmxAgentModes jmxAgentMode) throws Exception {
    final JBoss6Server server =
        app.createAndManageChild(
            EntitySpec.create(JBoss6Server.class)
                .configure(JBoss6Server.PORT_INCREMENT, PORT_INCREMENT)
                .configure(UsesJmx.JMX_AGENT_MODE, jmxAgentMode)
                .configure("war", getTestWarWithNoMapping()));

    app.start(ImmutableList.of(localhostProvisioningLocation));

    String httpUrl =
        "http://"
            + server.getAttribute(JBoss6Server.HOSTNAME)
            + ":"
            + server.getAttribute(JBoss6Server.HTTP_PORT)
            + "/";

    assertEquals(server.getAttribute(JBoss6Server.ROOT_URL).toLowerCase(), httpUrl.toLowerCase());

    HttpTestUtils.assertHttpStatusCodeEventuallyEquals(httpUrl, 200);
    HttpTestUtils.assertContentContainsText(httpUrl, "Hello");

    Asserts.succeedsEventually(
        new Runnable() {
          @Override
          public void run() {
            // TODO Could test other attributes as well; see jboss7 test
            assertNotNull(server.getAttribute(JBoss6Server.REQUEST_COUNT));
            assertNotNull(server.getAttribute(JBoss6Server.ERROR_COUNT));
            assertNotNull(server.getAttribute(JBoss6Server.TOTAL_PROCESSING_TIME));
          }
        });
  }
  @Test
  public void testUnsubscribeRemovesAllSubscriptionsForThatEntity() throws Exception {
    policy.subscribe(entity, TestEntity.SEQUENCE, listener);
    policy.subscribe(entity, TestEntity.NAME, listener);
    policy.subscribe(entity, TestEntity.MY_NOTIF, listener);
    policy.subscribe(otherEntity, TestEntity.SEQUENCE, listener);
    policy.unsubscribe(entity);

    entity.setAttribute(TestEntity.SEQUENCE, 123);
    entity.setAttribute(TestEntity.NAME, "myname");
    entity.emit(TestEntity.MY_NOTIF, 456);
    otherEntity.setAttribute(TestEntity.SEQUENCE, 789);

    Thread.sleep(SHORT_WAIT_MS);
    Asserts.succeedsEventually(
        new Runnable() {
          @Override
          public void run() {
            assertEquals(
                listener.events,
                ImmutableList.of(
                    new BasicSensorEvent<Integer>(TestEntity.SEQUENCE, otherEntity, 789)));
          }
        });
  }
  @Override
  protected void doTest(Location loc) throws Exception {
    final Tomcat8Server server =
        app.createAndManageChild(
            EntitySpec.create(Tomcat8Server.class).configure("war", getTestWar()));

    app.start(ImmutableList.of(loc));

    String url = server.getAttribute(Tomcat8Server.ROOT_URL);

    HttpTestUtils.assertHttpStatusCodeEventuallyEquals(url, 200);
    HttpTestUtils.assertContentContainsText(url, "Hello");

    Asserts.succeedsEventually(
        new Runnable() {
          @Override
          public void run() {
            assertNotNull(server.getAttribute(Tomcat8Server.REQUEST_COUNT));
            assertNotNull(server.getAttribute(Tomcat8Server.ERROR_COUNT));
            assertNotNull(server.getAttribute(Tomcat8Server.TOTAL_PROCESSING_TIME));

            // TODO These appear not to be set in TomcatServerImpl.connectSensors
            //      See TomcatServerEc2LiveTest, where these are also not included.
            //                assertNotNull(server.getAttribute(TomcatServer.MAX_PROCESSING_TIME));
            //                assertNotNull(server.getAttribute(TomcatServer.BYTES_RECEIVED));
            //                assertNotNull(server.getAttribute(TomcatServer.BYTES_SENT));
          }
        });
  }
  @Test(groups = "Integration") // because one second wait in succeedsContinually
  public void testDoesNotPromoteIfMasterTimeoutNotExpired() throws Exception {
    persister.delta(
        ManagementPlaneSyncRecordDeltaImpl.builder()
            .node(newManagerMemento(ownNodeId, ManagementNodeState.STANDBY, tickerCurrentMillis()))
            .node(newManagerMemento("node1", ManagementNodeState.MASTER, tickerCurrentMillis()))
            .setMaster("node1")
            .build());

    manager.start(HighAvailabilityMode.AUTO);

    tickerAdvance(Duration.seconds(29));

    // Expect not to be notified, as 29s < 30s timeout (it's a fake clock so won't hit 30, even
    // waiting 1s below)
    Asserts.succeedsContinually(
        new Runnable() {
          @Override
          public void run() {
            assertTrue(
                promotionListener.callTimestamps.isEmpty(),
                "calls=" + promotionListener.callTimestamps);
          }
        });
  }
  @Test
  public void testServiceReplacerWorksAfterRebind() throws Exception {
    Location origLoc =
        origManagementContext
            .getLocationManager()
            .createLocation(LocationSpec.create(SimulatedLocation.class));
    DynamicCluster origCluster =
        origApp.createAndManageChild(
            EntitySpec.create(DynamicCluster.class)
                .configure(DynamicCluster.MEMBER_SPEC, EntitySpec.create(TestEntity.class))
                .configure(DynamicCluster.INITIAL_SIZE, 3));
    origApp.start(ImmutableList.<Location>of(origLoc));

    origCluster.addPolicy(
        PolicySpec.create(ServiceReplacer.class)
            .configure(ServiceReplacer.FAILURE_SENSOR_TO_MONITOR, HASensors.ENTITY_FAILED));

    // rebind
    TestApplication newApp = rebind();
    final DynamicCluster newCluster =
        (DynamicCluster)
            Iterables.find(newApp.getChildren(), Predicates.instanceOf(DynamicCluster.class));

    // stimulate the policy
    final Set<Entity> initialMembers = ImmutableSet.copyOf(newCluster.getMembers());
    final TestEntity e1 = (TestEntity) Iterables.get(initialMembers, 1);

    newApp
        .getManagementContext()
        .getSubscriptionManager()
        .subscribe(e1, HASensors.ENTITY_FAILED, eventListener);
    newApp
        .getManagementContext()
        .getSubscriptionManager()
        .subscribe(e1, HASensors.ENTITY_RECOVERED, eventListener);

    e1.emit(HASensors.ENTITY_FAILED, new FailureDescriptor(e1, "simulate failure"));

    // Expect e1 to be replaced
    Asserts.succeedsEventually(
        new Runnable() {
          @Override
          public void run() {
            Set<Entity> newMembers =
                Sets.difference(ImmutableSet.copyOf(newCluster.getMembers()), initialMembers);
            Set<Entity> removedMembers =
                Sets.difference(initialMembers, ImmutableSet.copyOf(newCluster.getMembers()));
            assertEquals(removedMembers, ImmutableSet.of(e1));
            assertEquals(newMembers.size(), 1);
            assertEquals(
                ((TestEntity) Iterables.getOnlyElement(newMembers)).getCallHistory(),
                ImmutableList.of("start"));

            // TODO e1 not reporting "start" after rebind because callHistory is a field rather than
            // an attribute, so was not persisted
            Asserts.assertEqualsIgnoringOrder(e1.getCallHistory(), ImmutableList.of("stop"));
            assertFalse(Entities.isManaged(e1));
          }
        });
  }
 public void assertCalledEventually() {
   Asserts.succeedsEventually(
       new Runnable() {
         @Override
         public void run() {
           assertCalled();
         }
       });
 }
 private void assertNoEventsContinually() {
   Asserts.succeedsContinually(
       new Runnable() {
         @Override
         public void run() {
           assertTrue(events.isEmpty(), "events=" + events);
         }
       });
 }
 private void assertNoEventsContinually(Duration duration) {
   Asserts.succeedsContinually(
       ImmutableMap.of("timeout", duration),
       new Runnable() {
         @Override
         public void run() {
           assertTrue(events.isEmpty(), "events=" + events);
         }
       });
 }
 private <T> void assertEqualsEventually(final T actual, final T expected) {
   Asserts.succeedsEventually(
       MutableMap.of("timeout", TIMEOUT_MS),
       new Runnable() {
         @Override
         public void run() {
           assertEquals(actual, expected, "actual=" + actual);
         }
       });
 }
 public WebAppMonitor waitForAtLeastOneAttempt(Duration timeout) {
   Asserts.succeedsEventually(
       MutableMap.of("timeout", timeout),
       new Runnable() {
         @Override
         public void run() {
           Assert.assertTrue(getAttempts() >= 1);
         }
       });
   return this;
 }
 private void assertServiceStateEventually(
     final String app, final String entity, final Lifecycle state, Duration timeout) {
   Asserts.succeedsEventually(
       ImmutableMap.of("timeout", timeout),
       new Runnable() {
         public void run() {
           Object status = api.getSensorApi().get(app, entity, "service.state", false);
           assertTrue(state.toString().equalsIgnoreCase(status.toString()), "status=" + status);
         }
       });
 }
  @Test(groups = {"Integration"})
  public void testRedisClusterReplicates() throws Exception {
    final String key = "mykey";
    final String val = "1234567890";

    cluster =
        app.createAndManageChild(
            EntitySpec.create(RedisCluster.class).configure(DynamicCluster.INITIAL_SIZE, 3));
    app.start(ImmutableList.of(loc));

    EntityTestUtils.assertAttributeEqualsEventually(cluster, Startable.SERVICE_UP, true);

    RedisStore master = cluster.getMaster();
    List<RedisSlave> slaves =
        ImmutableList.<RedisSlave>copyOf((Collection) cluster.getSlaves().getMembers());

    assertEquals(slaves.size(), 3);

    JedisSupport viaMaster = new JedisSupport(master);
    viaMaster.writeData(key, val);
    assertEquals(viaMaster.readData(key), val);

    for (RedisSlave slave : slaves) {
      final JedisSupport viaSlave = new JedisSupport(slave);
      Asserts.succeedsEventually(
          new Callable<Void>() {
            @Override
            public Void call() throws Exception {
              assertEquals(viaSlave.readData(key), val);
              return null;
            }
          });
    }

    // Check that stopping slave will not stop anything else
    // (it used to stop master because wasn't supplying port!)
    slaves.get(0).stop();
    EntityTestUtils.assertAttributeEqualsEventually(slaves.get(0), Startable.SERVICE_UP, false);

    assertEquals(master.getAttribute(Startable.SERVICE_UP), Boolean.TRUE);
    for (RedisSlave slave : slaves.subList(1, slaves.size())) {
      assertEquals(slave.getAttribute(Startable.SERVICE_UP), Boolean.TRUE);
    }

    // Check that stopping cluster will stop everything
    cluster.stop();

    EntityTestUtils.assertAttributeEqualsEventually(cluster, Startable.SERVICE_UP, false);
    assertEquals(master.getAttribute(Startable.SERVICE_UP), Boolean.FALSE);
    for (RedisSlave slave : slaves) {
      assertEquals(slave.getAttribute(Startable.SERVICE_UP), Boolean.FALSE);
    }
  }
 @Test
 public void testComplex() throws InterruptedException, ExecutionException {
   Task<List<?>> t =
       Tasks.sequential(
           sayTask("1"), sayTask("2"), Tasks.parallel(sayTask("4"), sayTask("3")), sayTask("5"));
   ec.submit(t);
   Assert.assertEquals(t.get().size(), 4);
   Asserts.assertEqualsIgnoringOrder((List<?>) t.get().get(2), ImmutableSet.of("3", "4"));
   Assert.assertTrue(
       messages.equals(Arrays.asList("1", "2", "3", "4", "5"))
           || messages.equals(Arrays.asList("1", "2", "4", "3", "5")),
       "messages=" + messages);
 }
 private void assertHasEventEventually(
     final Sensor<?> sensor,
     final Predicate<Object> componentPredicate,
     final Predicate<? super CharSequence> descriptionPredicate) {
   Asserts.succeedsEventually(
       MutableMap.of("timeout", TIMEOUT_MS),
       new Runnable() {
         @Override
         public void run() {
           assertHasEvent(sensor, componentPredicate, descriptionPredicate);
         }
       });
 }
  @Test(groups = "Integration")
  public void testJavaStartStopSshDriverStartsAndStopsApp() {
    final MyEntity entity = app.createAndManageChild(EntitySpecs.spec(MyEntity.class));
    app.start(ImmutableList.of(localhost));
    Asserts.succeedsEventually(
        MutableMap.of("timeout", TIMEOUT_MS),
        new Runnable() {
          public void run() {
            assertTrue(entity.getAttribute(SoftwareProcess.SERVICE_UP));
          }
        });

    entity.stop();
    assertFalse(entity.getAttribute(SoftwareProcess.SERVICE_UP));
  }
  @Test
  public void testRebindsSubscriptions() throws Exception {
    MyEntity2 origE =
        origApp.createAndManageChild(
            EntitySpec.create(MyEntity2.class).configure("subscribe", true));

    newApp = rebind();
    MyEntity2 newE =
        (MyEntity2) Iterables.find(newApp.getChildren(), Predicates.instanceOf(MyEntity2.class));

    newApp.setAttribute(TestApplication.MY_ATTRIBUTE, "mysensorval");
    Asserts.eventually(
        Suppliers.ofInstance(newE.getEvents()),
        Predicates.<List<String>>equalTo(ImmutableList.of("mysensorval")));
    Assert.assertEquals(newE, origE);
  }
  @Test
  public void testModelIncludesItemsAndContainersStartedBeforePolicyCreated() {
    pool.removePolicy(policy);
    policy.destroy();

    // Set-up containers and items.
    final MockContainerEntity containerA = newContainer(app, "A", 10, 100);
    MockItemEntity item1 = newItem(app, containerA, "1", 10);

    policy = new LoadBalancingPolicy(MutableMap.of(), TEST_METRIC, model);
    pool.addPolicy(policy);

    Asserts.succeedsEventually(
        MutableMap.of("timeout", TIMEOUT_MS),
        new Runnable() {
          public void run() {
            assertEquals(model.getContainerWorkrates(), ImmutableMap.of(containerA, 10d));
          }
        });
  }
  @Test
  public void testServiceRestarterWorksAfterRebind() throws Exception {
    origEntity.addPolicy(
        PolicySpec.create(ServiceRestarter.class)
            .configure(ServiceRestarter.FAILURE_SENSOR_TO_MONITOR, HASensors.ENTITY_FAILED));

    TestApplication newApp = rebind();
    final TestEntity newEntity =
        (TestEntity) Iterables.find(newApp.getChildren(), Predicates.instanceOf(TestEntity.class));

    newEntity.emit(HASensors.ENTITY_FAILED, new FailureDescriptor(origEntity, "simulate failure"));

    Asserts.succeedsEventually(
        new Runnable() {
          @Override
          public void run() {
            assertEquals(newEntity.getCallHistory(), ImmutableList.of("restart"));
          }
        });
  }
  @Test(groups = "Integration")
  public void testSshCacheExpiresEvenIfNotUsed() throws Exception {
    SshMachineLocation host2 =
        managementContext
            .getLocationManager()
            .createLocation(
                LocationSpec.create(SshMachineLocation.class)
                    .configure("address", InetAddress.getLocalHost())
                    .configure(SshMachineLocation.SSH_CACHE_EXPIRY_DURATION, Duration.ONE_SECOND)
                    .configure(SshTool.PROP_TOOL_CLASS, RecordingSshjTool.class.getName()));

    Map<String, Object> props = customSshConfigKeys();
    host2.execScript(props, "mysummary", ImmutableList.of("exit"));

    Asserts.succeedsEventually(
        new Runnable() {
          @Override
          public void run() {
            assertEquals(RecordingSshjTool.disconnectionCount.get(), 1);
          }
        });
  }
  @Test
  public void testSubscriptionReceivesEvents() throws Exception {
    policy.subscribe(entity, TestEntity.SEQUENCE, listener);
    policy.subscribe(entity, TestEntity.NAME, listener);
    policy.subscribe(entity, TestEntity.MY_NOTIF, listener);

    otherEntity.setAttribute(TestEntity.SEQUENCE, 456);
    entity.setAttribute(TestEntity.SEQUENCE, 123);
    entity.setAttribute(TestEntity.NAME, "myname");
    entity.emit(TestEntity.MY_NOTIF, 789);

    Asserts.succeedsEventually(
        new Runnable() {
          @Override
          public void run() {
            assertEquals(
                listener.events,
                ImmutableList.of(
                    new BasicSensorEvent<Integer>(TestEntity.SEQUENCE, entity, 123),
                    new BasicSensorEvent<String>(TestEntity.NAME, entity, "myname"),
                    new BasicSensorEvent<Integer>(TestEntity.MY_NOTIF, entity, 789)));
          }
        });
  }
  @Test
  public void testUnsubscribeUsingHandleStopsEvents() throws Exception {
    SubscriptionHandle handle1 = policy.subscribe(entity, TestEntity.SEQUENCE, listener);
    SubscriptionHandle handle2 = policy.subscribe(entity, TestEntity.NAME, listener);
    SubscriptionHandle handle3 = policy.subscribe(otherEntity, TestEntity.SEQUENCE, listener);

    policy.unsubscribe(entity, handle2);

    entity.setAttribute(TestEntity.SEQUENCE, 123);
    entity.setAttribute(TestEntity.NAME, "myname");
    otherEntity.setAttribute(TestEntity.SEQUENCE, 456);

    Asserts.succeedsEventually(
        new Runnable() {
          @Override
          public void run() {
            assertEquals(
                listener.events,
                ImmutableList.of(
                    new BasicSensorEvent<Integer>(TestEntity.SEQUENCE, entity, 123),
                    new BasicSensorEvent<Integer>(TestEntity.SEQUENCE, otherEntity, 456)));
          }
        });
  }
  @Test(groups = "Integration") // takes more than 4 seconds, due to assertContinually calls
  public void testSubscriptionAndPublishingOnlyActiveWhenEntityIsManaged() throws Exception {
    MyLatchingEntityImpl.latching = false;
    origApp.createAndManageChild(
        EntitySpec.create(MyLatchingEntity.class)
            .configure("subscribe", TestApplication.MY_ATTRIBUTE)
            .configure("publish", "myvaltopublish"));
    MyLatchingEntityImpl.reset(); // after origE has been managed
    MyLatchingEntityImpl.latching = true;

    // Serialize and rebind, but don't yet manage the app
    RebindTestUtils.waitForPersisted(origApp);
    RebindTestUtils.checkCurrentMementoSerializable(origApp);
    newManagementContext = new LocalManagementContext();
    Thread thread =
        new Thread() {
          public void run() {
            try {
              RebindTestUtils.rebind(newManagementContext, mementoDir, getClass().getClassLoader());
            } catch (Exception e) {
              throw Throwables.propagate(e);
            }
          }
        };
    try {
      thread.start();
      final List<Object> events = new CopyOnWriteArrayList<Object>();

      newManagementContext
          .getSubscriptionManager()
          .subscribe(
              null,
              MyLatchingEntityImpl.MY_SENSOR,
              new SensorEventListener<Object>() {
                @Override
                public void onEvent(SensorEvent<Object> event) {
                  events.add(event.getValue());
                }
              });

      // In entity's reconstruct, publishes events are queued, and subscriptions don't yet take
      // effect
      assertTrue(Durations.await(MyLatchingEntityImpl.reconstructStartedLatch, TIMEOUT_MS));
      newManagementContext
          .getSubscriptionManager()
          .publish(
              new BasicSensorEvent<String>(TestApplication.MY_ATTRIBUTE, null, "myvaltooearly"));

      Asserts.continually(
          Suppliers.ofInstance(MyLatchingEntityImpl.events),
          Predicates.equalTo(Collections.emptyList()));
      Asserts.continually(
          Suppliers.ofInstance(events), Predicates.equalTo(Collections.emptyList()));

      // When the entity is notified of "managing", then subscriptions take effect (but missed
      // events not delivered);
      // published events remain queued
      MyLatchingEntityImpl.reconstructContinuesLatch.countDown();
      assertTrue(MyLatchingEntityImpl.managingStartedLatch.getCount() > 0);

      Asserts.continually(
          Suppliers.ofInstance(events), Predicates.equalTo(Collections.emptyList()));
      Asserts.continually(
          Suppliers.ofInstance(MyLatchingEntityImpl.events),
          Predicates.equalTo(Collections.emptyList()));

      newManagementContext
          .getSubscriptionManager()
          .publish(
              new BasicSensorEvent<String>(TestApplication.MY_ATTRIBUTE, null, "myvaltoreceive"));
      Asserts.eventually(
          Suppliers.ofInstance(MyLatchingEntityImpl.events),
          Predicates.<List<Object>>equalTo(ImmutableList.of((Object) "myvaltoreceive")));

      // When the entity is notified of "managed", its events are only then delivered
      MyLatchingEntityImpl.managingContinuesLatch.countDown();
      assertTrue(Durations.await(MyLatchingEntityImpl.managedStartedLatch, TIMEOUT_MS));

      Asserts.eventually(
          Suppliers.ofInstance(MyLatchingEntityImpl.events),
          Predicates.<List<Object>>equalTo(ImmutableList.of((Object) "myvaltoreceive")));

      MyLatchingEntityImpl.managedContinuesLatch.countDown();

      Durations.join(thread, TIMEOUT_MS);
      assertFalse(thread.isAlive());

    } finally {
      thread.interrupt();
      MyLatchingEntityImpl.reset();
    }
  }
  private void assertAppFunctional(StartableApplication app) throws Exception {
    // expect standard config to (still) be set
    assertNotNull(app.getConfig(WebClusterDatabaseExampleApp.WAR_PATH));
    assertEquals(app.getConfig(WebClusterDatabaseExampleApp.USE_HTTPS), Boolean.FALSE);
    assertNotNull(app.getConfig(WebClusterDatabaseExampleApp.DB_SETUP_SQL_URL));

    // expect entities to be there
    MySqlNode mysql =
        (MySqlNode) Iterables.find(app.getChildren(), Predicates.instanceOf(MySqlNode.class));
    ControlledDynamicWebAppCluster web =
        (ControlledDynamicWebAppCluster)
            Iterables.find(
                app.getChildren(), Predicates.instanceOf(ControlledDynamicWebAppCluster.class));
    final NginxController nginx =
        (NginxController)
            Iterables.find(web.getChildren(), Predicates.instanceOf(NginxController.class));
    DynamicWebAppCluster webCluster =
        (DynamicWebAppCluster)
            Iterables.find(web.getChildren(), Predicates.instanceOf(DynamicWebAppCluster.class));
    Collection<Entity> appservers = web.getMembers();
    assertEquals(appservers.size(), 2);
    String clusterUrl =
        checkNotNull(app.getAttribute(WebClusterDatabaseExampleApp.ROOT_URL), "cluster url");
    String dbUrl = checkNotNull(mysql.getAttribute(MySqlNode.DATASTORE_URL), "database url");
    final String expectedJdbcUrl =
        String.format(
            "jdbc:%s%s?user=%s\\&password=%s",
            dbUrl,
            WebClusterDatabaseExampleApp.DB_TABLE,
            WebClusterDatabaseExampleApp.DB_USERNAME,
            WebClusterDatabaseExampleApp.DB_PASSWORD);

    // expect web-app to be reachable, and wired up to database
    HttpTestUtils.assertHttpStatusCodeEventuallyEquals(clusterUrl, 200);
    for (Entity appserver : appservers) {
      String appserverUrl =
          checkNotNull(
              appserver.getAttribute(JBoss7Server.ROOT_URL), "appserver url of " + appserver);

      HttpTestUtils.assertHttpStatusCodeEventuallyEquals(appserverUrl, 200);
      assertEquals(
          expectedJdbcUrl,
          appserver.getConfig(JavaEntityMethods.javaSysProp("brooklyn.example.db.url")),
          "of " + appserver);
    }

    WebAppMonitor monitor = newWebAppMonitor(clusterUrl, 200);

    // expect auto-scaler policy to be there, and to be functional (e.g. can trigger resize)
    AutoScalerPolicy autoScalerPolicy =
        (AutoScalerPolicy)
            Iterables.find(webCluster.getPolicies(), Predicates.instanceOf(AutoScalerPolicy.class));

    autoScalerPolicy.config().set(AutoScalerPolicy.MIN_POOL_SIZE, 3);
    EntityTestUtils.assertGroupSizeEqualsEventually(web, 3);
    final Collection<Entity> webMembersAfterGrow = web.getMembers();

    for (final Entity appserver : webMembersAfterGrow) {
      Asserts.succeedsEventually(
          MutableMap.of("timeout", Duration.TWO_MINUTES),
          new Runnable() {
            @Override
            public void run() {
              String appserverUrl =
                  checkNotNull(
                      appserver.getAttribute(JBoss7Server.ROOT_URL),
                      "appserver url of " + appserver);
              HttpTestUtils.assertHttpStatusCodeEquals(appserverUrl, 200);
              assertEquals(
                  expectedJdbcUrl,
                  appserver.getConfig(JavaEntityMethods.javaSysProp("brooklyn.example.db.url")),
                  "of " + appserver);
              Asserts.assertEqualsIgnoringOrder(
                  nginx.getAttribute(NginxController.SERVER_POOL_TARGETS).keySet(),
                  webMembersAfterGrow);
            }
          });
    }

    // expect enrichers to be there
    Iterables.find(web.getEnrichers(), Predicates.instanceOf(HttpLatencyDetector.class));
    Iterable<Enricher> propagatorEnrichers =
        Iterables.filter(web.getEnrichers(), Predicates.instanceOf(Propagator.class));
    assertEquals(
        Iterables.size(propagatorEnrichers), 2, "propagatorEnrichers=" + propagatorEnrichers);

    // Check we see evidence of the enrichers having an effect.
    // Relying on WebAppMonitor to stimulate activity.
    EntityTestUtils.assertAttributeEqualsEventually(
        app, WebClusterDatabaseExampleApp.APPSERVERS_COUNT, 3);
    EntityTestUtils.assertAttributeChangesEventually(
        web, DynamicWebAppCluster.REQUESTS_PER_SECOND_IN_WINDOW);
    EntityTestUtils.assertAttributeChangesEventually(
        app, DynamicWebAppCluster.REQUESTS_PER_SECOND_IN_WINDOW);
    EntityTestUtils.assertAttributeChangesEventually(
        web, HttpLatencyDetector.REQUEST_LATENCY_IN_SECONDS_MOST_RECENT);
    EntityTestUtils.assertAttributeChangesEventually(
        web, HttpLatencyDetector.REQUEST_LATENCY_IN_SECONDS_IN_WINDOW);

    // Restore the web-cluster to its original size of 2
    autoScalerPolicy.config().set(AutoScalerPolicy.MIN_POOL_SIZE, 2);
    EntityTestUtils.assertGroupSizeEqualsEventually(web, 2);

    final Entity removedAppserver =
        Iterables.getOnlyElement(
            Sets.difference(
                ImmutableSet.copyOf(webMembersAfterGrow), ImmutableSet.copyOf(web.getMembers())));
    Asserts.succeedsEventually(
        new Runnable() {
          @Override
          public void run() {
            assertFalse(Entities.isManaged(removedAppserver));
          }
        });

    monitor.assertNoFailures("hitting nginx url");
    monitor.terminate();
  }