public void startAppServer(@Observes(precedence = -1) BeforeClass event)
     throws MalformedURLException, InterruptedException, IOException {
   if (testContext.isAdapterTest()) {
     ContainerController controller = containerConrollerInstance.get();
     if (!controller.isStarted(testContext.getAppServerInfo().getQualifier())) {
       controller.start(testContext.getAppServerInfo().getQualifier());
     }
   }
 }
  @Test
  @WithRunningServer({@RunningServer(name = CONTAINER1), @RunningServer(name = CONTAINER2)})
  public void testRebalanceWithFirstNodeStop() throws Exception {

    try {
      verifyRebalanceWith3rdNode();

      // Disabling rebalance.
      setAttribute(provider(0), LOCAL_TOPOLOGY_MANAGER, REBALANCE_ENABLED_ATTR_NAME, false);
      controller.stop(CONTAINER1);
      checkRpcManagerStatistics(
          new String[] {"null", OWNERS_2_MEMBERS_NODE1_NODE2},
          OWNERS_2_MEMBERS_NODE1_NODE2,
          provider(1),
          provider(2));
      checkRebalanceStatus(false, provider(1), provider(2));

      assertTrue(
          server(1)
                  .getCacheManager(getCacheManagerName())
                  .getCache(getCacheName())
                  .getNumberOfEntries()
              < NUMBER_ENTRIES);
      assertTrue(
          server(2)
                  .getCacheManager(getCacheManagerName())
                  .getCache(getCacheName())
                  .getNumberOfEntries()
              < NUMBER_ENTRIES);

      // Enabling rebalance
      setAttribute(provider(1), LOCAL_TOPOLOGY_MANAGER, REBALANCE_ENABLED_ATTR_NAME, true);
      checkRebalanceStatus(true, provider(1), provider(2));

      checkRpcManagerStatistics(
          new String[] {"null", OWNERS_2_MEMBERS_NODE1_NODE2},
          OWNERS_2_MEMBERS_NODE1_NODE2,
          provider(1),
          provider(2));

      assertTrue(
          server(1)
                  .getCacheManager(getCacheManagerName())
                  .getCache(getCacheName())
                  .getNumberOfEntries()
              == NUMBER_ENTRIES);
      assertTrue(
          server(2)
                  .getCacheManager(getCacheManagerName())
                  .getCache(getCacheName())
                  .getNumberOfEntries()
              == NUMBER_ENTRIES);
    } finally {
      controller.stop(CONTAINER3);
    }
  }
 @After
 public void tearDown() throws Exception {
   if (container.isStarted(SERVER1)) {
     container.stop(SERVER1);
   }
   restoreSnapshot(snapshotForServer1);
   if (container.isStarted(SERVER2)) {
     container.stop(SERVER2);
   }
   restoreSnapshot(snapshotForServer2);
 }
  @Test
  @InSequence(1)
  public void stopServers(
      @ArquillianResource @OperateOnDeployment(DEPLOYMENT_1) ManagementClient client1,
      @ArquillianResource @OperateOnDeployment(DEPLOYMENT_2) ManagementClient client2)
      throws Exception {

    SSOTestBase.removeSso(client1.getControllerClient());
    SSOTestBase.removeSso(client2.getControllerClient());

    deployer.undeploy(DEPLOYMENT_1);
    controller.stop(CONTAINER_1);
    deployer.undeploy(DEPLOYMENT_2);
    controller.stop(CONTAINER_2);
  }
  @Test
  @InSequence(-2)
  public void startServers() throws Exception {

    controller.start(CONTAINER_1);
    controller.start(CONTAINER_2);
  }
  @After
  public void stop() throws Exception {
    inputCache.getRemoteCacheManager().stop();
    outputCache.getRemoteCacheManager().stop();

    // If the last test is executed stop servers.
    if (testCounter == numberOfTestMethods) {
      if (controller.isStarted("container1")) {
        controller.stop("container1");
      }

      if (controller.isStarted("container2")) {
        controller.stop("container2");
      }
    }
  }
  @Test
  @WithRunningServer({@RunningServer(name = CONTAINER1), @RunningServer(name = CONTAINER2)})
  public void testRebalanceDisabledWithNewNode() throws Exception {

    try {
      verifyRebalanceWith3rdNode();
    } finally {
      controller.stop(CONTAINER3);
    }
  }
  @Test
  @InSequence(-1)
  public void setupSSO(
      @ArquillianResource @OperateOnDeployment(DEPLOYMENT_1) ManagementClient client1,
      @ArquillianResource @OperateOnDeployment(DEPLOYMENT_2) ManagementClient client2)
      throws Exception {

    // add sso valves
    SSOTestBase.addClusteredSso(client1.getControllerClient());
    SSOTestBase.addClusteredSso(client2.getControllerClient());

    controller.stop(CONTAINER_1);
    controller.stop(CONTAINER_2);

    controller.start(CONTAINER_1);
    controller.start(CONTAINER_2);

    deployer.deploy(DEPLOYMENT_1);
    deployer.deploy(DEPLOYMENT_2);
  }
 @Test
 @InSequence(2)
 public void shouldBeAbleToStartServer() {
   // server-groups are registered as STARTED by default for managed deployments to work..
   // In the eyes of the Domain they are always 'started' it's the servers in their group that are
   // started or stopped,
   // but Arquillian see them as one Container and won't actually call start unless it is
   // registered as stopped.... ??
   // controller.stop("other-server-group");
   // controller.start("other-server-group");
   controller.start("frontend-1");
 }
 @After
 public void stopContainer() throws Exception {
   final ModelControllerClient client = TestSuiteEnvironment.getModelControllerClient();
   try {
     // Remove the servlet
     deployer.undeploy(DEPLOYMENT);
     // Stop the container
     container.stop(CONTAINER);
   } finally {
     safeClose(client);
   }
 }
 private void start(String deployment, String container) {
   try {
     System.out.println(
         new Date() + "starting deployment=" + deployment + ", container=" + container);
     controller.start(container);
     deployer.deploy(deployment);
     System.out.println(
         new Date() + "started deployment=" + deployment + ", container=" + container);
   } catch (Throwable e) {
     e.printStackTrace(System.err);
   }
 }
  @Before
  public void setUp() throws Exception {

    container.start(CONTAINER_0);
    container.start(CONTAINER_1);

    // both servers are started and configured
    assertTrue(container.isStarted(CONTAINER_0));
    assertTrue(container.isStarted(CONTAINER_1));

    try (ModelControllerClient client = createClient1()) {
      JMSOperations jmsOperations = JMSOperationsProvider.getInstance(client);
      jmsOperations.createJmsQueue(jmsQueueName, "java:jboss/exported/" + jmsQueueLookup);
      jmsOperations.createJmsTopic(jmsTopicName, "java:jboss/exported/" + jmsTopicLookup);
    }
    try (ModelControllerClient client = createClient2()) {
      JMSOperations jmsOperations = JMSOperationsProvider.getInstance(client);
      jmsOperations.createJmsQueue(jmsQueueName, "java:jboss/exported/" + jmsQueueLookup);
      jmsOperations.createJmsTopic(jmsTopicName, "java:jboss/exported/" + jmsTopicLookup);
    }
  }
  @Before
  public void setUp() throws Exception {

    // start server1 and reload it in admin-only
    container.start(SERVER1);
    ModelControllerClient client1 = createClient1();
    snapshotForServer1 = takeSnapshot(client1);
    executeReloadAndWaitForCompletionOfServer1(client1, true);
    client1 = createClient1();

    // start server2 and reload it in admin-only
    container.start(SERVER2);
    ModelControllerClient client2 = createClient2();
    snapshotForServer2 = takeSnapshot(client2);
    executeReloadAndWaitForCompletionOfServer2(client2, true);
    client2 = createClient2();

    // setup both servers
    try {
      setUpServer1(client1);
      setUpServer2(client2);
    } catch (Exception e) {
      tearDown();
      throw e;
    }

    // reload server1 in normal mode
    executeReloadAndWaitForCompletionOfServer1(client1, false);
    client1 = createClient1();

    // reload server2  in normal mode
    executeReloadAndWaitForCompletionOfServer2(client2, false);
    client2 = createClient2();

    // both servers are started and configured
    assertTrue(container.isStarted(SERVER1));
    client1.close();
    assertTrue(container.isStarted(SERVER2));
    client2.close();
  }
  @Before
  public void prepare() throws Exception {
    // If already started don't start.
    if (!controller.isStarted("container1")) {
      controller.start("container1");
    }

    if (!controller.isStarted("container2")) {
      controller.start("container2");
    }
    testCounter++;

    String host1Address = server1.getHotrodEndpoint().getInetAddress().getHostAddress();
    int port1 = server1.getHotrodEndpoint().getPort();

    String host2Address = server2.getHotrodEndpoint().getInetAddress().getHostAddress();
    int port2 = server2.getHotrodEndpoint().getPort();

    inputCache =
        new RemoteCacheManager(
                new ConfigurationBuilder().addServer().host(host1Address).port(port1).build())
            .getCache(INPUT_CACHE_NAME);

    outputCache =
        new RemoteCacheManager(
                new ConfigurationBuilder().addServer().host(host2Address).port(port2).build())
            .getCache(OUTPUT_CACHE_NAME);

    inputCache.clear();
    outputCache.clear();

    List<WebPage> webPages = createData();

    saveToInputCache(webPages);
    saveToHDFS(miniHadoopCluster, webPages);
  }
  @Before
  public void startContainer() throws Exception {
    // Start the server
    container.start(CONTAINER);
    final ManagementClient managementClient = createManagementClient();
    final ModelControllerClient client = managementClient.getControllerClient();
    try {
      Assert.assertTrue("Container is not started", managementClient.isServerInRunningState());

      // Deploy the servlet
      deployer.deploy(DEPLOYMENT);
    } finally {
      safeClose(managementClient);
      safeClose(client);
    }
  }
  @After
  public void tearDown() throws Exception {
    try (ModelControllerClient client = createClient1()) {
      JMSOperations jmsOperations = JMSOperationsProvider.getInstance(client);
      jmsOperations.removeJmsQueue(jmsQueueName);
      jmsOperations.removeJmsTopic(jmsTopicName);
    }
    try (ModelControllerClient client = createClient2()) {
      JMSOperations jmsOperations = JMSOperationsProvider.getInstance(client);
      jmsOperations.removeJmsQueue(jmsQueueName);
      jmsOperations.removeJmsTopic(jmsTopicName);
    }

    container.stop(CONTAINER_0);
    container.stop(CONTAINER_1);
  }
  @Test
  public void testAuditLoggingFields() throws Exception {
    container.start(CONTAINER);
    if (file.exists()) {
      file.delete();
    }
    Assert.assertTrue(makeOneLog());

    List<ModelNode> logs = readFile(file, 1);
    ModelNode log = logs.get(0);
    Assert.assertEquals("core", log.get("type").asString());
    Assert.assertEquals("false", log.get("r/o").asString());
    Assert.assertEquals("false", log.get("booting").asString());
    Assert.assertTrue(log.get("version").isDefined());
    Assert.assertEquals("IAmAdmin", log.get("user").asString());
    Assert.assertFalse(log.get("domainUUID").isDefined());
    Assert.assertEquals("NATIVE", log.get("access").asString());
    Assert.assertTrue(log.get("remote-address").isDefined());
    Assert.assertEquals("true", log.get("success").asString());
    List<ModelNode> operations = log.get("ops").asList();
    Assert.assertEquals(1, operations.size());

    if (syslogFile.exists()) {
      syslogFile.delete();
      server.getConfig().removeAllEventHandlers();
      server
          .getConfig()
          .addEventHandler(
              new BlockedFileSyslogServerEventHandler(queue, syslogFile.getAbsolutePath(), false));
    }
    Assert.assertTrue(makeOneLog());
    queue.poll(15 * ADJUSTED_SECOND, TimeUnit.MILLISECONDS);
    List<ModelNode> syslogLogs = readFile(syslogFile, 1);
    ModelNode syslogLog = syslogLogs.get(0);
    Assert.assertEquals("core", syslogLog.get("type").asString());
    Assert.assertEquals("false", syslogLog.get("r/o").asString());
    Assert.assertEquals("false", syslogLog.get("booting").asString());
    Assert.assertTrue(log.get("version").isDefined());
    Assert.assertEquals("IAmAdmin", syslogLog.get("user").asString());
    Assert.assertFalse(syslogLog.get("domainUUID").isDefined());
    Assert.assertEquals("NATIVE", syslogLog.get("access").asString());
    Assert.assertTrue(syslogLog.get("remote-address").isDefined());
    Assert.assertEquals("true", syslogLog.get("success").asString());
    List<ModelNode> syslogOperations = syslogLog.get("ops").asList();
    Assert.assertEquals(1, syslogOperations.size());
  }
  @Test
  @InSequence(4)
  public void resetTestConfiguration() throws Exception {

    LOGGER.info("*** reseting test configuration");
    ModelControllerClient client = getNativeModelControllerClient();
    ManagementClient managementClient =
        new ManagementClient(
            client, TestSuiteEnvironment.getServerAddress(), MANAGEMENT_NATIVE_PORT, "remoting");

    resetHttpInterfaceConfiguration(client);

    // reload to apply changes
    CustomCLIExecutor.execute(null, RELOAD, NATIVE_CONTROLLER);
    CustomCLIExecutor.waitForServerToReload(MAX_RELOAD_TIME, NATIVE_CONTROLLER);

    keystoreFilesSetup.tearDown(managementClient, CONTAINER);
    managementNativeRealmSetup.tearDown(managementClient, CONTAINER);

    LOGGER.info("*** stopping container");
    containerController.stop(CONTAINER);
  }
  @Test
  @InSequence(-1)
  public void prepareServer() throws Exception {

    LOGGER.info("*** starting server");
    containerController.start(CONTAINER);
    final ModelControllerClient client = TestSuiteEnvironment.getModelControllerClient();
    ManagementClient mgmtClient =
        new ManagementClient(
            client,
            TestSuiteEnvironment.getServerAddress(),
            TestSuiteEnvironment.getServerPort(),
            "http-remoting");
    keystoreFilesSetup.setup(mgmtClient, CONTAINER);
    managementNativeRealmSetup.setup(mgmtClient, CONTAINER);

    mgmtClient.close();
    // To apply new security realm settings for http interface reload of
    // server is required
    LOGGER.info("*** restart server");
    CustomCLIExecutor.execute(null, RELOAD, NATIVE_CONTROLLER);
    CustomCLIExecutor.waitForServerToReload(MAX_RELOAD_TIME, NATIVE_CONTROLLER);
  }
  @After
  public void afterTest() throws Exception {
    // stop syslog server
    SyslogServer.shutdown();
    server.setThread(null);
    server.getConfig().removeAllEventHandlers();

    final ModelControllerClient client = TestSuiteEnvironment.getModelControllerClient();
    ModelNode op =
        Util.getWriteAttributeOperation(
            auditLogConfigAddress,
            AuditLogLoggerResourceDefinition.ENABLED.getName(),
            new ModelNode(false));
    client.execute(op);
    op =
        Util.getWriteAttributeOperation(
            mgmtRealmConfigAddress, "default-user", new ModelNode("$local"));
    client.execute(op);

    op = Util.getResourceRemoveOperation(addSyslogHandler);
    client.execute(op);
    op = Util.getResourceRemoveOperation(syslogHandlerAddress);
    client.execute(op);

    if (file.exists()) {
      file.delete();
    }
    if (syslogFile.exists()) {
      syslogFile.delete();
    }
    try {
      // Stop the container
      container.stop(CONTAINER);
    } finally {
      IoUtils.safeClose(client);
    }
  }
  @Test
  public void testHotRodRollingUpgradesDiffVersions() throws Exception {
    // Target node
    final int managementPortServer1 = 9990;
    MBeanServerConnectionProvider provider1;
    // Source node
    int managementPortServer2 = 10099; // jboss-as mgmt port
    MBeanServerConnectionProvider provider2;

    try {

      if (!Boolean.parseBoolean(System.getProperty("start.jboss.as.manually"))) {
        // start it by Arquillian
        controller.start("hotrod-rolling-upgrade-2-old");
        managementPortServer2 = 10090;
      }

      ConfigurationBuilder builder = new ConfigurationBuilder();
      builder
          .addServer()
          .host("127.0.0.1")
          .port(11322)
          .protocolVersion(ConfigurationProperties.PROTOCOL_VERSION_12);

      RemoteCacheManager rcm = new RemoteCacheManager(builder.build());
      final RemoteCache<String, String> c2 = rcm.getCache("default");

      c2.put("key1", "value1");
      assertEquals("value1", c2.get("key1"));

      for (int i = 0; i < 50; i++) {
        c2.put("keyLoad" + i, "valueLoad" + i);
      }

      controller.start("hotrod-rolling-upgrade-1");

      RemoteInfinispanMBeans s1 =
          createRemotes("hotrod-rolling-upgrade-1", "local", DEFAULT_CACHE_NAME);
      // hotrod.protocol.version, if explictily defined, is set in createRemotes() method
      final RemoteCache<Object, Object> c1 = createCache(s1);

      assertEquals(
          "Can't access etries stored in source node (target's RemoteCacheStore).",
          "value1",
          c1.get("key1"));

      provider2 = new MBeanServerConnectionProvider("127.0.0.1", managementPortServer2);

      final ObjectName rollMan =
          new ObjectName(
              "jboss."
                  + InfinispanSubsystem.SUBSYSTEM_NAME
                  + ":type=Cache,"
                  + "name=\"default(local)\","
                  + "manager=\"local\","
                  + "component=RollingUpgradeManager");

      invokeOperation(
          provider2,
          rollMan.toString(),
          "recordKnownGlobalKeyset",
          new Object[] {},
          new String[] {});

      provider1 =
          new MBeanServerConnectionProvider(
              s1.server.getHotrodEndpoint().getInetAddress().getHostName(), managementPortServer1);

      invokeOperation(
          provider1,
          rollMan.toString(),
          "synchronizeData",
          new Object[] {"hotrod"},
          new String[] {"java.lang.String"});

      invokeOperation(
          provider1,
          rollMan.toString(),
          "disconnectSource",
          new Object[] {"hotrod"},
          new String[] {"java.lang.String"});

      // is source (RemoteCacheStore) really disconnected?
      c2.put("disconnected", "source");
      assertEquals(
          "Can't obtain value from cache2 (source node).", "source", c2.get("disconnected"));
      assertNull(
          "Source node entries should NOT be accessible from target node (after RCS disconnection)",
          c1.get("disconnected"));

      // all entries migrated?
      assertEquals("Entry was not successfully migrated.", "value1", c1.get("key1"));
      for (int i = 0; i < 50; i++) {
        assertEquals(
            "Entry was not successfully migrated.", "valueLoad" + i, c1.get("keyLoad" + i));
      }
    } finally {
      if (controller.isStarted("hotrod-rolling-upgrade-1")) {
        controller.stop("hotrod-rolling-upgrade-1");
      }
      if (controller.isStarted("hotrod-rolling-upgrade-2-old")) {
        controller.stop("hotrod-rolling-upgrade-2-old");
      }
    }
  }
 @Before
 public void before() {
   controller.start(CONTAINER);
 }
  @Before
  public void beforeTest() throws Exception {
    file = new File(System.getProperty("jboss.home"));
    file = new File(file, "standalone");
    file = new File(file, "data");
    file = new File(file, "audit-log.log");
    if (file.exists()) {
      file.delete();
    }

    syslogFile = new File(System.getProperty("jboss.home"));
    syslogFile = new File(syslogFile, "standalone");
    syslogFile = new File(syslogFile, "data");
    syslogFile = new File(syslogFile, "syslog-audit-log.log");
    if (syslogFile.exists()) {
      syslogFile.delete();
    }

    // start and set syslog server
    server = SyslogServer.getInstance("udp");
    server.getConfig().setPort(PORT);
    queue = new LinkedBlockingQueue<String>();
    server
        .getConfig()
        .addEventHandler(
            new BlockedFileSyslogServerEventHandler(queue, syslogFile.getAbsolutePath(), false));
    SyslogServer.getThreadedInstance("udp");

    // Start the server
    container.start(CONTAINER);
    final ModelControllerClient client = TestSuiteEnvironment.getModelControllerClient();
    managementClient =
        new ManagementClient(
            client,
            TestSuiteEnvironment.getServerAddress(),
            TestSuiteEnvironment.getServerPort(),
            "http-remoting");

    ModelNode op;
    ModelNode result;

    mgmtRealmConfigAddress =
        PathAddress.pathAddress(
            PathElement.pathElement(CORE_SERVICE, MANAGEMENT),
            PathElement.pathElement(SECURITY_REALM, "ManagementRealm"),
            PathElement.pathElement(AUTHENTICATION, LOCAL));
    op =
        Util.getWriteAttributeOperation(
            mgmtRealmConfigAddress, "default-user", new ModelNode("IAmAdmin"));
    result = client.execute(op);

    auditLogConfigAddress =
        PathAddress.pathAddress(
            CoreManagementResourceDefinition.PATH_ELEMENT,
            AccessAuditResourceDefinition.PATH_ELEMENT,
            AuditLogLoggerResourceDefinition.PATH_ELEMENT);

    op =
        Util.getWriteAttributeOperation(
            auditLogConfigAddress,
            AuditLogLoggerResourceDefinition.ENABLED.getName(),
            new ModelNode(true));
    result = client.execute(op);
    Assert.assertEquals(
        result.get("failure-description").asString(), SUCCESS, result.get(OUTCOME).asString());

    ModelNode compositeOp = new ModelNode();
    compositeOp.get(OP).set(COMPOSITE);
    compositeOp.get(OP_ADDR).setEmptyList();
    ModelNode steps = compositeOp.get(STEPS);
    syslogHandlerAddress =
        PathAddress.pathAddress(
            PathElement.pathElement(CORE_SERVICE, MANAGEMENT),
            PathElement.pathElement(ACCESS, AUDIT),
            PathElement.pathElement(SYSLOG_HANDLER, SYSLOG_HANDLER_NAME));
    op = Util.createAddOperation(syslogHandlerAddress);
    op.get(FORMATTER).set(JSON_FORMATTER);
    op.get(SYSLOG_FORMAT).set("RFC5424");
    steps.add(op);
    op = new ModelNode();
    PathAddress syslogProtocol =
        PathAddress.pathAddress(syslogHandlerAddress, PathElement.pathElement(PROTOCOL, UDP));
    op = Util.createAddOperation(syslogProtocol);
    op.get("port").set(PORT);
    op.get("host").set("localhost");
    steps.add(op);
    result = client.execute(compositeOp);
    Assert.assertEquals(
        result.get("failure-description").asString(), SUCCESS, result.get(OUTCOME).asString());

    addSyslogHandler =
        PathAddress.pathAddress(
            auditLogConfigAddress, PathElement.pathElement(HANDLER, SYSLOG_HANDLER_NAME));
    op = Util.createAddOperation(addSyslogHandler);
    result = client.execute(op);
    Assert.assertEquals(
        result.get("failure-description").asString(), SUCCESS, result.get(OUTCOME).asString());

    container.stop(CONTAINER);
    Thread.sleep(1000);
    while (managementClient.isServerInRunningState()) {
      Thread.sleep(50);
    }
  }
 @After
 public void after() {
   controller.stop(CONTAINER);
 }
  @Test
  public void testHotRodRollingUpgradesDiffVersions() throws Exception {
    // Target node
    final int managementPortServer1 = 9999;
    MBeanServerConnectionProvider provider1;
    // Source node
    final int managementPortServer2 = 10099;
    MBeanServerConnectionProvider provider2;

    controller.start("hotrod-rolling-upgrade-2-old");
    try {
      // we use PROTOCOL_VERSION_12 here because older servers does not support higher versions
      RemoteInfinispanMBeans s2 =
          createRemotes("hotrod-rolling-upgrade-2-old", "local", DEFAULT_CACHE_NAME);
      final RemoteCache<Object, Object> c2 =
          createCache(s2, ConfigurationProperties.PROTOCOL_VERSION_12);

      c2.put("key1", "value1");
      assertEquals("value1", c2.get("key1"));

      for (int i = 0; i < 50; i++) {
        c2.put("keyLoad" + i, "valueLoad" + i);
      }

      controller.start("hotrod-rolling-upgrade-1");

      RemoteInfinispanMBeans s1 =
          createRemotes("hotrod-rolling-upgrade-1", "local", DEFAULT_CACHE_NAME);
      final RemoteCache<Object, Object> c1 = createCache(s1);

      assertEquals(
          "Can't access etries stored in source node (target's RemoteCacheStore).",
          "value1",
          c1.get("key1"));

      provider1 =
          new MBeanServerConnectionProvider(
              s1.server.getHotrodEndpoint().getInetAddress().getHostName(), managementPortServer1);
      provider2 =
          new MBeanServerConnectionProvider(
              s2.server.getHotrodEndpoint().getInetAddress().getHostName(), managementPortServer2);

      final ObjectName rollMan =
          new ObjectName(
              "jboss.infinispan:type=Cache,"
                  + "name=\"default(local)\","
                  + "manager=\"local\","
                  + "component=RollingUpgradeManager");

      invokeOperation(
          provider2,
          rollMan.toString(),
          "recordKnownGlobalKeyset",
          new Object[] {},
          new String[] {});

      invokeOperation(
          provider1,
          rollMan.toString(),
          "synchronizeData",
          new Object[] {"hotrod"},
          new String[] {"java.lang.String"});

      invokeOperation(
          provider1,
          rollMan.toString(),
          "disconnectSource",
          new Object[] {"hotrod"},
          new String[] {"java.lang.String"});

      // is source (RemoteCacheStore) really disconnected?
      c2.put("disconnected", "source");
      assertEquals(
          "Can't obtain value from cache2 (source node).", "source", c2.get("disconnected"));
      assertNull(
          "Source node entries should NOT be accessible from target node (after RCS disconnection)",
          c1.get("disconnected"));

      // all entries migrated?
      assertEquals("Entry was not successfully migrated.", "value1", c1.get("key1"));
      for (int i = 0; i < 50; i++) {
        assertEquals(
            "Entry was not successfully migrated.", "valueLoad" + i, c1.get("keyLoad" + i));
      }
    } finally {
      if (controller.isStarted("hotrod-rolling-upgrade-1")) {
        controller.stop("hotrod-rolling-upgrade-1");
      }
      if (controller.isStarted("hotrod-rolling-upgrade-2-old")) {
        controller.stop("hotrod-rolling-upgrade-2-old");
      }
    }
  }
 @Test
 @InSequence(4)
 public void shouldBeAbleToStopServer() {
   controller.stop("frontend-1");
 }
 private void restart(final ManagementClient managementClient) {
   container.stop(CONTAINER);
   container.start(CONTAINER);
   // Just make sure it's running
   Assert.assertTrue("Container is not started", managementClient.isServerInRunningState());
 }
  private void verifyRebalanceWith3rdNode() throws Exception {
    // Disabling Rebalance for verifying the join of the 3rd node.
    setAttribute(provider(0), LOCAL_TOPOLOGY_MANAGER, REBALANCE_ENABLED_ATTR_NAME, false);

    // putting data into cache before adding new node
    putDataIntoCache(NUMBER_ENTRIES);

    // Verifying that the rebalance is disabled.
    checkRebalanceStatus(false, provider(0), provider(1));
    checkRpcManagerStatistics(
        new String[] {"null"}, OWNERS_2_MEMBERS_NODE0_NODE1, provider(0), provider(1));

    controller.start(CONTAINER3);
    createNewProvider(2);

    checkRebalanceStatus(false, provider(2));

    checkRpcManagerStatistics(
        new String[] {"null"}, OWNERS_2_MEMBERS_NODE0_NODE1, provider(0), provider(1), provider(2));

    assertTrue(
        "The cache on server(2) should be empty.",
        server(2)
                .getCacheManager(getCacheManagerName())
                .getCache(getCacheName())
                .getNumberOfEntries()
            == 0);

    // Enabling the Rebalance and verifying that the consistent rehash takes place.
    setAttribute(provider(0), LOCAL_TOPOLOGY_MANAGER, REBALANCE_ENABLED_ATTR_NAME, true);
    checkRebalanceStatus(true, provider(0), provider(1), provider(2));

    checkRpcManagerStatistics(
        new String[] {"null", OWNERS_2_MEMBERS_NODE0_NODE1_NODE2},
        null,
        provider(0),
        provider(1),
        provider(2));

    // Waiting for rehash take place.
    checkRpcManagerStatistics(
        new String[] {"null"},
        OWNERS_2_MEMBERS_NODE0_NODE1_NODE2,
        provider(0),
        provider(1),
        provider(2));

    assertTrue(
        server(0)
                .getCacheManager(getCacheManagerName())
                .getCache(getCacheName())
                .getNumberOfEntries()
            < NUMBER_ENTRIES);
    assertTrue(
        server(1)
                .getCacheManager(getCacheManagerName())
                .getCache(getCacheName())
                .getNumberOfEntries()
            < NUMBER_ENTRIES);
    assertTrue(
        server(2)
                .getCacheManager(getCacheManagerName())
                .getCache(getCacheName())
                .getNumberOfEntries()
            < NUMBER_ENTRIES);
  }