@Test
 public void testStartBackupFirst() throws Exception {
   for (TestableServer backupServer : backupServers) {
     backupServer.start();
   }
   for (TestableServer liveServer : liveServers) {
     liveServer.start();
   }
   waitForTopology(liveServers.get(0).getServer(), liveServers.size(), liveServers.size());
   sendCrashReceive();
 }
  /**
   * @param node The node which we should fail
   * @param originalLiveNode The number of the original node, to locate session to fail
   * @throws Exception
   */
  protected void failNode(final int node, final int originalLiveNode) throws Exception {
    ClusterWithBackupFailoverTestBase.log.info("*** failing node " + node);

    HornetQServer server = getServer(node);

    TestableServer tstServer = new SameProcessHornetQServer(server);

    ClientSession[] sessionsArray = exploreSessions(originalLiveNode);

    tstServer.crash(sessionsArray);
  }
 /**
  * Waits for backup to be in the "started" state and to finish synchronization with its live.
  *
  * @param sessionFactory
  * @param seconds
  * @throws Exception
  */
 protected void waitForBackup(ClientSessionFactoryInternal sessionFactory, int seconds)
     throws Exception {
   final HornetQServerImpl actualServer = (HornetQServerImpl) backupServer.getServer();
   if (actualServer.getConfiguration().isSharedStore()) {
     waitForServer(actualServer);
   } else {
     waitForRemoteBackup(sessionFactory, seconds, true, actualServer);
   }
 }
  @Override
  protected void setUp() throws Exception {
    super.setUp();
    clearData();
    createConfigs();

    liveServer.setIdentity(this.getClass().getSimpleName() + "/liveServer");
    liveServer.start();
    waitForServer(liveServer.getServer());

    if (backupServer != null) {
      backupServer.setIdentity(this.getClass().getSimpleName() + "/backupServer");
      if (startBackupServer) {
        backupServer.start();
        waitForServer(backupServer.getServer());
      }
    }
  }
  protected void sendCrashReceive() throws Exception {
    ServerLocator[] locators = new ServerLocator[liveServers.size()];
    try {
      for (int i = 0; i < locators.length; i++) {
        locators[i] = getServerLocator(i);
      }

      ClientSessionFactory[] factories = new ClientSessionFactory[liveServers.size()];
      for (int i = 0; i < factories.length; i++) {
        factories[i] = createSessionFactory(locators[i]);
      }

      ClientSession[] sessions = new ClientSession[liveServers.size()];
      for (int i = 0; i < factories.length; i++) {
        sessions[i] = createSession(factories[i], true, true);
        sessions[i].createQueue(
            MultipleServerFailoverTestBase.ADDRESS,
            MultipleServerFailoverTestBase.ADDRESS,
            null,
            true);
      }

      // make sure bindings are ready before sending messages
      for (int i = 0; i < liveServers.size(); i++) {
        this.waitForBindings(liveServers.get(i).getServer(), ADDRESS.toString(), true, 1, 0, 2000);
        this.waitForBindings(liveServers.get(i).getServer(), ADDRESS.toString(), false, 1, 0, 2000);
      }

      ClientProducer producer = sessions[0].createProducer(MultipleServerFailoverTestBase.ADDRESS);

      for (int i = 0; i < liveServers.size() * 100; i++) {
        ClientMessage message = sessions[0].createMessage(true);

        setBody(i, message);

        message.putIntProperty("counter", i);

        producer.send(message);
      }

      producer.close();

      for (TestableServer liveServer : liveServers) {
        waitForDistribution(MultipleServerFailoverTestBase.ADDRESS, liveServer.getServer(), 100);
      }

      for (TestableServer liveServer : liveServers) {
        liveServer.crash();
      }
      ClientConsumer[] consumers = new ClientConsumer[liveServers.size()];
      for (int i = 0; i < factories.length; i++) {
        consumers[i] = sessions[i].createConsumer(MultipleServerFailoverTestBase.ADDRESS);
        sessions[i].start();
      }

      for (int i = 0; i < 100; i++) {
        for (ClientConsumer consumer : consumers) {
          ClientMessage message = consumer.receive(1000);
          Assert.assertNotNull("expecting durable msg " + i, message);
          message.acknowledge();
        }
      }
    } finally {
      for (ServerLocator locator : locators) {
        if (locator != null) {
          try {
            locator.close();
          } catch (Exception e) {
            // ignore
          }
        }
      }
    }
  }
 protected void crash(final boolean waitFailure, final ClientSession... sessions)
     throws Exception {
   liveServer.crash(waitFailure, sessions);
 }
 protected void crash(final ClientSession... sessions) throws Exception {
   liveServer.crash(sessions);
 }