@Test
  public void testNoAutoFailback() throws Exception {
    locator
        .setBlockOnNonDurableSend(true)
        .setBlockOnDurableSend(true)
        .setFailoverOnInitialConnection(true)
        .setReconnectAttempts(-1);

    ClientSessionFactoryInternal sf = createSessionFactoryAndWaitForTopology(locator, 2);

    ClientSession session = sendAndConsume(sf, true);

    CountDownSessionFailureListener listener = new CountDownSessionFailureListener(1, session);

    session.addFailureListener(listener);

    backupServer.stop();

    liveServer.crash();

    backupServer.start();

    assertTrue(listener.getLatch().await(5, TimeUnit.SECONDS));

    ClientProducer producer = session.createProducer(ADDRESS);

    ClientMessage message = session.createMessage(true);

    setBody(0, message);

    producer.send(message);

    session.removeFailureListener(listener);

    Thread t = new Thread(new ServerStarter(liveServer));

    t.start();

    waitForRemoteBackup(sf, 10, false, backupServer.getServer());

    assertTrue(backupServer.isStarted());

    backupServer.crash();

    waitForServerToStart(liveServer.getServer());

    assertTrue(liveServer.isStarted());

    sf.close();

    assertEquals(0, sf.numSessions());

    assertEquals(0, sf.numConnections());
  }
  /* Hook for creating session factory */
  protected ClientSessionFactoryInternal createSessionFactory() throws Exception {
    if (targetNodeID != null
        && (this.reconnectAttemptsSameNode < 0 || retryCount <= this.reconnectAttemptsSameNode)) {
      csf = reconnectOnOriginalNode();
    } else {
      serverLocator.resetToInitialConnectors();
      csf = (ClientSessionFactoryInternal) serverLocator.createSessionFactory();
    }

    // null here means the targetNodeIS is not available yet
    if (csf != null) {
      csf.setReconnectAttempts(0);
    }

    return csf;
  }
  private void doTestTransactional(final TestRunner runner) throws Throwable {
    // For duplication detection
    int executionId = 0;

    while (!runner.isFailed()) {
      ClientSession session = null;

      executionId++;

      log.info("#test doTestTransactional starting now. Execution " + executionId);

      try {

        boolean retry = false;

        final int numMessages = 1000;

        session = sf.createSession(false, false);

        listener = new CountDownSessionFailureListener(session);
        session.addFailureListener(listener);

        do {
          try {
            ClientProducer producer = session.createProducer(FailoverTestBase.ADDRESS);

            for (int i = 0; i < numMessages; i++) {
              ClientMessage message = session.createMessage(true);

              message.getBodyBuffer().writeString("message" + i);

              message.putIntProperty("counter", i);

              message.putStringProperty(
                  Message.HDR_DUPLICATE_DETECTION_ID,
                  new SimpleString("id:" + i + ",exec:" + executionId));

              addPayload(message);

              if (log.isDebugEnabled()) {
                log.debug("Sending message " + message);
              }

              producer.send(message);
            }

            log.debug("Sending commit");
            session.commit();

            retry = false;
          } catch (ActiveMQDuplicateIdException die) {
            logAndSystemOut("#test duplicate id rejected on sending");
            break;
          } catch (ActiveMQTransactionRolledBackException trbe) {
            log.info("#test transaction rollback retrying on sending");
            // OK
            retry = true;
          } catch (ActiveMQUnBlockedException ube) {
            log.info("#test transaction rollback retrying on sending");
            // OK
            retry = true;
          } catch (ActiveMQTransactionOutcomeUnknownException toue) {
            log.info("#test transaction rollback retrying on sending");
            // OK
            retry = true;
          } catch (ActiveMQException e) {
            log.info("#test Exception " + e, e);
            throw e;
          }
        } while (retry);

        logAndSystemOut("#test Finished sending, starting consumption now");

        boolean blocked = false;

        retry = false;

        ClientConsumer consumer = null;
        do {
          ArrayList<Integer> msgs = new ArrayList<>();
          try {
            if (consumer == null) {
              consumer = session.createConsumer(FailoverTestBase.ADDRESS);
              session.start();
            }

            for (int i = 0; i < numMessages; i++) {
              if (log.isDebugEnabled()) {
                log.debug("Consumer receiving message " + i);
              }
              ClientMessage message = consumer.receive(10000);
              if (message == null) {
                break;
              }

              if (log.isDebugEnabled()) {
                log.debug("Received message " + message);
              }

              int count = message.getIntProperty("counter");

              if (count != i) {
                log.warn("count was received out of order, " + count + "!=" + i);
              }

              msgs.add(count);

              message.acknowledge();
            }

            log.info("#test commit");
            try {
              session.commit();
            } catch (ActiveMQTransactionRolledBackException trbe) {
              // we know the tx has been rolled back so we just consume again
              retry = true;
              continue;
            } catch (ActiveMQException e) {
              // This could eventually happen
              // We will get rid of this when we implement 2 phase commit on failover
              log.warn("exception during commit, it will be ignored for now" + e.getMessage(), e);
            }

            try {
              if (blocked) {
                assertTrue(
                    "msgs.size is expected to be 0 or "
                        + numMessages
                        + " but it was "
                        + msgs.size(),
                    msgs.size() == 0 || msgs.size() == numMessages);
              } else {
                assertTrue(
                    "msgs.size is expected to be " + numMessages + " but it was " + msgs.size(),
                    msgs.size() == numMessages);
              }
            } catch (Throwable e) {
              log.info(threadDump("Thread dump, messagesReceived = " + msgs.size()));
              logAndSystemOut(e.getMessage() + " messages received");
              for (Integer msg : msgs) {
                logAndSystemOut(msg.toString());
              }
              throw e;
            }

            int i = 0;
            for (Integer msg : msgs) {
              assertEquals(i++, (int) msg);
            }

            retry = false;
            blocked = false;
          } catch (ActiveMQTransactionRolledBackException trbe) {
            logAndSystemOut("Transaction rolled back with " + msgs.size(), trbe);
            // TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
            // ATM RolledBack exception is being called with the transaction is committed.
            // the test will fail if you remove this next line
            blocked = true;
            retry = true;
          } catch (ActiveMQTransactionOutcomeUnknownException tou) {
            logAndSystemOut("Transaction rolled back with " + msgs.size(), tou);
            // TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
            // ATM RolledBack exception is being called with the transaction is committed.
            // the test will fail if you remove this next line
            blocked = true;
            retry = true;
          } catch (ActiveMQUnBlockedException ube) {
            logAndSystemOut("Unblocked with " + msgs.size(), ube);
            // TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
            // This part of the test is never being called.
            blocked = true;
            retry = true;
          } catch (ActiveMQException e) {
            logAndSystemOut(e.getMessage(), e);
            throw e;
          }
        } while (retry);
      } finally {
        if (session != null) {
          session.close();
        }
      }

      listener = null;
    }
  }
  private void doTestNonTransactional(final TestRunner runner) throws Exception {
    while (!runner.isFailed()) {
      AsynchronousFailoverTest.log.info("looping");

      ClientSession session = sf.createSession(true, true, 0);

      listener = new CountDownSessionFailureListener(session);

      session.addFailureListener(listener);

      ClientProducer producer = session.createProducer(FailoverTestBase.ADDRESS);

      final int numMessages = 1000;

      for (int i = 0; i < numMessages; i++) {
        boolean retry = false;
        do {
          try {
            ClientMessage message = session.createMessage(true);

            message.getBodyBuffer().writeString("message" + i);

            message.putIntProperty("counter", i);

            addPayload(message);

            producer.send(message);

            retry = false;
          } catch (ActiveMQUnBlockedException ube) {
            AsynchronousFailoverTest.log.info("exception when sending message with counter " + i);

            ube.printStackTrace();

            retry = true;

          } catch (ActiveMQException e) {
            fail("Invalid Exception type:" + e.getType());
          }
        } while (retry);
      }

      // create the consumer with retry if failover occurs during createConsumer call
      ClientConsumer consumer = null;
      boolean retry = false;
      do {
        try {
          consumer = session.createConsumer(FailoverTestBase.ADDRESS);

          retry = false;
        } catch (ActiveMQUnBlockedException ube) {
          AsynchronousFailoverTest.log.info("exception when creating consumer");

          retry = true;

        } catch (ActiveMQException e) {
          fail("Invalid Exception type:" + e.getType());
        }
      } while (retry);

      session.start();

      List<Integer> counts = new ArrayList<>(1000);
      int lastCount = -1;
      boolean counterGap = false;
      while (true) {
        ClientMessage message = consumer.receive(500);

        if (message == null) {
          break;
        }

        // messages must remain ordered but there could be a "jump" if messages
        // are missing or duplicated
        int count = message.getIntProperty("counter");
        counts.add(count);
        if (count != lastCount + 1) {
          if (counterGap) {
            Assert.fail("got another counter gap at " + count + ": " + counts);
          } else {
            if (lastCount != -1) {
              AsynchronousFailoverTest.log.info("got first counter gap at " + count);
              counterGap = true;
            }
          }
        }

        lastCount = count;

        message.acknowledge();
      }

      session.close();

      this.listener = null;
    }
  }
  private void runTest(final TestRunner runnable) throws Throwable {
    final int numIts = 1;

    try {
      for (int i = 0; i < numIts; i++) {
        AsynchronousFailoverTest.log.info("Iteration " + i);
        ServerLocator locator =
            getServerLocator()
                .setBlockOnNonDurableSend(true)
                .setBlockOnDurableSend(true)
                .setReconnectAttempts(-1)
                .setConfirmationWindowSize(10 * 1024 * 1024);
        sf = createSessionFactoryAndWaitForTopology(locator, 2);
        try {

          ClientSession createSession = sf.createSession(true, true);

          createSession.createQueue(FailoverTestBase.ADDRESS, FailoverTestBase.ADDRESS, null, true);

          RemotingConnection conn = ((ClientSessionInternal) createSession).getConnection();

          Thread t = new Thread(runnable);

          t.setName("MainTEST");

          t.start();

          long randomDelay = (long) (2000 * Math.random());

          AsynchronousFailoverTest.log.info("Sleeping " + randomDelay);

          Thread.sleep(randomDelay);

          AsynchronousFailoverTest.log.info("Failing asynchronously");

          // Simulate failure on connection
          synchronized (lockFail) {
            if (log.isDebugEnabled()) {
              log.debug("#test crashing test");
            }
            crash(createSession);
          }

          /*if (listener != null)
          {
             boolean ok = listener.latch.await(10000, TimeUnit.MILLISECONDS);

             Assert.assertTrue(ok);
          }*/

          runnable.setFailed();

          AsynchronousFailoverTest.log.info("Fail complete");

          t.join(TimeUnit.SECONDS.toMillis(20));
          if (t.isAlive()) {
            System.out.println(threadDump("Thread still running from the test"));
            t.interrupt();
            fail("Test didn't complete successful, thread still running");
          }

          runnable.checkForExceptions();

          createSession.close();

          Assert.assertEquals(0, sf.numSessions());

          locator.close();
        } finally {
          locator.close();

          Assert.assertEquals(0, sf.numConnections());
        }

        if (i != numIts - 1) {
          tearDown();
          runnable.checkForExceptions();
          runnable.reset();
          setUp();
        }
      }
    } finally {
    }
  }
  /* This is called only when the bridge is activated */
  protected void connect() {
    if (stopping) return;

    synchronized (connectionGuard) {
      if (!keepConnecting) return;

      logger.debug(
          "Connecting  "
              + this
              + " to its destination ["
              + nodeUUID.toString()
              + "], csf="
              + this.csf);

      retryCount++;

      try {
        if (csf == null || csf.isClosed()) {
          if (stopping) return;
          csf = createSessionFactory();
          if (csf == null) {
            // Retrying. This probably means the node is not available (for the cluster connection
            // case)
            scheduleRetryConnect();
            return;
          }
          // Session is pre-acknowledge
          session =
              (ClientSessionInternal) csf.createSession(user, password, false, true, true, true, 1);
          sessionConsumer =
              (ClientSessionInternal) csf.createSession(user, password, false, true, true, true, 1);
        }

        if (forwardingAddress != null) {
          ClientSession.AddressQuery query = null;

          try {
            query = session.addressQuery(forwardingAddress);
          } catch (Throwable e) {
            ActiveMQServerLogger.LOGGER.errorQueryingBridge(e, name);
            // This was an issue during startup, we will not count this retry
            retryCount--;

            scheduleRetryConnectFixedTimeout(100);
            return;
          }

          if (forwardingAddress.startsWith(BridgeImpl.JMS_QUEUE_ADDRESS_PREFIX)
              || forwardingAddress.startsWith(BridgeImpl.JMS_TOPIC_ADDRESS_PREFIX)) {
            if (!query.isExists()) {
              ActiveMQServerLogger.LOGGER.errorQueryingBridge(forwardingAddress, retryCount);
              scheduleRetryConnect();
              return;
            }
          } else {
            if (!query.isExists()) {
              ActiveMQServerLogger.LOGGER.bridgeNoBindings(
                  getName(), getForwardingAddress(), getForwardingAddress());
            }
          }
        }

        producer = session.createProducer();
        session.addFailureListener(BridgeImpl.this);

        session.setSendAcknowledgementHandler(BridgeImpl.this);

        afterConnect();

        active = true;

        queue.addConsumer(BridgeImpl.this);
        queue.deliverAsync();

        ActiveMQServerLogger.LOGGER.bridgeConnected(this);

        // We only do this on plain core bridges
        if (isPlainCoreBridge()) {
          serverLocator.addClusterTopologyListener(new TopologyListener());
        }

        keepConnecting = false;
        return;
      } catch (ActiveMQException e) {
        // the session was created while its server was starting, retry it:
        if (e.getType() == ActiveMQExceptionType.SESSION_CREATION_REJECTED) {
          ActiveMQServerLogger.LOGGER.errorStartingBridge(name);

          // We are not going to count this one as a retry
          retryCount--;

          scheduleRetryConnectFixedTimeout(this.retryInterval);
          return;
        } else {
          if (logger.isDebugEnabled()) {
            logger.debug("Bridge " + this + " is unable to connect to destination. Retrying", e);
          }

          scheduleRetryConnect();
        }
      } catch (ActiveMQInterruptedException | InterruptedException e) {
        ActiveMQServerLogger.LOGGER.errorConnectingBridge(e, this);
      } catch (Exception e) {
        ActiveMQServerLogger.LOGGER.errorConnectingBridge(e, this);
        if (csf != null) {
          try {
            csf.close();
            csf = null;
          } catch (Throwable ignored) {
          }
        }
        fail(false);
        scheduleRetryConnect();
      }
    }
  }
 private static void cleanUpSessionFactory(ClientSessionFactoryInternal factory) {
   if (factory != null) factory.cleanup();
 }