@Test
  public void testLinkedListOrder() throws Exception {
    ServerLocator locator = createInVMNonHALocator();

    ClientSessionFactory sf = createSessionFactory(locator);

    ClientSession session = sf.createSession();

    session.start();

    ClientProducer producer = session.createProducer("foo");

    ClientConsumer redConsumer = session.createConsumer("foo", "color='red'");

    ClientConsumer anyConsumer = session.createConsumer("foo");

    sendMessage(session, producer, "any", "msg1");

    sendMessage(session, producer, "any", "msg2");

    sendMessage(session, producer, "any", "msg3");

    sendMessage(session, producer, "red", "msgRed4");

    sendMessage(session, producer, "red", "msgRed5");

    readConsumer("anyConsumer", anyConsumer);

    readConsumer("anyConsumer", anyConsumer);

    log.info("### closing consumer ###");

    anyConsumer.close();

    readConsumer("redConsumer", redConsumer);

    readConsumer("redConsumer", redConsumer);

    log.info("### recreating consumer ###");

    anyConsumer = session.createConsumer("foo");

    session.start();

    readConsumer("anyConsumer", anyConsumer);

    session.close();

    sf.close();

    locator.close();
  }
  @Test
  public void testNonMatchingMessagesFollowedByMatchingMessages() throws Exception {

    ClientMessage message = session.createMessage(false);

    message.putStringProperty("animal", "hippo");

    producer.send(message);

    assertNull(consumer.receiveImmediate());

    message = session.createMessage(false);

    message.putStringProperty("animal", "giraffe");

    log.info("sending second msg");

    producer.send(message);

    ClientMessage received = consumer.receiveImmediate();

    assertNotNull(received);

    assertEquals("giraffe", received.getStringProperty("animal"));

    assertNull(consumer.receiveImmediate());

    session.close();
  }
  private void doTestTransactional(final TestRunner runner) throws Throwable {
    // For duplication detection
    int executionId = 0;

    while (!runner.isFailed()) {
      ClientSession session = null;

      executionId++;

      log.info("#test doTestTransactional starting now. Execution " + executionId);

      try {

        boolean retry = false;

        final int numMessages = 1000;

        session = sf.createSession(false, false);

        listener = new CountDownSessionFailureListener(session);
        session.addFailureListener(listener);

        do {
          try {
            ClientProducer producer = session.createProducer(FailoverTestBase.ADDRESS);

            for (int i = 0; i < numMessages; i++) {
              ClientMessage message = session.createMessage(true);

              message.getBodyBuffer().writeString("message" + i);

              message.putIntProperty("counter", i);

              message.putStringProperty(
                  Message.HDR_DUPLICATE_DETECTION_ID,
                  new SimpleString("id:" + i + ",exec:" + executionId));

              addPayload(message);

              if (log.isDebugEnabled()) {
                log.debug("Sending message " + message);
              }

              producer.send(message);
            }

            log.debug("Sending commit");
            session.commit();

            retry = false;
          } catch (ActiveMQDuplicateIdException die) {
            logAndSystemOut("#test duplicate id rejected on sending");
            break;
          } catch (ActiveMQTransactionRolledBackException trbe) {
            log.info("#test transaction rollback retrying on sending");
            // OK
            retry = true;
          } catch (ActiveMQUnBlockedException ube) {
            log.info("#test transaction rollback retrying on sending");
            // OK
            retry = true;
          } catch (ActiveMQTransactionOutcomeUnknownException toue) {
            log.info("#test transaction rollback retrying on sending");
            // OK
            retry = true;
          } catch (ActiveMQException e) {
            log.info("#test Exception " + e, e);
            throw e;
          }
        } while (retry);

        logAndSystemOut("#test Finished sending, starting consumption now");

        boolean blocked = false;

        retry = false;

        ClientConsumer consumer = null;
        do {
          ArrayList<Integer> msgs = new ArrayList<>();
          try {
            if (consumer == null) {
              consumer = session.createConsumer(FailoverTestBase.ADDRESS);
              session.start();
            }

            for (int i = 0; i < numMessages; i++) {
              if (log.isDebugEnabled()) {
                log.debug("Consumer receiving message " + i);
              }
              ClientMessage message = consumer.receive(10000);
              if (message == null) {
                break;
              }

              if (log.isDebugEnabled()) {
                log.debug("Received message " + message);
              }

              int count = message.getIntProperty("counter");

              if (count != i) {
                log.warn("count was received out of order, " + count + "!=" + i);
              }

              msgs.add(count);

              message.acknowledge();
            }

            log.info("#test commit");
            try {
              session.commit();
            } catch (ActiveMQTransactionRolledBackException trbe) {
              // we know the tx has been rolled back so we just consume again
              retry = true;
              continue;
            } catch (ActiveMQException e) {
              // This could eventually happen
              // We will get rid of this when we implement 2 phase commit on failover
              log.warn("exception during commit, it will be ignored for now" + e.getMessage(), e);
            }

            try {
              if (blocked) {
                assertTrue(
                    "msgs.size is expected to be 0 or "
                        + numMessages
                        + " but it was "
                        + msgs.size(),
                    msgs.size() == 0 || msgs.size() == numMessages);
              } else {
                assertTrue(
                    "msgs.size is expected to be " + numMessages + " but it was " + msgs.size(),
                    msgs.size() == numMessages);
              }
            } catch (Throwable e) {
              log.info(threadDump("Thread dump, messagesReceived = " + msgs.size()));
              logAndSystemOut(e.getMessage() + " messages received");
              for (Integer msg : msgs) {
                logAndSystemOut(msg.toString());
              }
              throw e;
            }

            int i = 0;
            for (Integer msg : msgs) {
              assertEquals(i++, (int) msg);
            }

            retry = false;
            blocked = false;
          } catch (ActiveMQTransactionRolledBackException trbe) {
            logAndSystemOut("Transaction rolled back with " + msgs.size(), trbe);
            // TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
            // ATM RolledBack exception is being called with the transaction is committed.
            // the test will fail if you remove this next line
            blocked = true;
            retry = true;
          } catch (ActiveMQTransactionOutcomeUnknownException tou) {
            logAndSystemOut("Transaction rolled back with " + msgs.size(), tou);
            // TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
            // ATM RolledBack exception is being called with the transaction is committed.
            // the test will fail if you remove this next line
            blocked = true;
            retry = true;
          } catch (ActiveMQUnBlockedException ube) {
            logAndSystemOut("Unblocked with " + msgs.size(), ube);
            // TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
            // This part of the test is never being called.
            blocked = true;
            retry = true;
          } catch (ActiveMQException e) {
            logAndSystemOut(e.getMessage(), e);
            throw e;
          }
        } while (retry);
      } finally {
        if (session != null) {
          session.close();
        }
      }

      listener = null;
    }
  }
  private void runTest(final TestRunner runnable) throws Throwable {
    final int numIts = 1;

    try {
      for (int i = 0; i < numIts; i++) {
        AsynchronousFailoverTest.log.info("Iteration " + i);
        ServerLocator locator =
            getServerLocator()
                .setBlockOnNonDurableSend(true)
                .setBlockOnDurableSend(true)
                .setReconnectAttempts(-1)
                .setConfirmationWindowSize(10 * 1024 * 1024);
        sf = createSessionFactoryAndWaitForTopology(locator, 2);
        try {

          ClientSession createSession = sf.createSession(true, true);

          createSession.createQueue(FailoverTestBase.ADDRESS, FailoverTestBase.ADDRESS, null, true);

          RemotingConnection conn = ((ClientSessionInternal) createSession).getConnection();

          Thread t = new Thread(runnable);

          t.setName("MainTEST");

          t.start();

          long randomDelay = (long) (2000 * Math.random());

          AsynchronousFailoverTest.log.info("Sleeping " + randomDelay);

          Thread.sleep(randomDelay);

          AsynchronousFailoverTest.log.info("Failing asynchronously");

          // Simulate failure on connection
          synchronized (lockFail) {
            if (log.isDebugEnabled()) {
              log.debug("#test crashing test");
            }
            crash(createSession);
          }

          /*if (listener != null)
          {
             boolean ok = listener.latch.await(10000, TimeUnit.MILLISECONDS);

             Assert.assertTrue(ok);
          }*/

          runnable.setFailed();

          AsynchronousFailoverTest.log.info("Fail complete");

          t.join(TimeUnit.SECONDS.toMillis(20));
          if (t.isAlive()) {
            System.out.println(threadDump("Thread still running from the test"));
            t.interrupt();
            fail("Test didn't complete successful, thread still running");
          }

          runnable.checkForExceptions();

          createSession.close();

          Assert.assertEquals(0, sf.numSessions());

          locator.close();
        } finally {
          locator.close();

          Assert.assertEquals(0, sf.numConnections());
        }

        if (i != numIts - 1) {
          tearDown();
          runnable.checkForExceptions();
          runnable.reset();
          setUp();
        }
      }
    } finally {
    }
  }
  private void doTestMultipleGroupingTXRollback() throws Exception {
    log.info("*** starting test");
    ServerLocator locator = createInVMNonHALocator();
    locator.setBlockOnAcknowledge(true);
    ClientSessionFactory sessionFactory = createSessionFactory(locator);
    ClientSession clientSession = sessionFactory.createSession(false, false, false);
    ClientProducer clientProducer = this.clientSession.createProducer(qName);
    ClientConsumer consumer = clientSession.createConsumer(qName);
    ClientConsumer consumer2 = clientSession.createConsumer(qName);
    clientSession.start();

    // need to wait a bit or consumers might be busy
    Thread.sleep(200);

    SimpleString groupId = new SimpleString("grp1");
    SimpleString groupId2 = new SimpleString("grp2");
    int numMessages = 100;
    for (int i = 0; i < numMessages; i++) {
      ClientMessage message = createTextMessage(clientSession, "m" + i);
      if (i % 2 == 0 || i == 0) {
        message.putStringProperty(Message.HDR_GROUP_ID, groupId);
      } else {
        message.putStringProperty(Message.HDR_GROUP_ID, groupId2);
      }
      clientProducer.send(message);
    }

    CountDownLatch latch = new CountDownLatch(numMessages);
    DummyMessageHandler dummyMessageHandler = new DummyMessageHandler(latch, true);
    consumer.setMessageHandler(dummyMessageHandler);
    DummyMessageHandler dummyMessageHandler2 = new DummyMessageHandler(latch, true);
    consumer2.setMessageHandler(dummyMessageHandler2);
    Assert.assertTrue(latch.await(10, TimeUnit.SECONDS));
    Assert.assertEquals(50, dummyMessageHandler.list.size(), dummyMessageHandler.list.size());
    int i = 0;
    for (ClientMessage message : dummyMessageHandler.list) {
      Assert.assertEquals(message.getBodyBuffer().readString(), "m" + i);
      i += 2;
    }
    Assert.assertEquals(dummyMessageHandler2.list.size(), 50);
    i = 1;
    for (ClientMessage message : dummyMessageHandler2.list) {
      Assert.assertEquals(message.getBodyBuffer().readString(), "m" + i);
      i += 2;
    }
    latch = new CountDownLatch(numMessages);
    dummyMessageHandler.reset(latch);
    dummyMessageHandler2.reset(latch);
    clientSession.rollback();
    Assert.assertTrue(latch.await(10, TimeUnit.SECONDS));
    Assert.assertEquals(dummyMessageHandler.list.size(), 50);
    i = 0;
    for (ClientMessage message : dummyMessageHandler.list) {
      Assert.assertEquals(message.getBodyBuffer().readString(), "m" + i);
      i += 2;
    }
    Assert.assertEquals(dummyMessageHandler2.list.size(), 50);
    i = 1;
    for (ClientMessage message : dummyMessageHandler2.list) {
      Assert.assertEquals(message.getBodyBuffer().readString(), "m" + i);
      i += 2;
    }
    consumer = this.clientSession.createConsumer(qName);
    Assert.assertNull(consumer.receiveImmediate());
    clientSession.close();
    locator.close();
  }