@Test
public void testNoAutoFailback() throws Exception {
locator
.setBlockOnNonDurableSend(true)
.setBlockOnDurableSend(true)
.setFailoverOnInitialConnection(true)
.setReconnectAttempts(-1);
ClientSessionFactoryInternal sf = createSessionFactoryAndWaitForTopology(locator, 2);
ClientSession session = sendAndConsume(sf, true);
CountDownSessionFailureListener listener = new CountDownSessionFailureListener(1, session);
session.addFailureListener(listener);
backupServer.stop();
liveServer.crash();
backupServer.start();
assertTrue(listener.getLatch().await(5, TimeUnit.SECONDS));
ClientProducer producer = session.createProducer(ADDRESS);
ClientMessage message = session.createMessage(true);
setBody(0, message);
producer.send(message);
session.removeFailureListener(listener);
Thread t = new Thread(new ServerStarter(liveServer));
t.start();
waitForRemoteBackup(sf, 10, false, backupServer.getServer());
assertTrue(backupServer.isStarted());
backupServer.crash();
waitForServerToStart(liveServer.getServer());
assertTrue(liveServer.isStarted());
sf.close();
assertEquals(0, sf.numSessions());
assertEquals(0, sf.numConnections());
}
private void doTestTransactional(final TestRunner runner) throws Throwable {
// For duplication detection
int executionId = 0;
while (!runner.isFailed()) {
ClientSession session = null;
executionId++;
log.info("#test doTestTransactional starting now. Execution " + executionId);
try {
boolean retry = false;
final int numMessages = 1000;
session = sf.createSession(false, false);
listener = new CountDownSessionFailureListener(session);
session.addFailureListener(listener);
do {
try {
ClientProducer producer = session.createProducer(FailoverTestBase.ADDRESS);
for (int i = 0; i < numMessages; i++) {
ClientMessage message = session.createMessage(true);
message.getBodyBuffer().writeString("message" + i);
message.putIntProperty("counter", i);
message.putStringProperty(
Message.HDR_DUPLICATE_DETECTION_ID,
new SimpleString("id:" + i + ",exec:" + executionId));
addPayload(message);
if (log.isDebugEnabled()) {
log.debug("Sending message " + message);
}
producer.send(message);
}
log.debug("Sending commit");
session.commit();
retry = false;
} catch (ActiveMQDuplicateIdException die) {
logAndSystemOut("#test duplicate id rejected on sending");
break;
} catch (ActiveMQTransactionRolledBackException trbe) {
log.info("#test transaction rollback retrying on sending");
// OK
retry = true;
} catch (ActiveMQUnBlockedException ube) {
log.info("#test transaction rollback retrying on sending");
// OK
retry = true;
} catch (ActiveMQTransactionOutcomeUnknownException toue) {
log.info("#test transaction rollback retrying on sending");
// OK
retry = true;
} catch (ActiveMQException e) {
log.info("#test Exception " + e, e);
throw e;
}
} while (retry);
logAndSystemOut("#test Finished sending, starting consumption now");
boolean blocked = false;
retry = false;
ClientConsumer consumer = null;
do {
ArrayList<Integer> msgs = new ArrayList<>();
try {
if (consumer == null) {
consumer = session.createConsumer(FailoverTestBase.ADDRESS);
session.start();
}
for (int i = 0; i < numMessages; i++) {
if (log.isDebugEnabled()) {
log.debug("Consumer receiving message " + i);
}
ClientMessage message = consumer.receive(10000);
if (message == null) {
break;
}
if (log.isDebugEnabled()) {
log.debug("Received message " + message);
}
int count = message.getIntProperty("counter");
if (count != i) {
log.warn("count was received out of order, " + count + "!=" + i);
}
msgs.add(count);
message.acknowledge();
}
log.info("#test commit");
try {
session.commit();
} catch (ActiveMQTransactionRolledBackException trbe) {
// we know the tx has been rolled back so we just consume again
retry = true;
continue;
} catch (ActiveMQException e) {
// This could eventually happen
// We will get rid of this when we implement 2 phase commit on failover
log.warn("exception during commit, it will be ignored for now" + e.getMessage(), e);
}
try {
if (blocked) {
assertTrue(
"msgs.size is expected to be 0 or "
+ numMessages
+ " but it was "
+ msgs.size(),
msgs.size() == 0 || msgs.size() == numMessages);
} else {
assertTrue(
"msgs.size is expected to be " + numMessages + " but it was " + msgs.size(),
msgs.size() == numMessages);
}
} catch (Throwable e) {
log.info(threadDump("Thread dump, messagesReceived = " + msgs.size()));
logAndSystemOut(e.getMessage() + " messages received");
for (Integer msg : msgs) {
logAndSystemOut(msg.toString());
}
throw e;
}
int i = 0;
for (Integer msg : msgs) {
assertEquals(i++, (int) msg);
}
retry = false;
blocked = false;
} catch (ActiveMQTransactionRolledBackException trbe) {
logAndSystemOut("Transaction rolled back with " + msgs.size(), trbe);
// TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
// ATM RolledBack exception is being called with the transaction is committed.
// the test will fail if you remove this next line
blocked = true;
retry = true;
} catch (ActiveMQTransactionOutcomeUnknownException tou) {
logAndSystemOut("Transaction rolled back with " + msgs.size(), tou);
// TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
// ATM RolledBack exception is being called with the transaction is committed.
// the test will fail if you remove this next line
blocked = true;
retry = true;
} catch (ActiveMQUnBlockedException ube) {
logAndSystemOut("Unblocked with " + msgs.size(), ube);
// TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
// This part of the test is never being called.
blocked = true;
retry = true;
} catch (ActiveMQException e) {
logAndSystemOut(e.getMessage(), e);
throw e;
}
} while (retry);
} finally {
if (session != null) {
session.close();
}
}
listener = null;
}
}
private void doTestNonTransactional(final TestRunner runner) throws Exception {
while (!runner.isFailed()) {
AsynchronousFailoverTest.log.info("looping");
ClientSession session = sf.createSession(true, true, 0);
listener = new CountDownSessionFailureListener(session);
session.addFailureListener(listener);
ClientProducer producer = session.createProducer(FailoverTestBase.ADDRESS);
final int numMessages = 1000;
for (int i = 0; i < numMessages; i++) {
boolean retry = false;
do {
try {
ClientMessage message = session.createMessage(true);
message.getBodyBuffer().writeString("message" + i);
message.putIntProperty("counter", i);
addPayload(message);
producer.send(message);
retry = false;
} catch (ActiveMQUnBlockedException ube) {
AsynchronousFailoverTest.log.info("exception when sending message with counter " + i);
ube.printStackTrace();
retry = true;
} catch (ActiveMQException e) {
fail("Invalid Exception type:" + e.getType());
}
} while (retry);
}
// create the consumer with retry if failover occurs during createConsumer call
ClientConsumer consumer = null;
boolean retry = false;
do {
try {
consumer = session.createConsumer(FailoverTestBase.ADDRESS);
retry = false;
} catch (ActiveMQUnBlockedException ube) {
AsynchronousFailoverTest.log.info("exception when creating consumer");
retry = true;
} catch (ActiveMQException e) {
fail("Invalid Exception type:" + e.getType());
}
} while (retry);
session.start();
List<Integer> counts = new ArrayList<>(1000);
int lastCount = -1;
boolean counterGap = false;
while (true) {
ClientMessage message = consumer.receive(500);
if (message == null) {
break;
}
// messages must remain ordered but there could be a "jump" if messages
// are missing or duplicated
int count = message.getIntProperty("counter");
counts.add(count);
if (count != lastCount + 1) {
if (counterGap) {
Assert.fail("got another counter gap at " + count + ": " + counts);
} else {
if (lastCount != -1) {
AsynchronousFailoverTest.log.info("got first counter gap at " + count);
counterGap = true;
}
}
}
lastCount = count;
message.acknowledge();
}
session.close();
this.listener = null;
}
}
