@Test
public void testNoAutoFailback() throws Exception {
locator
.setBlockOnNonDurableSend(true)
.setBlockOnDurableSend(true)
.setFailoverOnInitialConnection(true)
.setReconnectAttempts(-1);
ClientSessionFactoryInternal sf = createSessionFactoryAndWaitForTopology(locator, 2);
ClientSession session = sendAndConsume(sf, true);
CountDownSessionFailureListener listener = new CountDownSessionFailureListener(1, session);
session.addFailureListener(listener);
backupServer.stop();
liveServer.crash();
backupServer.start();
assertTrue(listener.getLatch().await(5, TimeUnit.SECONDS));
ClientProducer producer = session.createProducer(ADDRESS);
ClientMessage message = session.createMessage(true);
setBody(0, message);
producer.send(message);
session.removeFailureListener(listener);
Thread t = new Thread(new ServerStarter(liveServer));
t.start();
waitForRemoteBackup(sf, 10, false, backupServer.getServer());
assertTrue(backupServer.isStarted());
backupServer.crash();
waitForServerToStart(liveServer.getServer());
assertTrue(liveServer.isStarted());
sf.close();
assertEquals(0, sf.numSessions());
assertEquals(0, sf.numConnections());
}
/* Hook for creating session factory */
protected ClientSessionFactoryInternal createSessionFactory() throws Exception {
if (targetNodeID != null
&& (this.reconnectAttemptsSameNode < 0 || retryCount <= this.reconnectAttemptsSameNode)) {
csf = reconnectOnOriginalNode();
} else {
serverLocator.resetToInitialConnectors();
csf = (ClientSessionFactoryInternal) serverLocator.createSessionFactory();
}
// null here means the targetNodeIS is not available yet
if (csf != null) {
csf.setReconnectAttempts(0);
}
return csf;
}
private void doTestTransactional(final TestRunner runner) throws Throwable {
// For duplication detection
int executionId = 0;
while (!runner.isFailed()) {
ClientSession session = null;
executionId++;
log.info("#test doTestTransactional starting now. Execution " + executionId);
try {
boolean retry = false;
final int numMessages = 1000;
session = sf.createSession(false, false);
listener = new CountDownSessionFailureListener(session);
session.addFailureListener(listener);
do {
try {
ClientProducer producer = session.createProducer(FailoverTestBase.ADDRESS);
for (int i = 0; i < numMessages; i++) {
ClientMessage message = session.createMessage(true);
message.getBodyBuffer().writeString("message" + i);
message.putIntProperty("counter", i);
message.putStringProperty(
Message.HDR_DUPLICATE_DETECTION_ID,
new SimpleString("id:" + i + ",exec:" + executionId));
addPayload(message);
if (log.isDebugEnabled()) {
log.debug("Sending message " + message);
}
producer.send(message);
}
log.debug("Sending commit");
session.commit();
retry = false;
} catch (ActiveMQDuplicateIdException die) {
logAndSystemOut("#test duplicate id rejected on sending");
break;
} catch (ActiveMQTransactionRolledBackException trbe) {
log.info("#test transaction rollback retrying on sending");
// OK
retry = true;
} catch (ActiveMQUnBlockedException ube) {
log.info("#test transaction rollback retrying on sending");
// OK
retry = true;
} catch (ActiveMQTransactionOutcomeUnknownException toue) {
log.info("#test transaction rollback retrying on sending");
// OK
retry = true;
} catch (ActiveMQException e) {
log.info("#test Exception " + e, e);
throw e;
}
} while (retry);
logAndSystemOut("#test Finished sending, starting consumption now");
boolean blocked = false;
retry = false;
ClientConsumer consumer = null;
do {
ArrayList<Integer> msgs = new ArrayList<>();
try {
if (consumer == null) {
consumer = session.createConsumer(FailoverTestBase.ADDRESS);
session.start();
}
for (int i = 0; i < numMessages; i++) {
if (log.isDebugEnabled()) {
log.debug("Consumer receiving message " + i);
}
ClientMessage message = consumer.receive(10000);
if (message == null) {
break;
}
if (log.isDebugEnabled()) {
log.debug("Received message " + message);
}
int count = message.getIntProperty("counter");
if (count != i) {
log.warn("count was received out of order, " + count + "!=" + i);
}
msgs.add(count);
message.acknowledge();
}
log.info("#test commit");
try {
session.commit();
} catch (ActiveMQTransactionRolledBackException trbe) {
// we know the tx has been rolled back so we just consume again
retry = true;
continue;
} catch (ActiveMQException e) {
// This could eventually happen
// We will get rid of this when we implement 2 phase commit on failover
log.warn("exception during commit, it will be ignored for now" + e.getMessage(), e);
}
try {
if (blocked) {
assertTrue(
"msgs.size is expected to be 0 or "
+ numMessages
+ " but it was "
+ msgs.size(),
msgs.size() == 0 || msgs.size() == numMessages);
} else {
assertTrue(
"msgs.size is expected to be " + numMessages + " but it was " + msgs.size(),
msgs.size() == numMessages);
}
} catch (Throwable e) {
log.info(threadDump("Thread dump, messagesReceived = " + msgs.size()));
logAndSystemOut(e.getMessage() + " messages received");
for (Integer msg : msgs) {
logAndSystemOut(msg.toString());
}
throw e;
}
int i = 0;
for (Integer msg : msgs) {
assertEquals(i++, (int) msg);
}
retry = false;
blocked = false;
} catch (ActiveMQTransactionRolledBackException trbe) {
logAndSystemOut("Transaction rolled back with " + msgs.size(), trbe);
// TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
// ATM RolledBack exception is being called with the transaction is committed.
// the test will fail if you remove this next line
blocked = true;
retry = true;
} catch (ActiveMQTransactionOutcomeUnknownException tou) {
logAndSystemOut("Transaction rolled back with " + msgs.size(), tou);
// TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
// ATM RolledBack exception is being called with the transaction is committed.
// the test will fail if you remove this next line
blocked = true;
retry = true;
} catch (ActiveMQUnBlockedException ube) {
logAndSystemOut("Unblocked with " + msgs.size(), ube);
// TODO: https://jira.jboss.org/jira/browse/HORNETQ-369
// This part of the test is never being called.
blocked = true;
retry = true;
} catch (ActiveMQException e) {
logAndSystemOut(e.getMessage(), e);
throw e;
}
} while (retry);
} finally {
if (session != null) {
session.close();
}
}
listener = null;
}
}
private void doTestNonTransactional(final TestRunner runner) throws Exception {
while (!runner.isFailed()) {
AsynchronousFailoverTest.log.info("looping");
ClientSession session = sf.createSession(true, true, 0);
listener = new CountDownSessionFailureListener(session);
session.addFailureListener(listener);
ClientProducer producer = session.createProducer(FailoverTestBase.ADDRESS);
final int numMessages = 1000;
for (int i = 0; i < numMessages; i++) {
boolean retry = false;
do {
try {
ClientMessage message = session.createMessage(true);
message.getBodyBuffer().writeString("message" + i);
message.putIntProperty("counter", i);
addPayload(message);
producer.send(message);
retry = false;
} catch (ActiveMQUnBlockedException ube) {
AsynchronousFailoverTest.log.info("exception when sending message with counter " + i);
ube.printStackTrace();
retry = true;
} catch (ActiveMQException e) {
fail("Invalid Exception type:" + e.getType());
}
} while (retry);
}
// create the consumer with retry if failover occurs during createConsumer call
ClientConsumer consumer = null;
boolean retry = false;
do {
try {
consumer = session.createConsumer(FailoverTestBase.ADDRESS);
retry = false;
} catch (ActiveMQUnBlockedException ube) {
AsynchronousFailoverTest.log.info("exception when creating consumer");
retry = true;
} catch (ActiveMQException e) {
fail("Invalid Exception type:" + e.getType());
}
} while (retry);
session.start();
List<Integer> counts = new ArrayList<>(1000);
int lastCount = -1;
boolean counterGap = false;
while (true) {
ClientMessage message = consumer.receive(500);
if (message == null) {
break;
}
// messages must remain ordered but there could be a "jump" if messages
// are missing or duplicated
int count = message.getIntProperty("counter");
counts.add(count);
if (count != lastCount + 1) {
if (counterGap) {
Assert.fail("got another counter gap at " + count + ": " + counts);
} else {
if (lastCount != -1) {
AsynchronousFailoverTest.log.info("got first counter gap at " + count);
counterGap = true;
}
}
}
lastCount = count;
message.acknowledge();
}
session.close();
this.listener = null;
}
}
private void runTest(final TestRunner runnable) throws Throwable {
final int numIts = 1;
try {
for (int i = 0; i < numIts; i++) {
AsynchronousFailoverTest.log.info("Iteration " + i);
ServerLocator locator =
getServerLocator()
.setBlockOnNonDurableSend(true)
.setBlockOnDurableSend(true)
.setReconnectAttempts(-1)
.setConfirmationWindowSize(10 * 1024 * 1024);
sf = createSessionFactoryAndWaitForTopology(locator, 2);
try {
ClientSession createSession = sf.createSession(true, true);
createSession.createQueue(FailoverTestBase.ADDRESS, FailoverTestBase.ADDRESS, null, true);
RemotingConnection conn = ((ClientSessionInternal) createSession).getConnection();
Thread t = new Thread(runnable);
t.setName("MainTEST");
t.start();
long randomDelay = (long) (2000 * Math.random());
AsynchronousFailoverTest.log.info("Sleeping " + randomDelay);
Thread.sleep(randomDelay);
AsynchronousFailoverTest.log.info("Failing asynchronously");
// Simulate failure on connection
synchronized (lockFail) {
if (log.isDebugEnabled()) {
log.debug("#test crashing test");
}
crash(createSession);
}
/*if (listener != null)
{
boolean ok = listener.latch.await(10000, TimeUnit.MILLISECONDS);
Assert.assertTrue(ok);
}*/
runnable.setFailed();
AsynchronousFailoverTest.log.info("Fail complete");
t.join(TimeUnit.SECONDS.toMillis(20));
if (t.isAlive()) {
System.out.println(threadDump("Thread still running from the test"));
t.interrupt();
fail("Test didn't complete successful, thread still running");
}
runnable.checkForExceptions();
createSession.close();
Assert.assertEquals(0, sf.numSessions());
locator.close();
} finally {
locator.close();
Assert.assertEquals(0, sf.numConnections());
}
if (i != numIts - 1) {
tearDown();
runnable.checkForExceptions();
runnable.reset();
setUp();
}
}
} finally {
}
}
/* This is called only when the bridge is activated */
protected void connect() {
if (stopping) return;
synchronized (connectionGuard) {
if (!keepConnecting) return;
logger.debug(
"Connecting "
+ this
+ " to its destination ["
+ nodeUUID.toString()
+ "], csf="
+ this.csf);
retryCount++;
try {
if (csf == null || csf.isClosed()) {
if (stopping) return;
csf = createSessionFactory();
if (csf == null) {
// Retrying. This probably means the node is not available (for the cluster connection
// case)
scheduleRetryConnect();
return;
}
// Session is pre-acknowledge
session =
(ClientSessionInternal) csf.createSession(user, password, false, true, true, true, 1);
sessionConsumer =
(ClientSessionInternal) csf.createSession(user, password, false, true, true, true, 1);
}
if (forwardingAddress != null) {
ClientSession.AddressQuery query = null;
try {
query = session.addressQuery(forwardingAddress);
} catch (Throwable e) {
ActiveMQServerLogger.LOGGER.errorQueryingBridge(e, name);
// This was an issue during startup, we will not count this retry
retryCount--;
scheduleRetryConnectFixedTimeout(100);
return;
}
if (forwardingAddress.startsWith(BridgeImpl.JMS_QUEUE_ADDRESS_PREFIX)
|| forwardingAddress.startsWith(BridgeImpl.JMS_TOPIC_ADDRESS_PREFIX)) {
if (!query.isExists()) {
ActiveMQServerLogger.LOGGER.errorQueryingBridge(forwardingAddress, retryCount);
scheduleRetryConnect();
return;
}
} else {
if (!query.isExists()) {
ActiveMQServerLogger.LOGGER.bridgeNoBindings(
getName(), getForwardingAddress(), getForwardingAddress());
}
}
}
producer = session.createProducer();
session.addFailureListener(BridgeImpl.this);
session.setSendAcknowledgementHandler(BridgeImpl.this);
afterConnect();
active = true;
queue.addConsumer(BridgeImpl.this);
queue.deliverAsync();
ActiveMQServerLogger.LOGGER.bridgeConnected(this);
// We only do this on plain core bridges
if (isPlainCoreBridge()) {
serverLocator.addClusterTopologyListener(new TopologyListener());
}
keepConnecting = false;
return;
} catch (ActiveMQException e) {
// the session was created while its server was starting, retry it:
if (e.getType() == ActiveMQExceptionType.SESSION_CREATION_REJECTED) {
ActiveMQServerLogger.LOGGER.errorStartingBridge(name);
// We are not going to count this one as a retry
retryCount--;
scheduleRetryConnectFixedTimeout(this.retryInterval);
return;
} else {
if (logger.isDebugEnabled()) {
logger.debug("Bridge " + this + " is unable to connect to destination. Retrying", e);
}
scheduleRetryConnect();
}
} catch (ActiveMQInterruptedException | InterruptedException e) {
ActiveMQServerLogger.LOGGER.errorConnectingBridge(e, this);
} catch (Exception e) {
ActiveMQServerLogger.LOGGER.errorConnectingBridge(e, this);
if (csf != null) {
try {
csf.close();
csf = null;
} catch (Throwable ignored) {
}
}
fail(false);
scheduleRetryConnect();
}
}
}
private static void cleanUpSessionFactory(ClientSessionFactoryInternal factory) {
if (factory != null) factory.cleanup();
}