@Override
public void send(String routingKey, CommandMessage<?> commandMessage) throws Exception {
String destination =
consistentHash.getMember(routingKey, commandMessage.getPayloadType().getName());
if (destination == null) {
throw new CommandDispatchException(
"No node known to accept " + commandMessage.getPayloadType().getName());
}
Address dest = getAddress(destination);
channel.send(dest, new DispatchMessage(commandMessage, serializer, false));
}
private void waitForConnectorSync(int timeoutInSeconds) throws InterruptedException {
int t = 0;
while (ConsistentHash.emptyRing().equals(connector1.getConsistentHash())
|| !connector1.getConsistentHash().equals(connector2.getConsistentHash())) {
// don't have a member for String yet, which means we must wait a little longer
if (t++ > timeoutInSeconds * 10) {
fail(
"Connectors did not manage to synchronize consistent hash ring within "
+ timeoutInSeconds
+ " seconds...");
}
Thread.sleep(100);
}
}
private void waitForConnectorSync() throws InterruptedException {
int t = 0;
while (ConsistentHash.emptyRing().equals(connector1.getConsistentHash())
|| !connector1.getConsistentHash().equals(connector2.getConsistentHash())) {
// don't have a member for String yet, which means we must wait a little longer
if (t++ > 1500) {
assertEquals(
"Connectors did not synchronize within 30 seconds.",
connector1.getConsistentHash().toString(),
connector2.getConsistentHash().toString());
}
Thread.sleep(20);
}
}
@Override
public <R> void send(
String routingKey, CommandMessage<?> commandMessage, CommandCallback<R> callback)
throws Exception {
String destination =
consistentHash.getMember(routingKey, commandMessage.getPayloadType().getName());
if (destination == null) {
throw new CommandDispatchException(
"No node known to accept " + commandMessage.getPayloadType().getName());
}
Address dest = getAddress(destination);
callbacks.put(
commandMessage.getIdentifier(), new MemberAwareCommandCallback<R>(dest, callback));
channel.send(dest, new DispatchMessage(commandMessage, serializer, true));
}
/**
* A CommandBusConnector that uses JGroups to discover and connect to other JGroupsConnectors in the
* network. Depending on the configuration of the {@link JChannel channel} that was provided, this
* implementation allows for a dynamic discovery and addition of new members. When members
* disconnect, their portion of the processing is divided over the remaining members.
*
* <p>This connector uses a consistent hashing algorithm to route commands. This ensures that
* commands with the same routing key will be sent to the same member, regardless of the sending
* member of that message.
*
* <p>Members join the CommandBus using a load factor (see {@link #connect(int)}). This load factor
* defines the number of sections on the consistent hash ring a node will receive. The more nodes on
* the ring, the bigger the relative load a member receives. Using a higher number of hashes will
* also result in a more evenly distribution of load over the different members.
*
* @author Allard Buijze
* @since 2.0
*/
public class JGroupsConnector implements CommandBusConnector {
private static final Logger logger = LoggerFactory.getLogger(JGroupsConnector.class);
private final JChannel channel;
private volatile ConsistentHash consistentHash = ConsistentHash.emptyRing();
private final String clusterName;
private final CommandBus localSegment;
private final Serializer serializer;
private final JoinCondition joinedCondition = new JoinCondition();
private final ConcurrentMap<String, MemberAwareCommandCallback> callbacks =
new ConcurrentHashMap<String, MemberAwareCommandCallback>();
private final Set<String> supportedCommandTypes = new CopyOnWriteArraySet<String>();
private volatile int currentLoadFactor;
private final JGroupsConnector.MessageReceiver messageReceiver;
/**
* Initializes the Connector using given resources. The <code>channel</code> is used to connect
* this connector to the other members. The <code>clusterName</code> is the name of the cluster
* the channel will be connected to. For local dispatching of commands, the given <code>
* localSegment</code> is used. When messages are remotely dispatched, the given <code>serializer
* </code> is used to serialize and deserialize the messages.
*
* <p>Note that Connectors on different members need to have the same <code>channel</code>
* configuration, <code>clusterName</code> and <code>serializer</code> configuration in order to
* successfully set up a distributed cluster.
*
* @param channel The channel (configured, but not connected) used to discover and connect with
* the other members
* @param clusterName The name of the cluster to connect to
* @param localSegment The command bus on which messages with this member as destination are
* dispatched on
* @param serializer The serialized used to serialize messages before sending them to other
* members.
*/
public JGroupsConnector(
JChannel channel, String clusterName, CommandBus localSegment, Serializer serializer) {
this.channel = channel;
this.clusterName = clusterName;
this.localSegment = localSegment;
this.serializer = serializer;
this.messageReceiver = new MessageReceiver();
}
/**
* Connects this member to the cluster using the given <code>loadFactor</code>. The <code>
* loadFactor</code> defines the (approximate) relative load that this member will receive.
*
* <p>A good default value is 100, which will give this member 100 nodes on the distributed hash
* ring. Giving all members (proportionally) lower values will result in a less evenly distributed
* hash.
*
* @param loadFactor The load factor for this node.
* @throws ConnectionFailedException when an error occurs while connecting
*/
public synchronized void connect(int loadFactor) throws ConnectionFailedException {
this.currentLoadFactor = loadFactor;
Assert.isTrue(loadFactor >= 0, "Load Factor must be a positive integer value.");
Assert.isTrue(
channel.getReceiver() == null || channel.getReceiver() == messageReceiver,
"The given channel already has a receiver configured. "
+ "Has the channel been reused with other Connectors?");
try {
channel.setReceiver(messageReceiver);
if (channel.isConnected() && !clusterName.equals(channel.getClusterName())) {
throw new AxonConfigurationException(
"The Channel that has been configured with this JGroupsConnector "
+ "is already connected to another cluster.");
} else if (channel.isConnected()) {
// we need to fetch state now that we have attached our MessageReceiver
channel.getState(null, 10000);
} else {
// we need to connect. This will automatically fetch state as well.
channel.connect(clusterName, null, 10000);
}
updateMembership();
} catch (Exception e) {
joinedCondition.markJoined(false);
channel.disconnect();
throw new ConnectionFailedException("Failed to connect to JGroupsConnectorFactoryBean", e);
}
}
private void updateMembership() throws MembershipUpdateFailedException {
try {
if (channel.isConnected()) {
channel.send(
new Message(
null,
new JoinMessage(currentLoadFactor, new HashSet<String>(supportedCommandTypes)))
.setFlag(Message.Flag.RSVP));
}
} catch (Exception e) {
throw new MembershipUpdateFailedException(
"Failed to dispatch Join message to Distributed Command Bus Members", e);
}
}
/**
* this method blocks until this member has successfully joined the other members, until the
* thread is interrupted, or when joining has failed.
*
* @return <code>true</code> if the member successfully joined, otherwise <code>false</code>.
* @throws InterruptedException when the thread is interrupted while joining
*/
public boolean awaitJoined() throws InterruptedException {
joinedCondition.await();
return joinedCondition.isJoined();
}
/**
* this method blocks until this member has successfully joined the other members, until the
* thread is interrupted, when the given number of milliseconds have passed, or when joining has
* failed.
*
* @param timeout The amount of time to wait for the connection to complete
* @param timeUnit The time unit of the timeout
* @return <code>true</code> if the member successfully joined, otherwise <code>false</code>.
* @throws InterruptedException when the thread is interrupted while joining
*/
public boolean awaitJoined(long timeout, TimeUnit timeUnit) throws InterruptedException {
joinedCondition.await(timeout, timeUnit);
return joinedCondition.isJoined();
}
@Override
public <R> void send(
String routingKey, CommandMessage<?> commandMessage, CommandCallback<R> callback)
throws Exception {
String destination =
consistentHash.getMember(routingKey, commandMessage.getPayloadType().getName());
if (destination == null) {
throw new CommandDispatchException(
"No node known to accept " + commandMessage.getPayloadType().getName());
}
Address dest = getAddress(destination);
callbacks.put(
commandMessage.getIdentifier(), new MemberAwareCommandCallback<R>(dest, callback));
channel.send(dest, new DispatchMessage(commandMessage, serializer, true));
}
@Override
public void send(String routingKey, CommandMessage<?> commandMessage) throws Exception {
String destination =
consistentHash.getMember(routingKey, commandMessage.getPayloadType().getName());
if (destination == null) {
throw new CommandDispatchException(
"No node known to accept " + commandMessage.getPayloadType().getName());
}
Address dest = getAddress(destination);
channel.send(dest, new DispatchMessage(commandMessage, serializer, false));
}
@Override
public synchronized <C> void subscribe(Class<C> commandType, CommandHandler<? super C> handler) {
localSegment.subscribe(commandType, handler);
if (supportedCommandTypes.add(commandType.getName())) {
updateMembership();
}
}
@Override
public synchronized <C> boolean unsubscribe(
Class<C> commandType, CommandHandler<? super C> handler) {
if (localSegment.unsubscribe(commandType, handler)) {
if (supportedCommandTypes.remove(commandType.getName())) {
updateMembership();
}
return true;
}
return false;
}
private Address getAddress(String nodeName) {
for (Address member : channel.getView()) {
if (channel.getName(member).equals(nodeName)) {
return member;
}
}
throw new IllegalArgumentException(
"Given node doesn't seem to be a member of the DistributedCommandBus");
}
/**
* Returns the consistent hash on which current assignment of commands to nodes is being executed.
*
* @return the consistent hash on which current assignment of commands to nodes is being executed
*/
public ConsistentHash getConsistentHash() {
return consistentHash;
}
private class MessageReceiver extends ReceiverAdapter {
@Override
public void getState(OutputStream ostream) throws Exception {
Util.objectToStream(consistentHash, new DataOutputStream(ostream));
}
@Override
public void setState(InputStream istream) throws Exception {
consistentHash = (ConsistentHash) Util.objectFromStream(new DataInputStream(istream));
}
@Override
public void viewAccepted(View view) {
ConsistentHash newHash = consistentHash.withExclusively(getMemberNames(view));
if (!consistentHash.equals(newHash)) {
int messagesLost = 0;
// check whether the members with outstanding callbacks are all alive
for (Map.Entry<String, MemberAwareCommandCallback> entry : callbacks.entrySet()) {
if (!entry.getValue().isMemberLive(view)) {
MemberAwareCommandCallback callback = callbacks.remove(entry.getKey());
if (callback != null) {
messagesLost++;
callback.onFailure(
new RemoteCommandHandlingException(
"The connection with the destination was lost before the result was reported."));
}
}
}
consistentHash = newHash;
logger.info("Membership has changed. Rebuilt consistent hash ring.");
logger.debug("New distributed hash: {}", consistentHash.toString());
if (messagesLost > 0 && logger.isWarnEnabled()) {
logger.warn(
"A member was disconnected while waiting for a reply. {} messages are lost without reply.",
messagesLost);
}
}
}
@Override
public void suspect(Address mbr) {
if (logger.isWarnEnabled()) {
logger.warn("Suspect member: {}.", channel.getName(mbr));
}
}
@Override
public void receive(Message msg) {
Object message = msg.getObject();
if (message instanceof JoinMessage) {
processJoinMessage(msg, (JoinMessage) message);
} else if (message instanceof DispatchMessage) {
processDispatchMessage(msg, (DispatchMessage) message);
} else if (message instanceof ReplyMessage) {
processReplyMessage((ReplyMessage) message);
}
}
private void processDispatchMessage(final Message msg, final DispatchMessage message) {
final CommandMessage commandMessage = message.getCommandMessage(serializer);
if (message.isExpectReply()) {
localSegment.dispatch(commandMessage, new ReplyingCallback(msg, commandMessage));
} else {
localSegment.dispatch(commandMessage);
}
}
private void processJoinMessage(Message msg, JoinMessage joinMessage) {
String channelName = channel.getName(msg.getSrc());
consistentHash =
consistentHash.withAdditionalNode(
channelName, joinMessage.getLoadFactor(), joinMessage.getCommandTypes());
if (logger.isInfoEnabled()) {
logger.info("{} joined with load factor: {}", msg.getSrc(), joinMessage.getLoadFactor());
}
if (msg.getSrc().equals(channel.getAddress())) {
joinedCondition.markJoined(true);
logger.info("Local segment successfully joined the distributed command bus");
}
}
@SuppressWarnings("unchecked")
private void processReplyMessage(ReplyMessage replyMessage) {
MemberAwareCommandCallback callback = callbacks.remove(replyMessage.getCommandIdentifier());
if (callback != null) {
if (replyMessage.isSuccess()) {
callback.onSuccess(replyMessage.getReturnValue(serializer));
} else {
callback.onFailure(replyMessage.getError(serializer));
}
}
}
private class ReplyingCallback implements CommandCallback<Object> {
private final Message msg;
private final CommandMessage commandMessage;
public ReplyingCallback(Message msg, CommandMessage commandMessage) {
this.msg = msg;
this.commandMessage = commandMessage;
}
@Override
public void onSuccess(Object result) {
try {
channel.send(
msg.getSrc(),
new ReplyMessage(commandMessage.getIdentifier(), result, null, serializer));
} catch (Exception e) {
logger.error(
"Unable to send reply to command [type: {}, id: {}]. ",
new Object[] {
commandMessage.getPayloadType().getSimpleName(), commandMessage.getIdentifier(), e
});
}
}
@Override
public void onFailure(Throwable cause) {
try {
channel.send(
msg.getSrc(),
new ReplyMessage(commandMessage.getIdentifier(), null, cause, serializer));
} catch (Exception e) {
logger.error("Unable to send reply:", e);
}
}
}
}
private List<String> getMemberNames(View view) {
List<String> memberNames = new ArrayList<String>(view.size());
for (Address member : view.getMembers()) {
memberNames.add(channel.getName(member));
}
return memberNames;
}
private static final class JoinCondition {
private final CountDownLatch joinCountDown = new CountDownLatch(1);
private volatile boolean success;
public void await() throws InterruptedException {
joinCountDown.await();
}
public void await(long timeout, TimeUnit timeUnit) throws InterruptedException {
joinCountDown.await(timeout, timeUnit);
}
private void markJoined(boolean joinSucceeded) {
this.success = joinSucceeded;
joinCountDown.countDown();
}
public boolean isJoined() {
return success;
}
}
private static class MemberAwareCommandCallback<R> implements CommandCallback<R> {
private final Address dest;
private final CommandCallback<R> callback;
public MemberAwareCommandCallback(Address dest, CommandCallback<R> callback) {
this.dest = dest;
this.callback = callback;
}
public boolean isMemberLive(View currentView) {
return currentView.containsMember(dest);
}
@Override
public void onSuccess(R result) {
callback.onSuccess(result);
}
@Override
public void onFailure(Throwable cause) {
callback.onFailure(cause);
}
}
}