/**
* Tests a simple split: {A,B} and {C,D} need to merge back into one subgroup. Checks how many
* MergeViews are installed
*/
public void testSplitInTheMiddle2() throws Exception {
View v1 = View.create(a.getAddress(), 10, a.getAddress(), b.getAddress());
View v2 = View.create(c.getAddress(), 10, c.getAddress(), d.getAddress());
injectView(v1, a, b);
injectView(v2, c, d);
enableInfoSender(false, a, b, c, d);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, c, d);
enableInfoSender(false, a, b, c, d);
for (JChannel ch : Arrays.asList(a, b, c, d))
System.out.println(ch.getName() + ": " + ch.getView());
System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters");
enableInfoSender(true, a, b, c, d);
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c, d);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(a, b, c, d)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(a, b, c, d)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 4;
assert ((MergeView) mv).getSubgroups().size() == 2;
}
for (JChannel ch : Arrays.asList(a, b, c, d)) {
View view = ch.getView();
assert view.size() == 4 : "view should have 4 members: " + view;
}
}
/** Tests the scenario described by Dan in https://issues.jboss.org/browse/JGRP-1876 */
public void testJGRP_1876_Dan() throws Exception {
Util.close(d, c, b, a);
s = createChannel("S", true);
t = createChannel("T", true);
u = createChannel("U", true);
v = createChannel("V", true);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, s, t, u, v);
enableInfoSender(false, s, t, u, v); // stops INFO sending in MERGE3
for (JChannel ch : Arrays.asList(s, t, u, v))
System.out.println(ch.getName() + ": " + ch.getView());
View v1 = View.create(s.getAddress(), 10, s.getAddress());
View v2 = View.create(t.getAddress(), 10, t.getAddress());
View v3 = View.create(u.getAddress(), 11, u.getAddress(), v.getAddress());
injectView(v1, s);
injectView(v2, t);
injectView(v3, u, v);
enableInfoSender(false, s, t, u, v); // stops INFO sending in MERGE3
Util.waitUntilAllChannelsHaveSameSize(10000, 500, s);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, t);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, u, v);
System.out.printf("\nPartitions:\n");
for (JChannel ch : Arrays.asList(s, t, u, v))
System.out.println(ch.getName() + ": " + ch.getView());
enableInfoSender(true, s, t, u, v);
System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters");
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, s, t, u, v);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(s, t, u, v)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(s, t, u, v)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 4;
assert ((MergeView) mv).getSubgroups().size() == 3;
}
for (JChannel ch : Arrays.asList(s, t, u, v)) {
View view = ch.getView();
assert view.size() == 4 : "view should have 4 members: " + view;
}
}
/** Tests a merge between ViewIds of the same coord, e.g. A|6, A|7, A|8, A|9 */
public void testViewsBySameCoord() {
View v1 =
View.create(
a.getAddress(),
6,
a.getAddress(),
b.getAddress(),
c.getAddress(),
d.getAddress()); // {A,B,C,D}
View v2 =
View.create(a.getAddress(), 7, a.getAddress(), b.getAddress(), c.getAddress()); // {A,B,C}
View v3 = View.create(a.getAddress(), 8, a.getAddress(), b.getAddress()); // {A,B}
View v4 = View.create(a.getAddress(), 9, a.getAddress()); // {A}
Util.close(b, c, d); // not interested in those...
MERGE3 merge = (MERGE3) a.getProtocolStack().findProtocol(MERGE3.class);
for (View view : Arrays.asList(v1, v2, v4, v3)) {
MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(view.getViewId(), null, null);
Message msg = new Message(null, a.getAddress(), null).putHeader(merge.getId(), hdr);
merge.up(new Event(Event.MSG, msg));
}
merge.checkInconsistencies(); // no merge will happen
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a);
System.out.println("A's view: " + a.getView());
assert a.getView().size() == 1;
assert a.getView().containsMember(a.getAddress());
}
/**
* Tests A|6, A|7, A|8 and B|7, B|8, B|9 -> we should have a subviews in MergeView consisting of
* only 2 elements: A|5 and B|5
*/
public void testMultipleViewsBySameMembers() throws Exception {
View a1 =
View.create(
a.getAddress(),
6,
a.getAddress(),
b.getAddress(),
c.getAddress(),
d.getAddress()); // {A,B,C,D}
View a2 =
View.create(a.getAddress(), 7, a.getAddress(), b.getAddress(), c.getAddress()); // {A,B,C}
View a3 = View.create(a.getAddress(), 8, a.getAddress(), b.getAddress()); // {A,B}
View a4 = View.create(a.getAddress(), 9, a.getAddress()); // {A}
View b1 = View.create(b.getAddress(), 7, b.getAddress(), c.getAddress(), d.getAddress());
View b2 = View.create(b.getAddress(), 8, b.getAddress(), c.getAddress());
View b3 = View.create(b.getAddress(), 9, b.getAddress());
Util.close(c, d); // not interested in those...
// A and B cannot communicate:
discard(true, a, b);
// inject view A|6={A} into A and B|5={B} into B
injectView(a4, a);
injectView(b3, b);
assert a.getView().equals(a4);
assert b.getView().equals(b3);
List<Event> merge_events = new ArrayList<>();
for (View view : Arrays.asList(a3, a4, a2, a1)) {
MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(view.getViewId(), null, null);
Message msg = new Message(null, a.getAddress(), null).putHeader(merge_id, hdr);
merge_events.add(new Event(Event.MSG, msg));
}
for (View view : Arrays.asList(b2, b3, b1)) {
MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(view.getViewId(), null, null);
Message msg = new Message(null, b.getAddress(), null).putHeader(merge_id, hdr);
merge_events.add(new Event(Event.MSG, msg));
}
// A and B can communicate again
discard(false, a, b);
injectMergeEvents(merge_events, a, b);
checkInconsistencies(a, b); // merge will happen between A and B
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b);
System.out.println("A's view: " + a.getView() + "\nB's view: " + b.getView());
assert a.getView().size() == 2;
assert a.getView().containsMember(a.getAddress());
assert a.getView().containsMember(b.getAddress());
assert a.getView().equals(b.getView());
for (View merge_view : Arrays.asList(getViewFromGMS(a), getViewFromGMS(b))) {
System.out.println(merge_view);
assert merge_view instanceof MergeView;
List<View> subviews = ((MergeView) merge_view).getSubgroups();
assert subviews.size() == 2;
}
}
public void testFlushWithCrashedParticipants() throws Exception {
JChannel c1 = null;
JChannel c2 = null;
JChannel c3 = null;
try {
c1 = createChannel(true, 3, "C1");
changeProps(c1);
c1.connect("testFlushWithCrashedFlushCoordinator");
c2 = createChannel(c1, "C2");
changeProps(c2);
c2.connect("testFlushWithCrashedFlushCoordinator");
c3 = createChannel(c1, "C3");
changeProps(c3);
c3.connect("testFlushWithCrashedFlushCoordinator");
// and then kill members other than flush coordinator
Util.shutdown(c3);
Util.shutdown(c1);
// start flush
Util.startFlush(c2);
c2.stopFlush();
Util.waitUntilAllChannelsHaveSameSize(10000, 500, c2);
// cluster should not hang and one remaining member should have a correct view
assertTrue("correct view size", c2.getView().size() == 1);
} finally {
Util.close(c3, c2, c1);
}
}
/**
* Takes a membership of 10, e.g. {A,B,C,D,E,F,G,H,I,J}, creates multiple partitions (e.g.
* {A,B,C}, {D}, {E,F,G,H}, {I,J}), and merges them back into one cluster. Asserts that there's
* only 1 MergeView
*/
public void testMultipleSplits() throws Exception {
e = createChannel("E", true);
f = createChannel("F", true);
g = createChannel("G", true);
h = createChannel("H", true);
i = createChannel("I", true);
j = createChannel("J", true);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b, c, d, e, f, g, h, i, j);
enableInfoSender(false, a, b, c, d, e, f, g, h, i, j); // stops INFO sending in MERGE3
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j))
System.out.println(ch.getName() + ": " + ch.getView());
List<View> partitions = createPartitions(4, a, b, c, d, e, f, g, h, i, j);
// Install views
for (View partition : partitions) {
for (Address mbr : partition.getMembersRaw()) {
JChannel ch = findChannel(mbr);
injectView(partition, ch);
}
}
System.out.printf("\n%d partitions:\n", partitions.size());
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j))
System.out.println(ch.getName() + ": " + ch.getView());
System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters");
enableInfoSender(true, a, b, c, d, e, f, g, h, i, j);
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c, d, e, f, g, h, i, j);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 10;
assert ((MergeView) mv).getSubgroups().size() == partitions.size();
}
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) {
View view = ch.getView();
assert view.size() == 10 : "view should have 10 members: " + view;
}
}
@BeforeMethod
void setUp() throws Exception {
a = createChannel("A", true);
b = createChannel("B", true);
c = createChannel("C", true);
d = createChannel("D", true);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b, c, d);
enableInfoSender(false, a, b, c, d); // stops INFO sending in MERGE3
}
/** A: {A,B} B: {A,B} C: {C} C receives INFO from B only */
public void testMergeWithIncompleteInfos() throws Exception {
Util.close(d);
enableInfoSender(false, a, b, c);
View one = View.create(a.getAddress(), 10, a.getAddress(), b.getAddress());
View two = View.create(c.getAddress(), 10, c.getAddress());
injectView(one, a, b);
injectView(two, c);
enableInfoSender(false, a, b, c);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, c);
System.out.printf("\nPartitions:\n");
for (JChannel ch : Arrays.asList(a, b, c))
System.out.println(ch.getName() + ": " + ch.getView());
MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(one.getViewId(), null, null);
Message msg =
new Message(null, b.getAddress(), null)
.putHeader(merge_id, hdr); // B sends the INFO message to C
Event merge_event = new Event(Event.MSG, msg);
MERGE3 merge = (MERGE3) c.getProtocolStack().findProtocol(MERGE3.class);
merge.up(merge_event);
enableInfoSender(true, a, b, c);
System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters");
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(a, b, c)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(a, b, c)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 3;
assert ((MergeView) mv).getSubgroups().size() == 2;
}
for (JChannel ch : Arrays.asList(a, b, c)) {
View view = ch.getView();
assert view.size() == 3 : "view should have 3 members: " + view;
}
}
public void testMergeWithAsymetricViewsCoordIsolated() {
// Isolate the coord
Address coord = a.getView().getCreator();
System.out.println("Isolating coord: " + coord);
List<Address> members = new ArrayList<>();
members.add(coord);
View coord_view = new View(coord, 4, members);
System.out.println("coord_view: " + coord_view);
Channel coord_channel = findChannel(coord);
System.out.println("coord_channel: " + coord_channel.getAddress());
MutableDigest digest = new MutableDigest(coord_view.getMembersRaw());
NAKACK2 nakack = (NAKACK2) coord_channel.getProtocolStack().findProtocol(NAKACK2.class);
digest.merge(nakack.getDigest(coord));
GMS gms = (GMS) coord_channel.getProtocolStack().findProtocol(GMS.class);
gms.installView(coord_view, digest);
System.out.println("gms.getView() " + gms.getView());
System.out.println("Views are:");
for (JChannel ch : Arrays.asList(a, b, c, d))
System.out.println(ch.getAddress() + ": " + ch.getView());
JChannel merge_leader = findChannel(coord);
MyReceiver receiver = new MyReceiver();
merge_leader.setReceiver(receiver);
System.out.println("merge_leader: " + merge_leader.getAddressAsString());
System.out.println("Injecting MERGE event into merge leader " + merge_leader.getAddress());
Map<Address, View> merge_views = new HashMap<>(4);
merge_views.put(a.getAddress(), a.getView());
merge_views.put(b.getAddress(), b.getView());
merge_views.put(c.getAddress(), c.getView());
merge_views.put(d.getAddress(), d.getView());
gms = (GMS) merge_leader.getProtocolStack().findProtocol(GMS.class);
gms.up(new Event(Event.MERGE, merge_views));
Util.waitUntilAllChannelsHaveSameSize(10000, 1000, a, b, c, d);
System.out.println("Views are:");
for (JChannel ch : Arrays.asList(a, b, c, d)) {
View view = ch.getView();
System.out.println(ch.getAddress() + ": " + view);
assert view.size() == 4;
}
MergeView merge_view = receiver.getView();
System.out.println("merge_view = " + merge_view);
assert merge_view.size() == 4;
assert merge_view.getSubgroups().size() == 2;
for (View view : merge_view.getSubgroups())
assert contains(view, a.getAddress())
|| contains(view, b.getAddress(), c.getAddress(), d.getAddress());
}
@BeforeMethod
void init() throws Exception {
a = createChannel(true, 2, "A");
b = createChannel(a, "B");
setOOBPoolSize(a, b);
setStableGossip(a, b);
a.connect("OOBTest");
b.connect("OOBTest");
Util.waitUntilAllChannelsHaveSameSize(10000, 1000, a, b);
}
public void testAndLoop() throws Exception {
int numChannels = 10;
Channel channels[] = new Channel[numChannels];
for (int j = 0; j < numChannels; j++) {
channels[j] = createChannel(String.valueOf((char) ('A' + j)));
channels[j].connect(cName);
if (j == 0) Util.sleep(1000);
}
Util.waitUntilAllChannelsHaveSameSize(10000, 500, channels);
for (int i = 1; i <= 2; i++) {
int killPositions[] = {0, 3, 5, 8};
for (int index : killPositions) Util.shutdown(channels[index]);
int ch = 'M';
for (int index : killPositions) {
channels[index] = createChannel(String.valueOf((char) ch++));
channels[index].connect(cName);
}
System.out.println("***** Round " + i + " done *****");
}
Util.waitUntilAllChannelsHaveSameSize(10000, 500, channels);
}
/**
* Tests a simple split: {A,B} and {C,D} need to merge back into one subgroup. Checks how many
* MergeViews are installed
*/
public void testSplitInTheMiddle() {
View v1 = View.create(a.getAddress(), 10, a.getAddress(), b.getAddress());
View v2 = View.create(c.getAddress(), 10, c.getAddress(), d.getAddress());
injectView(v1, a, b);
injectView(v2, c, d);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, c, d);
for (JChannel ch : Arrays.asList(a, b, c, d))
System.out.println(ch.getName() + ": " + ch.getView());
Address merge_leader_one =
new Membership().add(a.getAddress(), b.getAddress()).sort().elementAt(0);
Address merge_leader_two =
new Membership().add(c.getAddress(), d.getAddress()).sort().elementAt(0);
for (int x = 0; x < 20; x++) {
if (a.getView().size() == 4
&& b.getView().size() == 4
&& c.getView().size() == 4
&& d.getView().size() == 4) break;
for (JChannel ch : Arrays.asList(a, b, c, d)) {
MERGE3 merge = (MERGE3) ch.getProtocolStack().findProtocol(MERGE3.class);
merge.sendInfo(); // multicasts an INFO msg to everybody else
}
for (JChannel ch :
Arrays.asList(findChannel(merge_leader_one), findChannel(merge_leader_two))) {
MERGE3 merge = (MERGE3) ch.getProtocolStack().findProtocol(MERGE3.class);
merge.checkInconsistencies();
}
Util.sleep(1000);
}
for (JChannel ch : Arrays.asList(a, b, c, d))
System.out.println(ch.getName() + ": " + ch.getView());
for (JChannel ch : Arrays.asList(a, b, c, d))
assert ch.getView().size() == 4 : "view of " + ch.getName() + ": " + ch.getView();
}
private void _testChannels(String names[], int connectType) throws Exception {
int count = names.length;
List<FlushTestReceiver> channels = new ArrayList<FlushTestReceiver>(count);
try {
// Create a semaphore and take all its permits
Semaphore semaphore = new Semaphore(count);
semaphore.acquire(count);
// Create channels and their threads that will block on the
// semaphore
boolean first = true;
for (String channelName : names) {
FlushTestReceiver channel = null;
if (first) channel = new FlushTestReceiver(channelName, semaphore, 0, connectType);
else {
channel =
new FlushTestReceiver(
(JChannel) channels.get(0).getChannel(), channelName, semaphore, 0, connectType);
}
channels.add(channel);
// Release one ticket at a time to allow the thread to start working
channel.start();
semaphore.release(1);
if (first) Util.sleep(3000); // minimize changes of a merge happening
first = false;
}
Channel[] tmp = new Channel[channels.size()];
int cnt = 0;
for (FlushTestReceiver receiver : channels) tmp[cnt++] = receiver.getChannel();
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, tmp);
// Reacquire the semaphore tickets; when we have them all
// we know the threads are done
semaphore.tryAcquire(count, 40, TimeUnit.SECONDS);
Util.sleep(1000); // let all events propagate...
for (FlushTestReceiver app : channels) app.getChannel().setReceiver(null);
for (FlushTestReceiver app : channels) app.cleanup();
// verify block/unblock/view/get|set state sequences for all members
for (FlushTestReceiver receiver : channels) {
checkEventStateTransferSequence(receiver);
System.out.println("event sequence is OK");
}
} finally {
for (FlushTestReceiver app : channels) app.cleanup();
}
}
@BeforeMethod
void init() throws Exception {
a = createChannel(true, 2);
a.setName("A");
b = createChannel(a);
b.setName("B");
setOOBPoolSize(a, b);
setStableGossip(a, b);
a.connect("OOBTest");
b.connect("OOBTest");
View view = b.getView();
System.out.println("view = " + view);
Util.waitUntilAllChannelsHaveSameSize(20000, 1000, a, b);
}
@BeforeMethod
public void init() throws Exception {
c1 = createChannel(true, 2);
c1.setName("C1");
c2 = createChannel(c1);
c2.setName("C2");
setOOBPoolSize(c1, c2);
setStableGossip(c1, c2);
c1.connect("OOBTest");
c2.connect("OOBTest");
View view = c2.getView();
log.info("view = " + view);
Util.waitUntilAllChannelsHaveSameSize(20000, 1000, c1, c2);
}
@BeforeMethod
void setUp() throws Exception {
System.out.print("Connecting channels: ");
a = createChannel("A");
disp_a = new MessageDispatcher(a, null, null);
a.connect(RpcLockingTest.class.getSimpleName());
lock_a = new LockService(a).getLock("lock");
b = createChannel("B");
disp_b = new MessageDispatcher(b, null, null);
b.connect(RpcLockingTest.class.getSimpleName());
lock_b = new LockService(b).getLock("lock");
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b);
System.out.println("");
disp_a.setRequestHandler(
new RequestHandler() {
@Override
public Object handle(Message arg0) throws Exception {
System.out.println("A received a message, will now try to lock the lock");
if (lock_a.tryLock()) {
Assert.fail("Should not be able to lock the lock here");
System.out.println("A aquired the lock, this shouldn't be possible");
} else System.out.println("The lock was already locked, as it should be");
return "Hello";
}
});
disp_b.setRequestHandler(
new RequestHandler() {
@Override
public Object handle(Message arg0) throws Exception {
System.out.println("B received a message, will now try to lock the lock");
if (lock_b.tryLock()) {
Assert.fail("Should not be able to lock the lock here");
System.out.println("B aquired the lock, this shouldn't be possible");
} else System.out.println("The lock already was locked, as it should be");
return "Hello";
}
});
// Print who is the coordinator
if (b.getView().getMembers().get(0).equals(b.getAddress()))
System.out.println("B is the coordinator");
else System.out.println("A is the coordinator");
System.out.println("");
}
private void createChannels(
boolean copy_multicasts,
boolean copy_unicasts,
int num_outgoing_copies,
int num_incoming_copies)
throws Exception {
c1 = createChannel(true, 3);
DUPL dupl = new DUPL(copy_multicasts, copy_unicasts, num_incoming_copies, num_outgoing_copies);
ProtocolStack stack = c1.getProtocolStack();
stack.insertProtocol(dupl, ProtocolStack.BELOW, NAKACK.class);
c2 = createChannel(c1);
c3 = createChannel(c1);
c1.connect("DuplicateTest");
c2.connect("DuplicateTest");
c3.connect("DuplicateTest");
Util.waitUntilAllChannelsHaveSameSize(20000, 1000, c1, c2, c3);
}
public void testSingleChannel() throws Exception {
Semaphore s = new Semaphore(1);
FlushTestReceiver receivers[] = {
new FlushTestReceiver("c1", s, 0, FlushTestReceiver.CONNECT_ONLY)
};
receivers[0].start();
s.release(1);
// Make sure everyone is in sync
Channel[] tmp = new Channel[receivers.length];
for (int i = 0; i < receivers.length; i++) tmp[i] = receivers[i].getChannel();
Util.waitUntilAllChannelsHaveSameSize(10000, 1000, tmp);
// Reacquire the semaphore tickets; when we have them all we know the threads are done
s.tryAcquire(1, 10, TimeUnit.SECONDS);
receivers[0].cleanup();
Util.sleep(1000);
checkEventStateTransferSequence(receivers[0]);
}
public void testFlushWithCrashedFlushCoordinator() throws Exception {
JChannel a = null, b = null, c = null;
try {
a = createChannel(true, 3, "A");
changeProps(a);
a.connect("testFlushWithCrashedFlushCoordinator");
b = createChannel(a, "B");
changeProps(b);
b.connect("testFlushWithCrashedFlushCoordinator");
c = createChannel(a, "C");
changeProps(c);
c.connect("testFlushWithCrashedFlushCoordinator");
System.out.println("shutting down flush coordinator B");
b.down(new Event(Event.SUSPEND_BUT_FAIL)); // send out START_FLUSH and then return
// now shut down B. This means, after failure detection kicks in and the new coordinator takes
// over
// (either A or C), that the current flush started by B will be cancelled and a new flush (by
// A or C)
// will be started
Util.shutdown(b);
a.getProtocolStack().findProtocol(FLUSH.class).setLevel("debug");
c.getProtocolStack().findProtocol(FLUSH.class).setLevel("debug");
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, c);
// cluster should not hang and two remaining members should have a correct view
assertTrue("correct view size", a.getView().size() == 2);
assertTrue("correct view size", c.getView().size() == 2);
a.getProtocolStack().findProtocol(FLUSH.class).setLevel("warn");
c.getProtocolStack().findProtocol(FLUSH.class).setLevel("warn");
} finally {
Util.close(c, b, a);
}
}
/**
* When we flush the server queue and one or more of the delivered messages triggers a response
* (in the same thread), we need to make sure the channel is connected, or else the JOIN will fail
* as the exception happens on the same thread. Note that the suggested fix on JGRP-1522 will
* solve this. Issue: https://issues.jboss.org/browse/JGRP-1522
*/
@BMScript(dir = "scripts/BecomeServerTest", value = "testSendingOfMsgsOnUnconnectedChannel")
public void testSendingOfMsgsOnUnconnectedChannel() throws Exception {
a = createChannel("A");
a.setReceiver(
new ReceiverAdapter() {
public void receive(Message msg) {
System.out.println("A: received message from " + msg.getSrc() + ": " + msg.getObject());
}
});
a.connect("BecomeServerTest");
new Thread("MsgSender-A") {
public void run() {
sendMessage(a, "hello from A"); // will be blocked by byteman rendezvous
}
}.start();
b = createChannel("B");
b.setReceiver(
new ReceiverAdapter() {
public void receive(Message msg) {
System.out.println("B: received message from " + msg.getSrc() + ": " + msg.getObject());
if (msg.getSrc().equals(a.getAddress())) {
try {
b.send(
null,
"This message would trigger an exception if the channel was not yet connected");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
});
b.connect("BecomeServerTest");
Util.waitUntilAllChannelsHaveSameSize(20000, 1000, a, b);
System.out.println("\nA: " + a.getView() + "\nB: " + b.getView());
}
public void testFlushWithCrashedParticipant() throws Exception {
JChannel c1 = null;
JChannel c2 = null;
JChannel c3 = null;
try {
c1 = createChannel(true, 3, "C1");
changeProps(c1);
c1.connect("testFlushWithCrashedParticipant");
c2 = createChannel(c1, "C2");
changeProps(c2);
c2.connect("testFlushWithCrashedParticipant");
c3 = createChannel(c1, "C3");
changeProps(c3);
c3.connect("testFlushWithCrashedParticipant");
System.out.println("shutting down C3");
Util.shutdown(c3); // kill a flush participant
System.out.println("C2: starting flush");
boolean rc = Util.startFlush(c2);
System.out.println("flush " + (rc ? " was successful" : "failed"));
assert rc;
System.out.println("stopping flush");
c2.stopFlush();
System.out.println("waiting for view to contain C1 and C2");
Util.waitUntilAllChannelsHaveSameSize(10000, 500, c1, c2);
// cluster should not hang and two remaining members should have a correct view
System.out.println("C1: view=" + c1.getView() + "\nC2: view=" + c2.getView());
assertTrue("correct view size", c1.getView().size() == 2);
assertTrue("correct view size", c2.getView().size() == 2);
} finally {
Util.close(c3, c2, c1);
}
}
public void testSequentialFlushInvocation() throws Exception {
Channel a = null, b = null, c = null;
try {
a = createChannel(true, 3, "A");
a.connect("testSequentialFlushInvocation");
b = createChannel((JChannel) a, "B");
b.connect("testSequentialFlushInvocation");
c = createChannel((JChannel) a, "C");
c.connect("testSequentialFlushInvocation");
Util.waitUntilAllChannelsHaveSameSize(10000, 1000, a, b, c);
for (int i = 0; i < 100; i++) {
System.out.print("flush #" + i + ": ");
a.startFlush(false);
a.stopFlush();
System.out.println("OK");
}
} finally {
Util.close(a, b, c);
}
}
/**
* Tests the scenario described by Dan in https://issues.jboss.org/browse/JGRP-1876; INFO messages
* are injected
*/
public void testJGRP_1876_Dan2() throws Exception {
Util.close(d, c, b, a);
s = createChannel("S", false);
t = createChannel("T", false);
u = createChannel("U", false);
v = createChannel("V", false);
for (JChannel ch : Arrays.asList(s, t, u, v))
ch.getProtocolStack().removeProtocol(MERGE3.class);
s.connect("MergeTest4");
t.connect("MergeTest4");
u.connect("MergeTest4");
v.connect("MergeTest4");
Util.waitUntilAllChannelsHaveSameSize(10000, 500, s, t, u, v);
for (JChannel ch : Arrays.asList(s, t, u, v))
System.out.println(ch.getName() + ": " + ch.getView());
View v1 = View.create(s.getAddress(), 10, s.getAddress());
View v2 = View.create(t.getAddress(), 10, t.getAddress());
View v3 = View.create(u.getAddress(), 11, u.getAddress(), v.getAddress());
injectView(v1, s);
injectView(v2, t);
injectView(v3, u, v);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, s);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, t);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, u, v);
System.out.printf("\nPartitions:\n");
for (JChannel ch : Arrays.asList(s, t, u, v))
System.out.println(ch.getName() + ": " + ch.getView());
System.out.println("\nInjecting MERGE events into GMS to merge subclusters");
Map<Address, View> different_views = new HashMap<>();
different_views.put(s.getAddress(), v1);
different_views.put(t.getAddress(), v2);
different_views.put(v.getAddress(), v3);
Collection<Address> coords = Util.determineActualMergeCoords(different_views);
Address merge_leader = new Membership().add(coords).sort().elementAt(0);
System.out.printf("--> coords=%s, merge_leader=%s\n", coords, merge_leader);
GMS gms = (GMS) findChannel(merge_leader).getProtocolStack().findProtocol(GMS.class);
gms.up(new Event(Event.MERGE, different_views));
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, s, t, u, v);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(s, t, u, v)) {
gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(s, t, u, v)) {
gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 4;
assert ((MergeView) mv).getSubgroups().size() == 3;
}
for (JChannel ch : Arrays.asList(s, t, u, v)) {
View view = ch.getView();
assert view.size() == 4 : "view should have 4 members: " + view;
}
}
/** Tests that notifications are routed to all sites. */
public void testNotificationAndRpcRelay2Transit() throws Exception {
a.connect(LON_CLUSTER);
b.connect(LON_CLUSTER);
rpca.start();
rpcb.start();
Util.waitUntilAllChannelsHaveSameSize(30000, 500, a, b);
x.connect(SFO_CLUSTER);
y.connect(SFO_CLUSTER);
rpcx.start();
rpcy.start();
Util.waitUntilAllChannelsHaveSameSize(30000, 500, x, y);
assert x.getView().size() == 2;
RELAY2 ar = (RELAY2) a.getProtocolStack().findProtocol(RELAY2.class);
RELAY2 xr = (RELAY2) x.getProtocolStack().findProtocol(RELAY2.class);
assert ar != null && xr != null;
JChannel a_bridge = null, x_bridge = null;
for (int i = 0; i < 20; i++) {
a_bridge = ar.getBridge(SFO);
x_bridge = xr.getBridge(LON);
if (a_bridge != null
&& x_bridge != null
&& a_bridge.getView().size() == 2
&& x_bridge.getView().size() == 2) break;
Util.sleep(500);
}
assert a_bridge != null && x_bridge != null;
System.out.println("A's bridge channel: " + a_bridge.getView());
System.out.println("X's bridge channel: " + x_bridge.getView());
assert a_bridge.getView().size() == 2 : "bridge view is " + a_bridge.getView();
assert x_bridge.getView().size() == 2 : "bridge view is " + x_bridge.getView();
Relayer.Route route = getRoute(x, LON);
System.out.println("Route at sfo to lon: " + route);
assert route != null;
System.out.println("B: sending message 0 to the site master of SFO");
Address sm_sfo = new SiteMaster(SFO);
b.send(sm_sfo, 0);
checkMsgDelivery(xl);
System.out.println("B: sending message to all");
b.send(null, 0);
checkMsgDelivery(xl, yl, al, bl);
MethodCall call = new MethodCall(ServerObject.class.getMethod("foo"));
System.out.println("B: call foo method on A");
Object rsp =
rpcb.callRemoteMethod(a.getAddress(), call, new RequestOptions(ResponseMode.GET_ALL, 5000));
System.out.println("RSP is: " + rsp);
System.out.println("B: call foo method on SFO master site");
rsp = rpcb.callRemoteMethod(sm_sfo, call, new RequestOptions(ResponseMode.GET_ALL, 5000));
System.out.println("RSP is: " + rsp);
System.out.println("B: call foo method on all members dest = null");
RspList<Integer> rsps =
rpcb.callRemoteMethods(null, call, new RequestOptions(ResponseMode.GET_ALL, 5000));
System.out.println("RSPs are: \n" + rsps);
View bridge_view = xr.getBridgeView(BRIDGE_CLUSTER);
System.out.println("bridge_view = " + bridge_view);
route = getRoute(x, LON);
System.out.println("Route at sfo to lon: " + route);
assert route != null;
}
/** Tests the scenario described by Karim in https://issues.jboss.org/browse/JGRP-1876 */
public void testJGRP_1876() throws Exception {
e = createChannel("E", true);
f = createChannel("F", true);
g = createChannel("G", true);
h = createChannel("H", true);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b, c, d, e, f, g, h);
enableInfoSender(false, a, b, c, d, e, f, g, h); // stops INFO sending in MERGE3
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h))
System.out.println(ch.getName() + ": " + ch.getView());
View v1 =
View.create(
a.getAddress(), 10, a.getAddress(), b.getAddress(), c.getAddress(), d.getAddress());
View v2 =
View.create(
a.getAddress(),
9,
a.getAddress(),
b.getAddress(),
c.getAddress(),
d.getAddress(),
e.getAddress(),
f.getAddress());
View v3 = View.create(g.getAddress(), 8, g.getAddress(), h.getAddress());
injectView(v1, a, b, c, d);
injectView(v2, e, f);
injectView(v3, g, h);
enableInfoSender(false, a, b, c, d, e, f, g, h); // stops INFO sending in MERGE3
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b, c, d);
for (int x = 0;
x < 10;
x++) { // can't use waitUntilAllChannelsHaveSameSize() as the channel list's length != view
// !
if (e.getView().size() == v2.size() && f.getView().size() == v2.size()) break;
Util.sleep(500);
}
assert e.getView().size() == v2.size() : "E's view: " + e.getView();
assert f.getView().size() == v2.size() : "F's view: " + f.getView();
Util.waitUntilAllChannelsHaveSameSize(10000, 500, g, h);
System.out.printf("\nPartitions:\n");
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h))
System.out.println(ch.getName() + ": " + ch.getView());
enableInfoSender(true, a, b, c, d, e, f, g, h);
System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters");
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c, d, e, f, g, h);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 8;
assert ((MergeView) mv).getSubgroups().size() == 3;
}
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h)) {
View view = ch.getView();
assert view.size() == 8 : "view should have 8 members: " + view;
}
}