/** Tests https://jira.jboss.org/jira/browse/JGRP-1079 */
public void testOOBMessageLoss() throws Exception {
Util.close(b); // we only need 1 channel
MyReceiver receiver = new MySleepingReceiver("C1", 1000);
a.setReceiver(receiver);
TP transport = a.getProtocolStack().getTransport();
transport.setOOBRejectionPolicy("discard");
final int NUM = 10;
for (int i = 1; i <= NUM; i++) {
Message msg = new Message(null, null, i);
msg.setFlag(Message.OOB);
a.send(msg);
}
STABLE stable = (STABLE) a.getProtocolStack().findProtocol(STABLE.class);
if (stable != null) stable.runMessageGarbageCollection();
Collection<Integer> msgs = receiver.getMsgs();
for (int i = 0; i < 20; i++) {
if (msgs.size() == NUM) break;
Util.sleep(1000);
sendStableMessages(a, b);
}
System.out.println("msgs = " + Util.print(msgs));
assert msgs.size() == NUM
: "expected " + NUM + " messages but got " + msgs.size() + ", msgs=" + Util.print(msgs);
for (int i = 1; i <= NUM; i++) {
assert msgs.contains(i);
}
}
/**
* Tests a simple split: {A,B} and {C,D} need to merge back into one subgroup. Checks how many
* MergeViews are installed
*/
public void testSplitInTheMiddle2() throws Exception {
View v1 = View.create(a.getAddress(), 10, a.getAddress(), b.getAddress());
View v2 = View.create(c.getAddress(), 10, c.getAddress(), d.getAddress());
injectView(v1, a, b);
injectView(v2, c, d);
enableInfoSender(false, a, b, c, d);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, c, d);
enableInfoSender(false, a, b, c, d);
for (JChannel ch : Arrays.asList(a, b, c, d))
System.out.println(ch.getName() + ": " + ch.getView());
System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters");
enableInfoSender(true, a, b, c, d);
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c, d);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(a, b, c, d)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(a, b, c, d)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 4;
assert ((MergeView) mv).getSubgroups().size() == 2;
}
for (JChannel ch : Arrays.asList(a, b, c, d)) {
View view = ch.getView();
assert view.size() == 4 : "view should have 4 members: " + view;
}
}
protected void waitForBridgeView(
int expected_size, long timeout, long interval, JChannel... channels) {
long deadline = System.currentTimeMillis() + timeout;
while (System.currentTimeMillis() < deadline) {
boolean views_correct = true;
for (JChannel ch : channels) {
RELAY2 relay = (RELAY2) ch.getProtocolStack().findProtocol(RELAY2.class);
View bridge_view = relay.getBridgeView(BRIDGE_CLUSTER);
if (bridge_view == null || bridge_view.size() != expected_size) {
views_correct = false;
break;
}
}
if (views_correct) break;
Util.sleep(interval);
}
System.out.println("Bridge views:\n");
for (JChannel ch : channels) {
RELAY2 relay = (RELAY2) ch.getProtocolStack().findProtocol(RELAY2.class);
View bridge_view = relay.getBridgeView(BRIDGE_CLUSTER);
System.out.println(ch.getAddress() + ": " + bridge_view);
}
for (JChannel ch : channels) {
RELAY2 relay = (RELAY2) ch.getProtocolStack().findProtocol(RELAY2.class);
View bridge_view = relay.getBridgeView(BRIDGE_CLUSTER);
assert bridge_view != null && bridge_view.size() == expected_size
: ch.getAddress() + ": bridge view=" + bridge_view + ", expected=" + expected_size;
}
}
private static void sendStableMessages(JChannel... channels) {
for (JChannel ch : channels) {
STABLE stable = (STABLE) ch.getProtocolStack().findProtocol(STABLE.class);
if (stable != null) stable.runMessageGarbageCollection();
UNICAST2 uni = (UNICAST2) ch.getProtocolStack().findProtocol(UNICAST2.class);
if (uni != null) uni.sendStableMessages();
}
}
/**
* Multiple threads (NUM_THREADS) send messages (NUM_MSGS)
*
* @throws Exception
*/
@Test(dataProvider = "provider")
public void testMessageReceptionUnderHighLoad(String props) throws Exception {
CountDownLatch latch = new CountDownLatch(1);
c1 = new JChannel(props);
c2 = new JChannel(props);
MyReceiver r1 = new MyReceiver("c1"), r2 = new MyReceiver("c2");
c1.setReceiver(r1);
c2.setReceiver(r2);
c1.connect("testSimpleMessageReception");
c2.connect("testSimpleMessageReception");
Address c1_addr = c1.getAddress(), c2_addr = c2.getAddress();
MySender[] c1_senders = new MySender[NUM_THREADS];
for (int i = 0; i < c1_senders.length; i++) {
c1_senders[i] = new MySender(c1, c2_addr, latch);
c1_senders[i].start();
}
MySender[] c2_senders = new MySender[NUM_THREADS];
for (int i = 0; i < c2_senders.length; i++) {
c2_senders[i] = new MySender(c2, c1_addr, latch);
c2_senders[i].start();
}
Util.sleep(500);
latch.countDown(); // starts all threads
long NUM_EXPECTED_MSGS = NUM_THREADS * NUM_MSGS;
for (int i = 0; i < 20; i++) {
if (r1.getNum() == NUM_EXPECTED_MSGS && r2.getNum() == NUM_EXPECTED_MSGS) break;
Util.sleep(2000);
UNICAST2 unicast2 = (UNICAST2) c1.getProtocolStack().findProtocol(UNICAST2.class);
if (unicast2 != null) unicast2.sendStableMessages();
unicast2 = (UNICAST2) c2.getProtocolStack().findProtocol(UNICAST2.class);
if (unicast2 != null) unicast2.sendStableMessages();
}
System.out.println(
"c1 received "
+ r1.getNum()
+ " msgs, "
+ getNumberOfRetransmissions(c1)
+ " retransmissions");
System.out.println(
"c2 received "
+ r2.getNum()
+ " msgs, "
+ getNumberOfRetransmissions(c2)
+ " retransmissions");
assert r1.getNum() == NUM_EXPECTED_MSGS
: "expected " + NUM_EXPECTED_MSGS + ", but got " + r1.getNum();
assert r2.getNum() == NUM_EXPECTED_MSGS
: "expected " + NUM_EXPECTED_MSGS + ", but got " + r2.getNum();
}
private static void createPartitions(JChannel... channels) throws Exception {
for (JChannel ch : channels) {
DISCARD discard = new DISCARD();
discard.setDiscardAll(true);
ch.getProtocolStack().insertProtocol(discard, ProtocolStack.Position.ABOVE, TP.class);
}
for (JChannel ch : channels) {
View view = View.create(ch.getAddress(), 10, ch.getAddress());
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
gms.installView(view);
}
}
private static void changeProps(JChannel... channels) {
for (JChannel ch : channels) {
FD fd = (FD) ch.getProtocolStack().findProtocol(FD.class);
if (fd != null) {
fd.setTimeout(1000);
fd.setMaxTries(2);
}
FD_ALL fd_all = (FD_ALL) ch.getProtocolStack().findProtocol(FD_ALL.class);
if (fd_all != null) {
fd_all.setTimeout(2000);
fd_all.setInterval(800);
}
}
}
/** Tests the scenario described by Dan in https://issues.jboss.org/browse/JGRP-1876 */
public void testJGRP_1876_Dan() throws Exception {
Util.close(d, c, b, a);
s = createChannel("S", true);
t = createChannel("T", true);
u = createChannel("U", true);
v = createChannel("V", true);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, s, t, u, v);
enableInfoSender(false, s, t, u, v); // stops INFO sending in MERGE3
for (JChannel ch : Arrays.asList(s, t, u, v))
System.out.println(ch.getName() + ": " + ch.getView());
View v1 = View.create(s.getAddress(), 10, s.getAddress());
View v2 = View.create(t.getAddress(), 10, t.getAddress());
View v3 = View.create(u.getAddress(), 11, u.getAddress(), v.getAddress());
injectView(v1, s);
injectView(v2, t);
injectView(v3, u, v);
enableInfoSender(false, s, t, u, v); // stops INFO sending in MERGE3
Util.waitUntilAllChannelsHaveSameSize(10000, 500, s);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, t);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, u, v);
System.out.printf("\nPartitions:\n");
for (JChannel ch : Arrays.asList(s, t, u, v))
System.out.println(ch.getName() + ": " + ch.getView());
enableInfoSender(true, s, t, u, v);
System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters");
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, s, t, u, v);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(s, t, u, v)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(s, t, u, v)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 4;
assert ((MergeView) mv).getSubgroups().size() == 3;
}
for (JChannel ch : Arrays.asList(s, t, u, v)) {
View view = ch.getView();
assert view.size() == 4 : "view should have 4 members: " + view;
}
}
public void testRegularAndOOBUnicasts2() throws Exception {
DISCARD discard = new DISCARD();
ProtocolStack stack = a.getProtocolStack();
stack.insertProtocol(
discard, ProtocolStack.BELOW, (Class<? extends Protocol>[]) Util.getUnicastProtocols());
Address dest = b.getAddress();
Message m1 = new Message(dest, 1);
Message m2 = new Message(dest, 2).setFlag(Message.Flag.OOB);
Message m3 = new Message(dest, 3).setFlag(Message.Flag.OOB);
Message m4 = new Message(dest, 4);
MyReceiver receiver = new MyReceiver("B");
b.setReceiver(receiver);
a.send(m1);
discard.setDropDownUnicasts(2);
a.send(m2); // dropped
a.send(m3); // dropped
a.send(m4);
Collection<Integer> list = receiver.getMsgs();
int count = 10;
while (list.size() < 4 && --count > 0) {
Util.sleep(500); // time for potential retransmission
sendStableMessages(a, b);
}
System.out.println("list = " + list);
assert list.size() == 4 : "list is " + list;
assert list.contains(1) && list.contains(2) && list.contains(3) && list.contains(4);
}
/** Creates a singleton view for each channel listed and injects it */
protected static void createPartition(JChannel... channels) {
for (JChannel ch : channels) {
View view = Util.createView(ch.getAddress(), 5, ch.getAddress());
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
gms.installView(view);
}
}
private void printConnections() {
Protocol prot = channel.getProtocolStack().findProtocol(unicast_protocols);
if (prot instanceof UNICAST)
System.out.println("connections:\n" + ((UNICAST) prot).printConnections());
else if (prot instanceof UNICAST2)
System.out.println("connections:\n" + ((UNICAST2) prot).printConnections());
}
/**
* Tests sending 1, 2 (OOB) and 3, where they are received in the order 1, 3, 2. Message 3 should
* not get delivered until message 4 is received (http://jira.jboss.com/jira/browse/JGRP-780)
*/
public void testRegularAndOOBUnicasts() throws Exception {
DISCARD discard = new DISCARD();
ProtocolStack stack = a.getProtocolStack();
stack.insertProtocol(discard, ProtocolStack.BELOW, UNICAST.class, UNICAST2.class);
Address dest = b.getAddress();
Message m1 = new Message(dest, null, 1);
Message m2 = new Message(dest, null, 2);
m2.setFlag(Message.OOB);
Message m3 = new Message(dest, null, 3);
MyReceiver receiver = new MyReceiver("C2");
b.setReceiver(receiver);
a.send(m1);
discard.setDropDownUnicasts(1);
a.send(m2);
a.send(m3);
Collection<Integer> list = receiver.getMsgs();
int count = 10;
while (list.size() < 3 && --count > 0) {
Util.sleep(500); // time for potential retransmission
sendStableMessages(a, b);
}
assert list.size() == 3 : "list is " + list;
assert list.contains(1) && list.contains(2) && list.contains(3);
}
protected void enableInfoSender(boolean enable, JChannel... channels) throws Exception {
for (JChannel ch : channels) {
MERGE3 merge = (MERGE3) ch.getProtocolStack().findProtocol(MERGE3.class);
Method meth = enable ? startInfoSender : stopInfoSender;
meth.invoke(merge);
}
}
/** Tests a merge between ViewIds of the same coord, e.g. A|6, A|7, A|8, A|9 */
public void testViewsBySameCoord() {
View v1 =
View.create(
a.getAddress(),
6,
a.getAddress(),
b.getAddress(),
c.getAddress(),
d.getAddress()); // {A,B,C,D}
View v2 =
View.create(a.getAddress(), 7, a.getAddress(), b.getAddress(), c.getAddress()); // {A,B,C}
View v3 = View.create(a.getAddress(), 8, a.getAddress(), b.getAddress()); // {A,B}
View v4 = View.create(a.getAddress(), 9, a.getAddress()); // {A}
Util.close(b, c, d); // not interested in those...
MERGE3 merge = (MERGE3) a.getProtocolStack().findProtocol(MERGE3.class);
for (View view : Arrays.asList(v1, v2, v4, v3)) {
MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(view.getViewId(), null, null);
Message msg = new Message(null, a.getAddress(), null).putHeader(merge.getId(), hdr);
merge.up(new Event(Event.MSG, msg));
}
merge.checkInconsistencies(); // no merge will happen
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a);
System.out.println("A's view: " + a.getView());
assert a.getView().size() == 1;
assert a.getView().containsMember(a.getAddress());
}
public void testRegularAndOOBMulticasts() throws Exception {
DISCARD discard = new DISCARD();
ProtocolStack stack = a.getProtocolStack();
stack.insertProtocol(discard, ProtocolStack.BELOW, NAKACK2.class);
a.setDiscardOwnMessages(true);
Address dest = null; // send to all
Message m1 = new Message(dest, null, 1);
Message m2 = new Message(dest, null, 2);
m2.setFlag(Message.OOB);
Message m3 = new Message(dest, null, 3);
MyReceiver receiver = new MyReceiver("C2");
b.setReceiver(receiver);
a.send(m1);
discard.setDropDownMulticasts(1);
a.send(m2);
a.send(m3);
Util.sleep(500);
Collection<Integer> list = receiver.getMsgs();
for (int i = 0; i < 10; i++) {
System.out.println("list = " + list);
if (list.size() == 3) break;
Util.sleep(1000); // give the asynchronous msgs some time to be received
sendStableMessages(a, b);
}
assert list.size() == 3 : "list is " + list;
assert list.contains(1) && list.contains(2) && list.contains(3);
}
/** Tests https://jira.jboss.org/jira/browse/JGRP-1079 for unicast messages */
public void testOOBUnicastMessageLoss() throws Exception {
MyReceiver receiver = new MySleepingReceiver("C2", 1000);
b.setReceiver(receiver);
a.getProtocolStack().getTransport().setOOBRejectionPolicy("discard");
final int NUM = 10;
final Address dest = b.getAddress();
for (int i = 1; i <= NUM; i++) {
Message msg = new Message(dest, null, i);
msg.setFlag(Message.OOB);
a.send(msg);
}
Collection<Integer> msgs = receiver.getMsgs();
for (int i = 0; i < 20; i++) {
if (msgs.size() == NUM) break;
Util.sleep(1000);
// sendStableMessages(c1,c2); // not needed for unicasts !
}
assert msgs.size() == NUM
: "expected " + NUM + " messages but got " + msgs.size() + ", msgs=" + Util.print(msgs);
for (int i = 1; i <= NUM; i++) {
assert msgs.contains(i);
}
}
private static void disableBundling(JChannel ch) {
ProtocolStack stack = ch.getProtocolStack();
TP transport = stack.getTransport();
if (transport != null) {
transport.setEnableBundling(false);
}
}
/** Kicks off the benchmark on all cluster nodes */
void startBenchmark() throws Throwable {
RequestOptions options = new RequestOptions(ResponseMode.GET_ALL, 0);
options.setFlags(Message.OOB, Message.DONT_BUNDLE, Message.NO_FC);
RspList<Object> responses = disp.callRemoteMethods(null, new MethodCall(START), options);
long total_reqs = 0;
long total_time = 0;
System.out.println("\n======================= Results: ===========================");
for (Map.Entry<Address, Rsp<Object>> entry : responses.entrySet()) {
Address mbr = entry.getKey();
Rsp rsp = entry.getValue();
Results result = (Results) rsp.getValue();
total_reqs += result.num_gets + result.num_puts;
total_time += result.time;
System.out.println(mbr + ": " + result);
}
double total_reqs_sec = total_reqs / (total_time / 1000.0);
double throughput = total_reqs_sec * msg_size;
double ms_per_req = total_time / (double) total_reqs;
Protocol prot = channel.getProtocolStack().findProtocol(unicast_protocols);
System.out.println("\n");
System.out.println(
Util.bold(
"Average of "
+ f.format(total_reqs_sec)
+ " requests / sec ("
+ Util.printBytes(throughput)
+ " / sec), "
+ f.format(ms_per_req)
+ " ms /request (prot="
+ prot.getName()
+ ")"));
System.out.println("\n\n");
}
/**
* Takes a membership of 10, e.g. {A,B,C,D,E,F,G,H,I,J}, creates multiple partitions (e.g.
* {A,B,C}, {D}, {E,F,G,H}, {I,J}), and merges them back into one cluster. Asserts that there's
* only 1 MergeView
*/
public void testMultipleSplits() throws Exception {
e = createChannel("E", true);
f = createChannel("F", true);
g = createChannel("G", true);
h = createChannel("H", true);
i = createChannel("I", true);
j = createChannel("J", true);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b, c, d, e, f, g, h, i, j);
enableInfoSender(false, a, b, c, d, e, f, g, h, i, j); // stops INFO sending in MERGE3
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j))
System.out.println(ch.getName() + ": " + ch.getView());
List<View> partitions = createPartitions(4, a, b, c, d, e, f, g, h, i, j);
// Install views
for (View partition : partitions) {
for (Address mbr : partition.getMembersRaw()) {
JChannel ch = findChannel(mbr);
injectView(partition, ch);
}
}
System.out.printf("\n%d partitions:\n", partitions.size());
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j))
System.out.println(ch.getName() + ": " + ch.getView());
System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters");
enableInfoSender(true, a, b, c, d, e, f, g, h, i, j);
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c, d, e, f, g, h, i, j);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 10;
assert ((MergeView) mv).getSubgroups().size() == partitions.size();
}
for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) {
View view = ch.getView();
assert view.size() == 10 : "view should have 10 members: " + view;
}
}
private static void modifyConfigs(JChannel... channels) throws Exception {
for (JChannel ch : channels) {
ProtocolStack stack = ch.getProtocolStack();
stack.removeProtocol("MERGE2");
stack.removeProtocol("VERIFY_SUSPECT");
stack.removeProtocol("FC");
}
}
private void removeConnection() {
Address member = getReceiver();
if (member != null) {
Protocol prot = channel.getProtocolStack().findProtocol(unicast_protocols);
if (prot instanceof UNICAST) ((UNICAST) prot).removeConnection(member);
else if (prot instanceof UNICAST2) ((UNICAST2) prot).removeConnection(member);
}
}
protected static void removeDUPL(JChannel... channels) {
for (JChannel ch : channels) {
DUPL dupl = (DUPL) ch.getProtocolStack().findProtocol(DUPL.class);
if (dupl != null) {
dupl.setCopyMulticastMsgs(false);
dupl.setCopyUnicastMsgs(false);
}
}
}
@BeforeMethod
void start() throws Exception {
r1 = new MyReceiver("A");
r2 = new MyReceiver("B");
a = new JChannel(props);
a.setName("A");
a.connect(CLUSTER);
a_addr = a.getAddress();
a.setReceiver(r1);
u1 = a.getProtocolStack().findProtocol(UNICAST.class);
b = new JChannel(props);
b.setName("B");
b.connect(CLUSTER);
b_addr = b.getAddress();
b.setReceiver(r2);
u2 = b.getProtocolStack().findProtocol(UNICAST.class);
System.out.println("A=" + a_addr + ", B=" + b_addr);
}
/** A: {A,B} B: {A,B} C: {C} C receives INFO from B only */
public void testMergeWithIncompleteInfos() throws Exception {
Util.close(d);
enableInfoSender(false, a, b, c);
View one = View.create(a.getAddress(), 10, a.getAddress(), b.getAddress());
View two = View.create(c.getAddress(), 10, c.getAddress());
injectView(one, a, b);
injectView(two, c);
enableInfoSender(false, a, b, c);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, c);
System.out.printf("\nPartitions:\n");
for (JChannel ch : Arrays.asList(a, b, c))
System.out.println(ch.getName() + ": " + ch.getView());
MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(one.getViewId(), null, null);
Message msg =
new Message(null, b.getAddress(), null)
.putHeader(merge_id, hdr); // B sends the INFO message to C
Event merge_event = new Event(Event.MSG, msg);
MERGE3 merge = (MERGE3) c.getProtocolStack().findProtocol(MERGE3.class);
merge.up(merge_event);
enableInfoSender(true, a, b, c);
System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters");
Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c);
System.out.println("\nResulting views:");
for (JChannel ch : Arrays.asList(a, b, c)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
System.out.println(mv);
}
for (JChannel ch : Arrays.asList(a, b, c)) {
GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class);
View mv = gms.view();
assert mv instanceof MergeView;
assert mv.size() == 3;
assert ((MergeView) mv).getSubgroups().size() == 2;
}
for (JChannel ch : Arrays.asList(a, b, c)) {
View view = ch.getView();
assert view.size() == 3 : "view should have 3 members: " + view;
}
}
public void testMergeWithAsymetricViewsCoordIsolated() {
// Isolate the coord
Address coord = a.getView().getCreator();
System.out.println("Isolating coord: " + coord);
List<Address> members = new ArrayList<>();
members.add(coord);
View coord_view = new View(coord, 4, members);
System.out.println("coord_view: " + coord_view);
Channel coord_channel = findChannel(coord);
System.out.println("coord_channel: " + coord_channel.getAddress());
MutableDigest digest = new MutableDigest(coord_view.getMembersRaw());
NAKACK2 nakack = (NAKACK2) coord_channel.getProtocolStack().findProtocol(NAKACK2.class);
digest.merge(nakack.getDigest(coord));
GMS gms = (GMS) coord_channel.getProtocolStack().findProtocol(GMS.class);
gms.installView(coord_view, digest);
System.out.println("gms.getView() " + gms.getView());
System.out.println("Views are:");
for (JChannel ch : Arrays.asList(a, b, c, d))
System.out.println(ch.getAddress() + ": " + ch.getView());
JChannel merge_leader = findChannel(coord);
MyReceiver receiver = new MyReceiver();
merge_leader.setReceiver(receiver);
System.out.println("merge_leader: " + merge_leader.getAddressAsString());
System.out.println("Injecting MERGE event into merge leader " + merge_leader.getAddress());
Map<Address, View> merge_views = new HashMap<>(4);
merge_views.put(a.getAddress(), a.getView());
merge_views.put(b.getAddress(), b.getView());
merge_views.put(c.getAddress(), c.getView());
merge_views.put(d.getAddress(), d.getView());
gms = (GMS) merge_leader.getProtocolStack().findProtocol(GMS.class);
gms.up(new Event(Event.MERGE, merge_views));
Util.waitUntilAllChannelsHaveSameSize(10000, 1000, a, b, c, d);
System.out.println("Views are:");
for (JChannel ch : Arrays.asList(a, b, c, d)) {
View view = ch.getView();
System.out.println(ch.getAddress() + ": " + view);
assert view.size() == 4;
}
MergeView merge_view = receiver.getView();
System.out.println("merge_view = " + merge_view);
assert merge_view.size() == 4;
assert merge_view.getSubgroups().size() == 2;
for (View view : merge_view.getSubgroups())
assert contains(view, a.getAddress())
|| contains(view, b.getAddress(), c.getAddress(), d.getAddress());
}
@BeforeClass
protected void setUp() throws Exception {
c1 = createChannel(true);
c1.setName("A");
GMS gms = (GMS) c1.getProtocolStack().findProtocol(GMS.class);
if (gms != null) gms.setPrintLocalAddress(false);
disableBundling(c1);
d1 = new MessageDispatcher(c1, null, null, null);
c1.connect("MessageDispatcherUnitTest");
}
public void setChannel(JChannel ch) {
this.ch = ch;
_execProt = (Executing) ch.getProtocolStack().findProtocol(Executing.class);
if (_execProt == null)
throw new IllegalStateException(
"Channel configuration must include a executing protocol "
+ "(subclass of "
+ Executing.class.getName()
+ ")");
}
protected void discard(boolean flag, JChannel... channels) throws Exception {
for (JChannel ch : channels) {
ProtocolStack stack = ch.getProtocolStack();
DISCARD discard = (DISCARD) stack.findProtocol(DISCARD.class);
if (discard == null)
stack.insertProtocol(
discard = new DISCARD(), ProtocolStack.ABOVE, stack.getTransport().getClass());
discard.setDiscardAll(flag);
}
}
public void testFlushWithCrashedFlushCoordinator() throws Exception {
JChannel a = null, b = null, c = null;
try {
a = createChannel(true, 3, "A");
changeProps(a);
a.connect("testFlushWithCrashedFlushCoordinator");
b = createChannel(a, "B");
changeProps(b);
b.connect("testFlushWithCrashedFlushCoordinator");
c = createChannel(a, "C");
changeProps(c);
c.connect("testFlushWithCrashedFlushCoordinator");
System.out.println("shutting down flush coordinator B");
b.down(new Event(Event.SUSPEND_BUT_FAIL)); // send out START_FLUSH and then return
// now shut down B. This means, after failure detection kicks in and the new coordinator takes
// over
// (either A or C), that the current flush started by B will be cancelled and a new flush (by
// A or C)
// will be started
Util.shutdown(b);
a.getProtocolStack().findProtocol(FLUSH.class).setLevel("debug");
c.getProtocolStack().findProtocol(FLUSH.class).setLevel("debug");
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, c);
// cluster should not hang and two remaining members should have a correct view
assertTrue("correct view size", a.getView().size() == 2);
assertTrue("correct view size", c.getView().size() == 2);
a.getProtocolStack().findProtocol(FLUSH.class).setLevel("warn");
c.getProtocolStack().findProtocol(FLUSH.class).setLevel("warn");
} finally {
Util.close(c, b, a);
}
}
/**
* Tests a simple split: {A,B} and {C,D} need to merge back into one subgroup. Checks how many
* MergeViews are installed
*/
public void testSplitInTheMiddle() {
View v1 = View.create(a.getAddress(), 10, a.getAddress(), b.getAddress());
View v2 = View.create(c.getAddress(), 10, c.getAddress(), d.getAddress());
injectView(v1, a, b);
injectView(v2, c, d);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b);
Util.waitUntilAllChannelsHaveSameSize(10000, 500, c, d);
for (JChannel ch : Arrays.asList(a, b, c, d))
System.out.println(ch.getName() + ": " + ch.getView());
Address merge_leader_one =
new Membership().add(a.getAddress(), b.getAddress()).sort().elementAt(0);
Address merge_leader_two =
new Membership().add(c.getAddress(), d.getAddress()).sort().elementAt(0);
for (int x = 0; x < 20; x++) {
if (a.getView().size() == 4
&& b.getView().size() == 4
&& c.getView().size() == 4
&& d.getView().size() == 4) break;
for (JChannel ch : Arrays.asList(a, b, c, d)) {
MERGE3 merge = (MERGE3) ch.getProtocolStack().findProtocol(MERGE3.class);
merge.sendInfo(); // multicasts an INFO msg to everybody else
}
for (JChannel ch :
Arrays.asList(findChannel(merge_leader_one), findChannel(merge_leader_two))) {
MERGE3 merge = (MERGE3) ch.getProtocolStack().findProtocol(MERGE3.class);
merge.checkInconsistencies();
}
Util.sleep(1000);
}
for (JChannel ch : Arrays.asList(a, b, c, d))
System.out.println(ch.getName() + ": " + ch.getView());
for (JChannel ch : Arrays.asList(a, b, c, d))
assert ch.getView().size() == 4 : "view of " + ch.getName() + ": " + ch.getView();
}
