/** * Tests a simple split: {A,B} and {C,D} need to merge back into one subgroup. Checks how many * MergeViews are installed */ public void testSplitInTheMiddle2() throws Exception { View v1 = View.create(a.getAddress(), 10, a.getAddress(), b.getAddress()); View v2 = View.create(c.getAddress(), 10, c.getAddress(), d.getAddress()); injectView(v1, a, b); injectView(v2, c, d); enableInfoSender(false, a, b, c, d); Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b); Util.waitUntilAllChannelsHaveSameSize(10000, 500, c, d); enableInfoSender(false, a, b, c, d); for (JChannel ch : Arrays.asList(a, b, c, d)) System.out.println(ch.getName() + ": " + ch.getView()); System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters"); enableInfoSender(true, a, b, c, d); Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c, d); System.out.println("\nResulting views:"); for (JChannel ch : Arrays.asList(a, b, c, d)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); System.out.println(mv); } for (JChannel ch : Arrays.asList(a, b, c, d)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); assert mv instanceof MergeView; assert mv.size() == 4; assert ((MergeView) mv).getSubgroups().size() == 2; } for (JChannel ch : Arrays.asList(a, b, c, d)) { View view = ch.getView(); assert view.size() == 4 : "view should have 4 members: " + view; } }
/** Tests the scenario described by Dan in https://issues.jboss.org/browse/JGRP-1876 */ public void testJGRP_1876_Dan() throws Exception { Util.close(d, c, b, a); s = createChannel("S", true); t = createChannel("T", true); u = createChannel("U", true); v = createChannel("V", true); Util.waitUntilAllChannelsHaveSameSize(10000, 500, s, t, u, v); enableInfoSender(false, s, t, u, v); // stops INFO sending in MERGE3 for (JChannel ch : Arrays.asList(s, t, u, v)) System.out.println(ch.getName() + ": " + ch.getView()); View v1 = View.create(s.getAddress(), 10, s.getAddress()); View v2 = View.create(t.getAddress(), 10, t.getAddress()); View v3 = View.create(u.getAddress(), 11, u.getAddress(), v.getAddress()); injectView(v1, s); injectView(v2, t); injectView(v3, u, v); enableInfoSender(false, s, t, u, v); // stops INFO sending in MERGE3 Util.waitUntilAllChannelsHaveSameSize(10000, 500, s); Util.waitUntilAllChannelsHaveSameSize(10000, 500, t); Util.waitUntilAllChannelsHaveSameSize(10000, 500, u, v); System.out.printf("\nPartitions:\n"); for (JChannel ch : Arrays.asList(s, t, u, v)) System.out.println(ch.getName() + ": " + ch.getView()); enableInfoSender(true, s, t, u, v); System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters"); Util.waitUntilAllChannelsHaveSameSize(30000, 1000, s, t, u, v); System.out.println("\nResulting views:"); for (JChannel ch : Arrays.asList(s, t, u, v)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); System.out.println(mv); } for (JChannel ch : Arrays.asList(s, t, u, v)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); assert mv instanceof MergeView; assert mv.size() == 4; assert ((MergeView) mv).getSubgroups().size() == 3; } for (JChannel ch : Arrays.asList(s, t, u, v)) { View view = ch.getView(); assert view.size() == 4 : "view should have 4 members: " + view; } }
/** Tests a merge between ViewIds of the same coord, e.g. A|6, A|7, A|8, A|9 */ public void testViewsBySameCoord() { View v1 = View.create( a.getAddress(), 6, a.getAddress(), b.getAddress(), c.getAddress(), d.getAddress()); // {A,B,C,D} View v2 = View.create(a.getAddress(), 7, a.getAddress(), b.getAddress(), c.getAddress()); // {A,B,C} View v3 = View.create(a.getAddress(), 8, a.getAddress(), b.getAddress()); // {A,B} View v4 = View.create(a.getAddress(), 9, a.getAddress()); // {A} Util.close(b, c, d); // not interested in those... MERGE3 merge = (MERGE3) a.getProtocolStack().findProtocol(MERGE3.class); for (View view : Arrays.asList(v1, v2, v4, v3)) { MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(view.getViewId(), null, null); Message msg = new Message(null, a.getAddress(), null).putHeader(merge.getId(), hdr); merge.up(new Event(Event.MSG, msg)); } merge.checkInconsistencies(); // no merge will happen Util.waitUntilAllChannelsHaveSameSize(10000, 500, a); System.out.println("A's view: " + a.getView()); assert a.getView().size() == 1; assert a.getView().containsMember(a.getAddress()); }
/** * Tests A|6, A|7, A|8 and B|7, B|8, B|9 -> we should have a subviews in MergeView consisting of * only 2 elements: A|5 and B|5 */ public void testMultipleViewsBySameMembers() throws Exception { View a1 = View.create( a.getAddress(), 6, a.getAddress(), b.getAddress(), c.getAddress(), d.getAddress()); // {A,B,C,D} View a2 = View.create(a.getAddress(), 7, a.getAddress(), b.getAddress(), c.getAddress()); // {A,B,C} View a3 = View.create(a.getAddress(), 8, a.getAddress(), b.getAddress()); // {A,B} View a4 = View.create(a.getAddress(), 9, a.getAddress()); // {A} View b1 = View.create(b.getAddress(), 7, b.getAddress(), c.getAddress(), d.getAddress()); View b2 = View.create(b.getAddress(), 8, b.getAddress(), c.getAddress()); View b3 = View.create(b.getAddress(), 9, b.getAddress()); Util.close(c, d); // not interested in those... // A and B cannot communicate: discard(true, a, b); // inject view A|6={A} into A and B|5={B} into B injectView(a4, a); injectView(b3, b); assert a.getView().equals(a4); assert b.getView().equals(b3); List<Event> merge_events = new ArrayList<>(); for (View view : Arrays.asList(a3, a4, a2, a1)) { MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(view.getViewId(), null, null); Message msg = new Message(null, a.getAddress(), null).putHeader(merge_id, hdr); merge_events.add(new Event(Event.MSG, msg)); } for (View view : Arrays.asList(b2, b3, b1)) { MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(view.getViewId(), null, null); Message msg = new Message(null, b.getAddress(), null).putHeader(merge_id, hdr); merge_events.add(new Event(Event.MSG, msg)); } // A and B can communicate again discard(false, a, b); injectMergeEvents(merge_events, a, b); checkInconsistencies(a, b); // merge will happen between A and B Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b); System.out.println("A's view: " + a.getView() + "\nB's view: " + b.getView()); assert a.getView().size() == 2; assert a.getView().containsMember(a.getAddress()); assert a.getView().containsMember(b.getAddress()); assert a.getView().equals(b.getView()); for (View merge_view : Arrays.asList(getViewFromGMS(a), getViewFromGMS(b))) { System.out.println(merge_view); assert merge_view instanceof MergeView; List<View> subviews = ((MergeView) merge_view).getSubgroups(); assert subviews.size() == 2; } }
public void testFlushWithCrashedParticipants() throws Exception { JChannel c1 = null; JChannel c2 = null; JChannel c3 = null; try { c1 = createChannel(true, 3, "C1"); changeProps(c1); c1.connect("testFlushWithCrashedFlushCoordinator"); c2 = createChannel(c1, "C2"); changeProps(c2); c2.connect("testFlushWithCrashedFlushCoordinator"); c3 = createChannel(c1, "C3"); changeProps(c3); c3.connect("testFlushWithCrashedFlushCoordinator"); // and then kill members other than flush coordinator Util.shutdown(c3); Util.shutdown(c1); // start flush Util.startFlush(c2); c2.stopFlush(); Util.waitUntilAllChannelsHaveSameSize(10000, 500, c2); // cluster should not hang and one remaining member should have a correct view assertTrue("correct view size", c2.getView().size() == 1); } finally { Util.close(c3, c2, c1); } }
/** * Takes a membership of 10, e.g. {A,B,C,D,E,F,G,H,I,J}, creates multiple partitions (e.g. * {A,B,C}, {D}, {E,F,G,H}, {I,J}), and merges them back into one cluster. Asserts that there's * only 1 MergeView */ public void testMultipleSplits() throws Exception { e = createChannel("E", true); f = createChannel("F", true); g = createChannel("G", true); h = createChannel("H", true); i = createChannel("I", true); j = createChannel("J", true); Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b, c, d, e, f, g, h, i, j); enableInfoSender(false, a, b, c, d, e, f, g, h, i, j); // stops INFO sending in MERGE3 for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) System.out.println(ch.getName() + ": " + ch.getView()); List<View> partitions = createPartitions(4, a, b, c, d, e, f, g, h, i, j); // Install views for (View partition : partitions) { for (Address mbr : partition.getMembersRaw()) { JChannel ch = findChannel(mbr); injectView(partition, ch); } } System.out.printf("\n%d partitions:\n", partitions.size()); for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) System.out.println(ch.getName() + ": " + ch.getView()); System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters"); enableInfoSender(true, a, b, c, d, e, f, g, h, i, j); Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c, d, e, f, g, h, i, j); System.out.println("\nResulting views:"); for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); System.out.println(mv); } for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); assert mv instanceof MergeView; assert mv.size() == 10; assert ((MergeView) mv).getSubgroups().size() == partitions.size(); } for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h, i, j)) { View view = ch.getView(); assert view.size() == 10 : "view should have 10 members: " + view; } }
@BeforeMethod void setUp() throws Exception { a = createChannel("A", true); b = createChannel("B", true); c = createChannel("C", true); d = createChannel("D", true); Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b, c, d); enableInfoSender(false, a, b, c, d); // stops INFO sending in MERGE3 }
/** A: {A,B} B: {A,B} C: {C} C receives INFO from B only */ public void testMergeWithIncompleteInfos() throws Exception { Util.close(d); enableInfoSender(false, a, b, c); View one = View.create(a.getAddress(), 10, a.getAddress(), b.getAddress()); View two = View.create(c.getAddress(), 10, c.getAddress()); injectView(one, a, b); injectView(two, c); enableInfoSender(false, a, b, c); Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b); Util.waitUntilAllChannelsHaveSameSize(10000, 500, c); System.out.printf("\nPartitions:\n"); for (JChannel ch : Arrays.asList(a, b, c)) System.out.println(ch.getName() + ": " + ch.getView()); MERGE3.MergeHeader hdr = MERGE3.MergeHeader.createInfo(one.getViewId(), null, null); Message msg = new Message(null, b.getAddress(), null) .putHeader(merge_id, hdr); // B sends the INFO message to C Event merge_event = new Event(Event.MSG, msg); MERGE3 merge = (MERGE3) c.getProtocolStack().findProtocol(MERGE3.class); merge.up(merge_event); enableInfoSender(true, a, b, c); System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters"); Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c); System.out.println("\nResulting views:"); for (JChannel ch : Arrays.asList(a, b, c)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); System.out.println(mv); } for (JChannel ch : Arrays.asList(a, b, c)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); assert mv instanceof MergeView; assert mv.size() == 3; assert ((MergeView) mv).getSubgroups().size() == 2; } for (JChannel ch : Arrays.asList(a, b, c)) { View view = ch.getView(); assert view.size() == 3 : "view should have 3 members: " + view; } }
public void testMergeWithAsymetricViewsCoordIsolated() { // Isolate the coord Address coord = a.getView().getCreator(); System.out.println("Isolating coord: " + coord); List<Address> members = new ArrayList<>(); members.add(coord); View coord_view = new View(coord, 4, members); System.out.println("coord_view: " + coord_view); Channel coord_channel = findChannel(coord); System.out.println("coord_channel: " + coord_channel.getAddress()); MutableDigest digest = new MutableDigest(coord_view.getMembersRaw()); NAKACK2 nakack = (NAKACK2) coord_channel.getProtocolStack().findProtocol(NAKACK2.class); digest.merge(nakack.getDigest(coord)); GMS gms = (GMS) coord_channel.getProtocolStack().findProtocol(GMS.class); gms.installView(coord_view, digest); System.out.println("gms.getView() " + gms.getView()); System.out.println("Views are:"); for (JChannel ch : Arrays.asList(a, b, c, d)) System.out.println(ch.getAddress() + ": " + ch.getView()); JChannel merge_leader = findChannel(coord); MyReceiver receiver = new MyReceiver(); merge_leader.setReceiver(receiver); System.out.println("merge_leader: " + merge_leader.getAddressAsString()); System.out.println("Injecting MERGE event into merge leader " + merge_leader.getAddress()); Map<Address, View> merge_views = new HashMap<>(4); merge_views.put(a.getAddress(), a.getView()); merge_views.put(b.getAddress(), b.getView()); merge_views.put(c.getAddress(), c.getView()); merge_views.put(d.getAddress(), d.getView()); gms = (GMS) merge_leader.getProtocolStack().findProtocol(GMS.class); gms.up(new Event(Event.MERGE, merge_views)); Util.waitUntilAllChannelsHaveSameSize(10000, 1000, a, b, c, d); System.out.println("Views are:"); for (JChannel ch : Arrays.asList(a, b, c, d)) { View view = ch.getView(); System.out.println(ch.getAddress() + ": " + view); assert view.size() == 4; } MergeView merge_view = receiver.getView(); System.out.println("merge_view = " + merge_view); assert merge_view.size() == 4; assert merge_view.getSubgroups().size() == 2; for (View view : merge_view.getSubgroups()) assert contains(view, a.getAddress()) || contains(view, b.getAddress(), c.getAddress(), d.getAddress()); }
@BeforeMethod void init() throws Exception { a = createChannel(true, 2, "A"); b = createChannel(a, "B"); setOOBPoolSize(a, b); setStableGossip(a, b); a.connect("OOBTest"); b.connect("OOBTest"); Util.waitUntilAllChannelsHaveSameSize(10000, 1000, a, b); }
public void testAndLoop() throws Exception { int numChannels = 10; Channel channels[] = new Channel[numChannels]; for (int j = 0; j < numChannels; j++) { channels[j] = createChannel(String.valueOf((char) ('A' + j))); channels[j].connect(cName); if (j == 0) Util.sleep(1000); } Util.waitUntilAllChannelsHaveSameSize(10000, 500, channels); for (int i = 1; i <= 2; i++) { int killPositions[] = {0, 3, 5, 8}; for (int index : killPositions) Util.shutdown(channels[index]); int ch = 'M'; for (int index : killPositions) { channels[index] = createChannel(String.valueOf((char) ch++)); channels[index].connect(cName); } System.out.println("***** Round " + i + " done *****"); } Util.waitUntilAllChannelsHaveSameSize(10000, 500, channels); }
/** * Tests a simple split: {A,B} and {C,D} need to merge back into one subgroup. Checks how many * MergeViews are installed */ public void testSplitInTheMiddle() { View v1 = View.create(a.getAddress(), 10, a.getAddress(), b.getAddress()); View v2 = View.create(c.getAddress(), 10, c.getAddress(), d.getAddress()); injectView(v1, a, b); injectView(v2, c, d); Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b); Util.waitUntilAllChannelsHaveSameSize(10000, 500, c, d); for (JChannel ch : Arrays.asList(a, b, c, d)) System.out.println(ch.getName() + ": " + ch.getView()); Address merge_leader_one = new Membership().add(a.getAddress(), b.getAddress()).sort().elementAt(0); Address merge_leader_two = new Membership().add(c.getAddress(), d.getAddress()).sort().elementAt(0); for (int x = 0; x < 20; x++) { if (a.getView().size() == 4 && b.getView().size() == 4 && c.getView().size() == 4 && d.getView().size() == 4) break; for (JChannel ch : Arrays.asList(a, b, c, d)) { MERGE3 merge = (MERGE3) ch.getProtocolStack().findProtocol(MERGE3.class); merge.sendInfo(); // multicasts an INFO msg to everybody else } for (JChannel ch : Arrays.asList(findChannel(merge_leader_one), findChannel(merge_leader_two))) { MERGE3 merge = (MERGE3) ch.getProtocolStack().findProtocol(MERGE3.class); merge.checkInconsistencies(); } Util.sleep(1000); } for (JChannel ch : Arrays.asList(a, b, c, d)) System.out.println(ch.getName() + ": " + ch.getView()); for (JChannel ch : Arrays.asList(a, b, c, d)) assert ch.getView().size() == 4 : "view of " + ch.getName() + ": " + ch.getView(); }
private void _testChannels(String names[], int connectType) throws Exception { int count = names.length; List<FlushTestReceiver> channels = new ArrayList<FlushTestReceiver>(count); try { // Create a semaphore and take all its permits Semaphore semaphore = new Semaphore(count); semaphore.acquire(count); // Create channels and their threads that will block on the // semaphore boolean first = true; for (String channelName : names) { FlushTestReceiver channel = null; if (first) channel = new FlushTestReceiver(channelName, semaphore, 0, connectType); else { channel = new FlushTestReceiver( (JChannel) channels.get(0).getChannel(), channelName, semaphore, 0, connectType); } channels.add(channel); // Release one ticket at a time to allow the thread to start working channel.start(); semaphore.release(1); if (first) Util.sleep(3000); // minimize changes of a merge happening first = false; } Channel[] tmp = new Channel[channels.size()]; int cnt = 0; for (FlushTestReceiver receiver : channels) tmp[cnt++] = receiver.getChannel(); Util.waitUntilAllChannelsHaveSameSize(30000, 1000, tmp); // Reacquire the semaphore tickets; when we have them all // we know the threads are done semaphore.tryAcquire(count, 40, TimeUnit.SECONDS); Util.sleep(1000); // let all events propagate... for (FlushTestReceiver app : channels) app.getChannel().setReceiver(null); for (FlushTestReceiver app : channels) app.cleanup(); // verify block/unblock/view/get|set state sequences for all members for (FlushTestReceiver receiver : channels) { checkEventStateTransferSequence(receiver); System.out.println("event sequence is OK"); } } finally { for (FlushTestReceiver app : channels) app.cleanup(); } }
@BeforeMethod void init() throws Exception { a = createChannel(true, 2); a.setName("A"); b = createChannel(a); b.setName("B"); setOOBPoolSize(a, b); setStableGossip(a, b); a.connect("OOBTest"); b.connect("OOBTest"); View view = b.getView(); System.out.println("view = " + view); Util.waitUntilAllChannelsHaveSameSize(20000, 1000, a, b); }
@BeforeMethod public void init() throws Exception { c1 = createChannel(true, 2); c1.setName("C1"); c2 = createChannel(c1); c2.setName("C2"); setOOBPoolSize(c1, c2); setStableGossip(c1, c2); c1.connect("OOBTest"); c2.connect("OOBTest"); View view = c2.getView(); log.info("view = " + view); Util.waitUntilAllChannelsHaveSameSize(20000, 1000, c1, c2); }
@BeforeMethod void setUp() throws Exception { System.out.print("Connecting channels: "); a = createChannel("A"); disp_a = new MessageDispatcher(a, null, null); a.connect(RpcLockingTest.class.getSimpleName()); lock_a = new LockService(a).getLock("lock"); b = createChannel("B"); disp_b = new MessageDispatcher(b, null, null); b.connect(RpcLockingTest.class.getSimpleName()); lock_b = new LockService(b).getLock("lock"); Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b); System.out.println(""); disp_a.setRequestHandler( new RequestHandler() { @Override public Object handle(Message arg0) throws Exception { System.out.println("A received a message, will now try to lock the lock"); if (lock_a.tryLock()) { Assert.fail("Should not be able to lock the lock here"); System.out.println("A aquired the lock, this shouldn't be possible"); } else System.out.println("The lock was already locked, as it should be"); return "Hello"; } }); disp_b.setRequestHandler( new RequestHandler() { @Override public Object handle(Message arg0) throws Exception { System.out.println("B received a message, will now try to lock the lock"); if (lock_b.tryLock()) { Assert.fail("Should not be able to lock the lock here"); System.out.println("B aquired the lock, this shouldn't be possible"); } else System.out.println("The lock already was locked, as it should be"); return "Hello"; } }); // Print who is the coordinator if (b.getView().getMembers().get(0).equals(b.getAddress())) System.out.println("B is the coordinator"); else System.out.println("A is the coordinator"); System.out.println(""); }
private void createChannels( boolean copy_multicasts, boolean copy_unicasts, int num_outgoing_copies, int num_incoming_copies) throws Exception { c1 = createChannel(true, 3); DUPL dupl = new DUPL(copy_multicasts, copy_unicasts, num_incoming_copies, num_outgoing_copies); ProtocolStack stack = c1.getProtocolStack(); stack.insertProtocol(dupl, ProtocolStack.BELOW, NAKACK.class); c2 = createChannel(c1); c3 = createChannel(c1); c1.connect("DuplicateTest"); c2.connect("DuplicateTest"); c3.connect("DuplicateTest"); Util.waitUntilAllChannelsHaveSameSize(20000, 1000, c1, c2, c3); }
public void testSingleChannel() throws Exception { Semaphore s = new Semaphore(1); FlushTestReceiver receivers[] = { new FlushTestReceiver("c1", s, 0, FlushTestReceiver.CONNECT_ONLY) }; receivers[0].start(); s.release(1); // Make sure everyone is in sync Channel[] tmp = new Channel[receivers.length]; for (int i = 0; i < receivers.length; i++) tmp[i] = receivers[i].getChannel(); Util.waitUntilAllChannelsHaveSameSize(10000, 1000, tmp); // Reacquire the semaphore tickets; when we have them all we know the threads are done s.tryAcquire(1, 10, TimeUnit.SECONDS); receivers[0].cleanup(); Util.sleep(1000); checkEventStateTransferSequence(receivers[0]); }
public void testFlushWithCrashedFlushCoordinator() throws Exception { JChannel a = null, b = null, c = null; try { a = createChannel(true, 3, "A"); changeProps(a); a.connect("testFlushWithCrashedFlushCoordinator"); b = createChannel(a, "B"); changeProps(b); b.connect("testFlushWithCrashedFlushCoordinator"); c = createChannel(a, "C"); changeProps(c); c.connect("testFlushWithCrashedFlushCoordinator"); System.out.println("shutting down flush coordinator B"); b.down(new Event(Event.SUSPEND_BUT_FAIL)); // send out START_FLUSH and then return // now shut down B. This means, after failure detection kicks in and the new coordinator takes // over // (either A or C), that the current flush started by B will be cancelled and a new flush (by // A or C) // will be started Util.shutdown(b); a.getProtocolStack().findProtocol(FLUSH.class).setLevel("debug"); c.getProtocolStack().findProtocol(FLUSH.class).setLevel("debug"); Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, c); // cluster should not hang and two remaining members should have a correct view assertTrue("correct view size", a.getView().size() == 2); assertTrue("correct view size", c.getView().size() == 2); a.getProtocolStack().findProtocol(FLUSH.class).setLevel("warn"); c.getProtocolStack().findProtocol(FLUSH.class).setLevel("warn"); } finally { Util.close(c, b, a); } }
/** * When we flush the server queue and one or more of the delivered messages triggers a response * (in the same thread), we need to make sure the channel is connected, or else the JOIN will fail * as the exception happens on the same thread. Note that the suggested fix on JGRP-1522 will * solve this. Issue: https://issues.jboss.org/browse/JGRP-1522 */ @BMScript(dir = "scripts/BecomeServerTest", value = "testSendingOfMsgsOnUnconnectedChannel") public void testSendingOfMsgsOnUnconnectedChannel() throws Exception { a = createChannel("A"); a.setReceiver( new ReceiverAdapter() { public void receive(Message msg) { System.out.println("A: received message from " + msg.getSrc() + ": " + msg.getObject()); } }); a.connect("BecomeServerTest"); new Thread("MsgSender-A") { public void run() { sendMessage(a, "hello from A"); // will be blocked by byteman rendezvous } }.start(); b = createChannel("B"); b.setReceiver( new ReceiverAdapter() { public void receive(Message msg) { System.out.println("B: received message from " + msg.getSrc() + ": " + msg.getObject()); if (msg.getSrc().equals(a.getAddress())) { try { b.send( null, "This message would trigger an exception if the channel was not yet connected"); } catch (Exception e) { throw new RuntimeException(e); } } } }); b.connect("BecomeServerTest"); Util.waitUntilAllChannelsHaveSameSize(20000, 1000, a, b); System.out.println("\nA: " + a.getView() + "\nB: " + b.getView()); }
public void testFlushWithCrashedParticipant() throws Exception { JChannel c1 = null; JChannel c2 = null; JChannel c3 = null; try { c1 = createChannel(true, 3, "C1"); changeProps(c1); c1.connect("testFlushWithCrashedParticipant"); c2 = createChannel(c1, "C2"); changeProps(c2); c2.connect("testFlushWithCrashedParticipant"); c3 = createChannel(c1, "C3"); changeProps(c3); c3.connect("testFlushWithCrashedParticipant"); System.out.println("shutting down C3"); Util.shutdown(c3); // kill a flush participant System.out.println("C2: starting flush"); boolean rc = Util.startFlush(c2); System.out.println("flush " + (rc ? " was successful" : "failed")); assert rc; System.out.println("stopping flush"); c2.stopFlush(); System.out.println("waiting for view to contain C1 and C2"); Util.waitUntilAllChannelsHaveSameSize(10000, 500, c1, c2); // cluster should not hang and two remaining members should have a correct view System.out.println("C1: view=" + c1.getView() + "\nC2: view=" + c2.getView()); assertTrue("correct view size", c1.getView().size() == 2); assertTrue("correct view size", c2.getView().size() == 2); } finally { Util.close(c3, c2, c1); } }
public void testSequentialFlushInvocation() throws Exception { Channel a = null, b = null, c = null; try { a = createChannel(true, 3, "A"); a.connect("testSequentialFlushInvocation"); b = createChannel((JChannel) a, "B"); b.connect("testSequentialFlushInvocation"); c = createChannel((JChannel) a, "C"); c.connect("testSequentialFlushInvocation"); Util.waitUntilAllChannelsHaveSameSize(10000, 1000, a, b, c); for (int i = 0; i < 100; i++) { System.out.print("flush #" + i + ": "); a.startFlush(false); a.stopFlush(); System.out.println("OK"); } } finally { Util.close(a, b, c); } }
/** * Tests the scenario described by Dan in https://issues.jboss.org/browse/JGRP-1876; INFO messages * are injected */ public void testJGRP_1876_Dan2() throws Exception { Util.close(d, c, b, a); s = createChannel("S", false); t = createChannel("T", false); u = createChannel("U", false); v = createChannel("V", false); for (JChannel ch : Arrays.asList(s, t, u, v)) ch.getProtocolStack().removeProtocol(MERGE3.class); s.connect("MergeTest4"); t.connect("MergeTest4"); u.connect("MergeTest4"); v.connect("MergeTest4"); Util.waitUntilAllChannelsHaveSameSize(10000, 500, s, t, u, v); for (JChannel ch : Arrays.asList(s, t, u, v)) System.out.println(ch.getName() + ": " + ch.getView()); View v1 = View.create(s.getAddress(), 10, s.getAddress()); View v2 = View.create(t.getAddress(), 10, t.getAddress()); View v3 = View.create(u.getAddress(), 11, u.getAddress(), v.getAddress()); injectView(v1, s); injectView(v2, t); injectView(v3, u, v); Util.waitUntilAllChannelsHaveSameSize(10000, 500, s); Util.waitUntilAllChannelsHaveSameSize(10000, 500, t); Util.waitUntilAllChannelsHaveSameSize(10000, 500, u, v); System.out.printf("\nPartitions:\n"); for (JChannel ch : Arrays.asList(s, t, u, v)) System.out.println(ch.getName() + ": " + ch.getView()); System.out.println("\nInjecting MERGE events into GMS to merge subclusters"); Map<Address, View> different_views = new HashMap<>(); different_views.put(s.getAddress(), v1); different_views.put(t.getAddress(), v2); different_views.put(v.getAddress(), v3); Collection<Address> coords = Util.determineActualMergeCoords(different_views); Address merge_leader = new Membership().add(coords).sort().elementAt(0); System.out.printf("--> coords=%s, merge_leader=%s\n", coords, merge_leader); GMS gms = (GMS) findChannel(merge_leader).getProtocolStack().findProtocol(GMS.class); gms.up(new Event(Event.MERGE, different_views)); Util.waitUntilAllChannelsHaveSameSize(30000, 1000, s, t, u, v); System.out.println("\nResulting views:"); for (JChannel ch : Arrays.asList(s, t, u, v)) { gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); System.out.println(mv); } for (JChannel ch : Arrays.asList(s, t, u, v)) { gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); assert mv instanceof MergeView; assert mv.size() == 4; assert ((MergeView) mv).getSubgroups().size() == 3; } for (JChannel ch : Arrays.asList(s, t, u, v)) { View view = ch.getView(); assert view.size() == 4 : "view should have 4 members: " + view; } }
/** Tests that notifications are routed to all sites. */ public void testNotificationAndRpcRelay2Transit() throws Exception { a.connect(LON_CLUSTER); b.connect(LON_CLUSTER); rpca.start(); rpcb.start(); Util.waitUntilAllChannelsHaveSameSize(30000, 500, a, b); x.connect(SFO_CLUSTER); y.connect(SFO_CLUSTER); rpcx.start(); rpcy.start(); Util.waitUntilAllChannelsHaveSameSize(30000, 500, x, y); assert x.getView().size() == 2; RELAY2 ar = (RELAY2) a.getProtocolStack().findProtocol(RELAY2.class); RELAY2 xr = (RELAY2) x.getProtocolStack().findProtocol(RELAY2.class); assert ar != null && xr != null; JChannel a_bridge = null, x_bridge = null; for (int i = 0; i < 20; i++) { a_bridge = ar.getBridge(SFO); x_bridge = xr.getBridge(LON); if (a_bridge != null && x_bridge != null && a_bridge.getView().size() == 2 && x_bridge.getView().size() == 2) break; Util.sleep(500); } assert a_bridge != null && x_bridge != null; System.out.println("A's bridge channel: " + a_bridge.getView()); System.out.println("X's bridge channel: " + x_bridge.getView()); assert a_bridge.getView().size() == 2 : "bridge view is " + a_bridge.getView(); assert x_bridge.getView().size() == 2 : "bridge view is " + x_bridge.getView(); Relayer.Route route = getRoute(x, LON); System.out.println("Route at sfo to lon: " + route); assert route != null; System.out.println("B: sending message 0 to the site master of SFO"); Address sm_sfo = new SiteMaster(SFO); b.send(sm_sfo, 0); checkMsgDelivery(xl); System.out.println("B: sending message to all"); b.send(null, 0); checkMsgDelivery(xl, yl, al, bl); MethodCall call = new MethodCall(ServerObject.class.getMethod("foo")); System.out.println("B: call foo method on A"); Object rsp = rpcb.callRemoteMethod(a.getAddress(), call, new RequestOptions(ResponseMode.GET_ALL, 5000)); System.out.println("RSP is: " + rsp); System.out.println("B: call foo method on SFO master site"); rsp = rpcb.callRemoteMethod(sm_sfo, call, new RequestOptions(ResponseMode.GET_ALL, 5000)); System.out.println("RSP is: " + rsp); System.out.println("B: call foo method on all members dest = null"); RspList<Integer> rsps = rpcb.callRemoteMethods(null, call, new RequestOptions(ResponseMode.GET_ALL, 5000)); System.out.println("RSPs are: \n" + rsps); View bridge_view = xr.getBridgeView(BRIDGE_CLUSTER); System.out.println("bridge_view = " + bridge_view); route = getRoute(x, LON); System.out.println("Route at sfo to lon: " + route); assert route != null; }
/** Tests the scenario described by Karim in https://issues.jboss.org/browse/JGRP-1876 */ public void testJGRP_1876() throws Exception { e = createChannel("E", true); f = createChannel("F", true); g = createChannel("G", true); h = createChannel("H", true); Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b, c, d, e, f, g, h); enableInfoSender(false, a, b, c, d, e, f, g, h); // stops INFO sending in MERGE3 for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h)) System.out.println(ch.getName() + ": " + ch.getView()); View v1 = View.create( a.getAddress(), 10, a.getAddress(), b.getAddress(), c.getAddress(), d.getAddress()); View v2 = View.create( a.getAddress(), 9, a.getAddress(), b.getAddress(), c.getAddress(), d.getAddress(), e.getAddress(), f.getAddress()); View v3 = View.create(g.getAddress(), 8, g.getAddress(), h.getAddress()); injectView(v1, a, b, c, d); injectView(v2, e, f); injectView(v3, g, h); enableInfoSender(false, a, b, c, d, e, f, g, h); // stops INFO sending in MERGE3 Util.waitUntilAllChannelsHaveSameSize(10000, 500, a, b, c, d); for (int x = 0; x < 10; x++) { // can't use waitUntilAllChannelsHaveSameSize() as the channel list's length != view // ! if (e.getView().size() == v2.size() && f.getView().size() == v2.size()) break; Util.sleep(500); } assert e.getView().size() == v2.size() : "E's view: " + e.getView(); assert f.getView().size() == v2.size() : "F's view: " + f.getView(); Util.waitUntilAllChannelsHaveSameSize(10000, 500, g, h); System.out.printf("\nPartitions:\n"); for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h)) System.out.println(ch.getName() + ": " + ch.getView()); enableInfoSender(true, a, b, c, d, e, f, g, h); System.out.println("\nEnabling INFO sending in merge protocols to merge subclusters"); Util.waitUntilAllChannelsHaveSameSize(30000, 1000, a, b, c, d, e, f, g, h); System.out.println("\nResulting views:"); for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); System.out.println(mv); } for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h)) { GMS gms = (GMS) ch.getProtocolStack().findProtocol(GMS.class); View mv = gms.view(); assert mv instanceof MergeView; assert mv.size() == 8; assert ((MergeView) mv).getSubgroups().size() == 3; } for (JChannel ch : Arrays.asList(a, b, c, d, e, f, g, h)) { View view = ch.getView(); assert view.size() == 8 : "view should have 8 members: " + view; } }