/**
* Sends the new view and digest to all subgroup coordinors in coords. Each coord will in turn
*
* <ol>
* <li>broadcast the new view and digest to all the members of its subgroup (MergeView)
* <li>on reception of the view, if it is a MergeView, each member will set the digest and
* install the new view
* </ol>
*/
private void sendMergeView(
Collection<Address> coords, MergeData combined_merge_data, MergeId merge_id) {
if (coords == null || combined_merge_data == null) return;
View view = combined_merge_data.view;
Digest digest = combined_merge_data.digest;
if (view == null || digest == null) {
if (log.isErrorEnabled())
log.error("view or digest is null, cannot send consolidated merge view/digest");
return;
}
if (log.isDebugEnabled())
log.debug(
gms.local_addr + ": sending merge view " + view.getVid() + " to coordinators " + coords);
gms.merge_ack_collector.reset(coords);
int size = gms.merge_ack_collector.size();
long timeout = gms.view_ack_collection_timeout;
long start = System.currentTimeMillis();
for (Address coord : coords) {
Message msg = new Message(coord, null, null);
GMS.GmsHeader hdr = new GMS.GmsHeader(GMS.GmsHeader.INSTALL_MERGE_VIEW);
hdr.view = view;
hdr.my_digest = digest;
hdr.merge_id = merge_id;
msg.putHeader(gms.getId(), hdr);
gms.getDownProtocol().down(new Event(Event.MSG, msg));
}
// [JGRP-700] - FLUSH: flushing should span merge
// if flush is in stack wait for acks from separated island coordinators
if (gms.flushProtocolInStack) {
try {
gms.merge_ack_collector.waitForAllAcks(timeout);
long stop = System.currentTimeMillis();
if (log.isTraceEnabled())
log.trace(
"received all ACKs ("
+ size
+ ") for merge view "
+ view
+ " in "
+ (stop - start)
+ "ms");
} catch (TimeoutException e) {
log.warn(
gms.local_addr
+ ": failed to collect all ACKs for merge ("
+ size
+ ") for view "
+ view
+ " after "
+ timeout
+ "ms, missing ACKs from "
+ gms.merge_ack_collector.printMissing());
}
}
}
public static void testDetermineMergeParticipantsAndMergeCoords4() {
Address a = Util.createRandomAddress(),
b = Util.createRandomAddress(),
c = Util.createRandomAddress(),
d = Util.createRandomAddress();
org.jgroups.util.UUID.add(a, "A");
org.jgroups.util.UUID.add(b, "B");
org.jgroups.util.UUID.add(c, "C");
org.jgroups.util.UUID.add(d, "D");
View v1 = View.create(a, 1, a, b);
View v2 = View.create(c, 1, c, d);
Map<Address, View> map = new HashMap<>();
map.put(a, v1);
map.put(b, v1);
map.put(d, v2);
StringBuilder sb = new StringBuilder("map:\n");
for (Map.Entry<Address, View> entry : map.entrySet())
sb.append(entry.getKey() + ": " + entry.getValue() + "\n");
System.out.println(sb);
Collection<Address> merge_participants = Util.determineMergeParticipants(map);
System.out.println("merge_participants = " + merge_participants);
assert merge_participants.size() == 3;
assert merge_participants.contains(a)
&& merge_participants.contains(c)
&& merge_participants.contains(d);
Collection<Address> merge_coords = Util.determineMergeCoords(map);
System.out.println("merge_coords = " + merge_coords);
assert merge_coords.size() == 2;
assert merge_coords.contains(a) && merge_coords.contains(c);
}
/**
* @param views Guaranteed to be non-null and to have >= 2 members, or else this thread would
* not be started
*/
public synchronized void start(Map<Address, View> views) {
if (thread == null || thread.isAlive()) {
this.coords.clear();
// now remove all members which don't have us in their view, so RPCs won't block (e.g.
// FLUSH)
// https://jira.jboss.org/browse/JGRP-1061
sanitizeViews(views);
// Add all different coordinators of the views into the hashmap and sets their members:
Collection<Address> coordinators = Util.determineMergeCoords(views);
for (Address coord : coordinators) {
View view = views.get(coord);
if (view != null) this.coords.put(coord, new ArrayList<Address>(view.getMembers()));
}
// For the merge participants which are not coordinator, we simply add them, and the
// associated
// membership list consists only of themselves
Collection<Address> merge_participants = Util.determineMergeParticipants(views);
merge_participants.removeAll(coordinators);
for (Address merge_participant : merge_participants) {
Collection<Address> tmp = new ArrayList<Address>();
tmp.add(merge_participant);
coords.putIfAbsent(merge_participant, tmp);
}
thread = gms.getThreadFactory().newThread(this, "MergeTask");
thread.setDaemon(true);
thread.start();
}
}
void setAnycastCount() throws Exception {
int tmp = Util.readIntFromStdin("Anycast count: ");
View view = channel.getView();
if (tmp > view.size()) {
System.err.println(
"anycast count must be smaller or equal to the view size (" + view + ")\n");
return;
}
disp.callRemoteMethods(null, new MethodCall(SET_ANYCAST_COUNT, tmp), RequestOptions.SYNC());
}
public static void testAllEqual() {
Address[] mbrs = Util.createRandomAddresses(5);
View[] views = {
View.create(mbrs[0], 1, mbrs), View.create(mbrs[0], 1, mbrs), View.create(mbrs[0], 1, mbrs)
};
boolean same = Util.allEqual(Arrays.asList(views));
System.out.println("views=" + Arrays.toString(views) + ", same = " + same);
assert same;
views =
new View[] {
View.create(mbrs[0], 1, mbrs),
View.create(mbrs[0], 2, mbrs),
View.create(mbrs[0], 1, mbrs)
};
same = Util.allEqual(Arrays.asList(views));
System.out.println("views=" + Arrays.toString(views) + ", same = " + same);
assert !same;
views =
new View[] {
View.create(mbrs[1], 1, mbrs),
View.create(mbrs[0], 1, mbrs),
View.create(mbrs[0], 1, mbrs)
};
same = Util.allEqual(Arrays.asList(views));
System.out.println("views=" + Arrays.toString(views) + ", same = " + same);
assert !same;
}
/**
* Merge all MergeData. All MergeData elements should be disjunct (both views and digests).
* However, this method is prepared to resolve duplicate entries (for the same member).
* Resolution strategy for views is to merge only 1 of the duplicate members. Resolution
* strategy for digests is to take the higher seqnos for duplicate digests.
*
* <p>After merging all members into a Membership and subsequent sorting, the first member of
* the sorted membership will be the new coordinator. This method has a lock on merge_rsps.
*
* @param merge_rsps A list of MergeData items. Elements with merge_rejected=true were removed
* before. Is guaranteed not to be null and to contain at least 1 member.
*/
private MergeData consolidateMergeData(Vector<MergeData> merge_rsps) {
long logical_time = 0; // for new_vid
Membership new_mbrs = new Membership();
Vector<View> subgroups =
new Vector<View>(11); // contains a list of Views, each View is a subgroup
for (MergeData tmp_data : merge_rsps) {
View tmp_view = tmp_data.getView();
if (tmp_view != null) {
ViewId tmp_vid = tmp_view.getVid();
if (tmp_vid != null) {
// compute the new view id (max of all vids +1)
logical_time = Math.max(logical_time, tmp_vid.getId());
}
// merge all membership lists into one (prevent duplicates)
new_mbrs.add(tmp_view.getMembers());
subgroups.addElement((View) tmp_view.clone());
}
}
// the new coordinator is the first member of the consolidated & sorted membership list
new_mbrs.sort();
Address new_coord = new_mbrs.size() > 0 ? new_mbrs.elementAt(0) : null;
if (new_coord == null) {
if (log.isErrorEnabled()) log.error("new_coord == null");
return null;
}
// should be the highest view ID seen up to now plus 1
ViewId new_vid = new ViewId(new_coord, logical_time + 1);
// determine the new view
MergeView new_view = new MergeView(new_vid, new_mbrs.getMembers(), subgroups);
// determine the new digest
Digest new_digest = consolidateDigests(merge_rsps, new_mbrs.size());
if (new_digest == null) {
if (log.isErrorEnabled())
log.error("Merge leader " + gms.local_addr + ": could not consolidate digest for merge");
return null;
}
if (log.isDebugEnabled())
log.debug(
"Merge leader "
+ gms.local_addr
+ ": consolidated view="
+ new_view
+ "\nconsolidated digest="
+ new_digest);
return new MergeData(gms.local_addr, new_view, new_digest);
}
protected void sendDiscoveryResponse(
Address logical_addr,
List<PhysicalAddress> physical_addrs,
boolean is_server,
boolean return_view_only,
String logical_name,
final Address sender) {
PingData data;
if (return_view_only) {
data = new PingData(logical_addr, view, is_server, null, null);
} else {
ViewId view_id = view != null ? view.getViewId() : null;
data = new PingData(logical_addr, null, view_id, is_server, logical_name, physical_addrs);
}
final Message rsp_msg = new Message(sender, null, null);
rsp_msg.setFlag(Message.OOB);
final PingHeader rsp_hdr = new PingHeader(PingHeader.GET_MBRS_RSP, data);
rsp_msg.putHeader(this.id, rsp_hdr);
if (stagger_timeout > 0) {
int view_size = view != null ? view.size() : 10;
int rank = Util.getRank(view, local_addr); // returns 0 if view or local_addr are null
long sleep_time =
rank == 0
? Util.random(stagger_timeout)
: stagger_timeout * rank / view_size - (stagger_timeout / view_size);
timer.schedule(
new Runnable() {
public void run() {
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": received GET_MBRS_REQ from "
+ sender
+ ", sending staggered response "
+ rsp_hdr);
down_prot.down(new Event(Event.MSG, rsp_msg));
}
},
sleep_time,
TimeUnit.MILLISECONDS);
return;
}
if (log.isTraceEnabled())
log.trace("received GET_MBRS_REQ from " + sender + ", sending response " + rsp_hdr);
down_prot.down(new Event(Event.MSG, rsp_msg));
}
/**
* Merge all MergeData. All MergeData elements should be disjunct (both views and digests).
* However, this method is prepared to resolve duplicate entries (for the same member).
* Resolution strategy for views is to merge only 1 of the duplicate members. Resolution
* strategy for digests is to take the higher seqnos for duplicate digests.
*
* <p>After merging all members into a Membership and subsequent sorting, the first member of
* the sorted membership will be the new coordinator. This method has a lock on merge_rsps.
*
* @param merge_rsps A list of MergeData items. Elements with merge_rejected=true were removed
* before. Is guaranteed not to be null and to contain at least 1 member.
*/
private MergeData consolidateMergeData(List<MergeData> merge_rsps) {
long logical_time = 0; // for new_vid
List<View> subgroups =
new ArrayList<View>(11); // contains a list of Views, each View is a subgroup
Collection<Collection<Address>> sub_mbrships = new ArrayList<Collection<Address>>();
for (MergeData tmp_data : merge_rsps) {
View tmp_view = tmp_data.getView();
if (tmp_view != null) {
ViewId tmp_vid = tmp_view.getVid();
if (tmp_vid != null) {
// compute the new view id (max of all vids +1)
logical_time = Math.max(logical_time, tmp_vid.getId());
}
// merge all membership lists into one (prevent duplicates)
sub_mbrships.add(new ArrayList<Address>(tmp_view.getMembers()));
subgroups.add(tmp_view.copy());
}
}
// determine the new digest
Digest new_digest = consolidateDigests(merge_rsps, merge_rsps.size());
if (new_digest == null) return null;
// remove all members from the new member list that are not in the digest
Collection<Address> digest_mbrs = new_digest.getMembers();
for (Collection<Address> coll : sub_mbrships) coll.retainAll(digest_mbrs);
List<Address> merged_mbrs = gms.computeNewMembership(sub_mbrships);
// the new coordinator is the first member of the consolidated & sorted membership list
Address new_coord = merged_mbrs.isEmpty() ? null : merged_mbrs.get(0);
if (new_coord == null) return null;
// should be the highest view ID seen up to now plus 1
ViewId new_vid = new ViewId(new_coord, logical_time + 1);
// determine the new view
MergeView new_view = new MergeView(new_vid, merged_mbrs, subgroups);
if (log.isTraceEnabled())
log.trace(
gms.local_addr
+ ": consolidated view="
+ new_view
+ "\nconsolidated digest="
+ new_digest);
return new MergeData(gms.local_addr, new_view, new_digest);
}
/**
* An event is to be sent down the stack. The layer may want to examine its type and perform some
* action on it, depending on the event's type. If the event is a message MSG, then the layer may
* need to add a header to it (or do nothing at all) before sending it down the stack using <code>
* PassDown</code>. In case of a GET_ADDRESS event (which tries to retrieve the stack's address
* from one of the bottom layers), the layer may need to send a new response event back up the
* stack using <code>up_prot.up()</code>. The PING protocol is interested in several different
* down events, Event.FIND_INITIAL_MBRS - sent by the GMS layer and expecting a GET_MBRS_OK
* Event.TMP_VIEW and Event.VIEW_CHANGE - a view change event Event.BECOME_SERVER - called after
* client has joined and is fully working group member Event.CONNECT, Event.DISCONNECT.
*/
@SuppressWarnings("unchecked")
public Object down(Event evt) {
switch (evt.getType()) {
case Event.FIND_INITIAL_MBRS: // sent by GMS layer
case Event.FIND_ALL_VIEWS:
// sends the GET_MBRS_REQ to all members, waits 'timeout' ms or until 'num_initial_members'
// have been retrieved
long start = System.currentTimeMillis();
boolean find_all_views = evt.getType() == Event.FIND_ALL_VIEWS;
Promise<JoinRsp> promise = (Promise<JoinRsp>) evt.getArg();
List<PingData> rsps = find_all_views ? findAllViews(promise) : findInitialMembers(promise);
long diff = System.currentTimeMillis() - start;
if (log.isTraceEnabled())
log.trace("discovery took " + diff + " ms: responses: " + Util.printPingData(rsps));
return rsps;
case Event.TMP_VIEW:
case Event.VIEW_CHANGE:
List<Address> tmp;
view = (View) evt.getArg();
if ((tmp = view.getMembers()) != null) {
synchronized (members) {
members.clear();
members.addAll(tmp);
}
}
current_coord = !members.isEmpty() ? members.get(0) : null;
is_coord = current_coord != null && local_addr != null && current_coord.equals(local_addr);
return down_prot.down(evt);
case Event.BECOME_SERVER: // called after client has joined and is fully working group member
down_prot.down(evt);
is_server = true;
return null;
case Event.SET_LOCAL_ADDRESS:
local_addr = (Address) evt.getArg();
return down_prot.down(evt);
case Event.CONNECT:
case Event.CONNECT_WITH_STATE_TRANSFER:
case Event.CONNECT_USE_FLUSH:
case Event.CONNECT_WITH_STATE_TRANSFER_USE_FLUSH:
is_leaving = false;
group_addr = (String) evt.getArg();
Object ret = down_prot.down(evt);
handleConnect();
return ret;
case Event.DISCONNECT:
is_leaving = true;
handleDisconnect();
return down_prot.down(evt);
default:
return down_prot.down(evt); // Pass on to the layer below us
}
}
// If we're becoming coordinator, we need to handle TMP_VIEW as
// an immediate change of view. See JGRP-1452.
private void handleTmpView(View v) {
List<Address> mbrs = v.getMembers();
if (mbrs.isEmpty()) return;
Address new_coord = mbrs.get(0);
if (!new_coord.equals(coord) && local_addr != null && local_addr.equals(new_coord))
handleViewChange(v);
}
public List<PingData> findAllViews(Promise<JoinRsp> promise) {
int num_expected_mbrs =
Math.max(
max_found_members,
Math.max(num_initial_members, view != null ? view.size() : num_initial_members));
max_found_members = Math.max(max_found_members, num_expected_mbrs);
return findMembers(promise, num_expected_mbrs, false, getViewId());
}
protected void handleViewChange(View v) {
List<Address> mbrs = v.getMembers();
if (mbrs.isEmpty()) return;
if (view == null || view.compareTo(v) < 0) view = v;
else return;
delivery_table.keySet().retainAll(mbrs);
Address existing_coord = coord, new_coord = mbrs.get(0);
boolean coord_changed = !Objects.equals(existing_coord, new_coord);
if (coord_changed && new_coord != null) {
stopFlusher();
startFlusher(
new_coord); // needs to be done in the background, to prevent blocking if down() would
// block
}
}
public void startFlush(List<Address> flushParticipants, boolean automatic_resume)
throws Exception {
if (!flushSupported())
throw new IllegalStateException(
"Flush is not supported, add pbcast.FLUSH protocol to your configuration");
View v = getView();
boolean validParticipants = v != null && v.getMembers().containsAll(flushParticipants);
if (!validParticipants)
throw new IllegalArgumentException(
"Current view " + v + " does not contain all flush participants " + flushParticipants);
try {
down(new Event(Event.SUSPEND, flushParticipants));
} catch (Exception e) {
throw new Exception("Flush failed", e.getCause());
} finally {
if (automatic_resume) stopFlush(flushParticipants);
}
}
@BeforeMethod
protected void setUp() throws Exception {
a = Util.createRandomAddress("A");
b = Util.createRandomAddress("B");
c = Util.createRandomAddress("C");
v1 = View.create(a, 1, a, b);
v2 = View.create(a, 2, a, b, c);
nak = new NAKACK2();
d1 = new Digest(v1.getMembersRaw(), new long[] {11, 11, 30, 35});
d2 = new Digest(v2.getMembersRaw(), new long[] {10, 10, 30, 30, 50, 50});
TP transport =
new TP() {
public boolean supportsMulticasting() {
return false;
}
public void sendMulticast(byte[] data, int offset, int length) throws Exception {}
public void sendUnicast(PhysicalAddress dest, byte[] data, int offset, int length)
throws Exception {}
public String getInfo() {
return null;
}
public Object down(Event evt) {
return null;
}
protected PhysicalAddress getPhysicalAddress() {
return null;
}
public TimeScheduler getTimer() {
return new DefaultTimeScheduler(1);
}
};
transport.setId(TP_ID);
nak.setDownProtocol(transport);
nak.start();
}
protected boolean _startFlush(
final View new_view, int maxAttempts, long randomFloor, long randomCeiling) {
if (!flushProtocolInStack) return true;
if (flushInvokerClass != null) {
try {
Callable<Boolean> invoker =
flushInvokerClass.getDeclaredConstructor(View.class).newInstance(new_view);
return invoker.call();
} catch (Throwable e) {
return false;
}
}
try {
boolean successfulFlush = false;
boolean validView = new_view != null && new_view.size() > 0;
if (validView && flushProtocolInStack) {
int attemptCount = 0;
while (attemptCount < maxAttempts) {
try {
up_prot.up(new Event(Event.SUSPEND, new ArrayList<Address>(new_view.getMembers())));
successfulFlush = true;
break;
} catch (Exception e) {
Util.sleepRandom(randomFloor, randomCeiling);
attemptCount++;
}
}
if (successfulFlush) {
if (log.isTraceEnabled()) log.trace(local_addr + ": successful GMS flush by coordinator");
} else {
if (log.isWarnEnabled()) log.warn(local_addr + ": GMS flush by coordinator failed");
}
}
return successfulFlush;
} catch (Exception e) {
return false;
}
}
@BeforeMethod
protected void setup() throws Exception {
receiver = new MockProtocol();
nak = (NAKACK2) new NAKACK2().setValue("use_mcast_xmit", false);
transport = new MockTransport();
ProtocolStack stack = new ProtocolStack();
stack.addProtocols(transport, nak, receiver);
stack.init();
nak.down(new Event(Event.BECOME_SERVER));
nak.down(new Event(Event.SET_LOCAL_ADDRESS, A));
Digest digest = new Digest(view.getMembersRaw(), new long[] {0, 0, 0, 0});
nak.down(new Event(Event.SET_DIGEST, digest));
}
/**
* Removes all members from a given view which don't have us in their view
* (https://jira.jboss.org/browse/JGRP-1061). Example:
*
* <pre>
* A: AB
* B: AB
* C: ABC
* </pre>
*
* becomes
*
* <pre>
* A: AB
* B: AB
* C: C // A and B don't have C in their views
* </pre>
*
* @param map A map of members and their associated views
*/
public static void sanitizeViews(Map<Address, View> map) {
if (map == null) return;
for (Map.Entry<Address, View> entry : map.entrySet()) {
Address key = entry.getKey();
Collection<Address> members = new ArrayList<Address>(entry.getValue().getMembers());
boolean modified = false;
for (Iterator<Address> it = members.iterator(); it.hasNext(); ) {
Address val = it.next();
if (val.equals(key)) // we can always talk to ourself !
continue;
View view = map.get(val);
final Collection<Address> tmp_mbrs = view != null ? view.getMembers() : null;
if (tmp_mbrs != null && !tmp_mbrs.contains(key)) {
it.remove();
modified = true;
}
}
if (modified) {
View old_view = entry.getValue();
entry.setValue(new View(old_view.getVid(), members));
}
}
}
/**
* Computes the next view. Returns a copy that has <code>old_mbrs</code> and <code>suspected_mbrs
* </code> removed and <code>new_mbrs</code> added.
*/
public View getNextView(
Collection<Address> new_mbrs,
Collection<Address> old_mbrs,
Collection<Address> suspected_mbrs) {
synchronized (members) {
ViewId view_id = view != null ? view.getViewId() : null;
if (view_id == null) {
log.error("view_id is null");
return null; // this should *never* happen !
}
long vid = Math.max(view_id.getId(), ltime) + 1;
ltime = vid;
Membership tmp_mbrs = tmp_members.copy(); // always operate on the temporary membership
tmp_mbrs.remove(suspected_mbrs);
tmp_mbrs.remove(old_mbrs);
tmp_mbrs.add(new_mbrs);
List<Address> mbrs = tmp_mbrs.getMembers();
Address new_coord = local_addr;
if (!mbrs.isEmpty()) new_coord = mbrs.get(0);
View v = new View(new_coord, vid, mbrs);
// Update membership (see DESIGN for explanation):
tmp_members.set(mbrs);
// Update joining list (see DESIGN for explanation)
if (new_mbrs != null) {
for (Address tmp_mbr : new_mbrs) {
if (!joining.contains(tmp_mbr)) joining.add(tmp_mbr);
}
}
// Update leaving list (see DESIGN for explanations)
if (old_mbrs != null) {
for (Address addr : old_mbrs) {
if (!leaving.contains(addr)) leaving.add(addr);
}
}
if (suspected_mbrs != null) {
for (Address addr : suspected_mbrs) {
if (!leaving.contains(addr)) leaving.add(addr);
}
}
return v;
}
}
public int size() {
int retval = Global.BYTE_SIZE * 2; // type + merge_rejected
retval += Global.BYTE_SIZE; // presence view
retval += Global.BYTE_SIZE; // MergeView or View
if (view != null) retval += view.serializedSize();
retval += Util.size(mbr);
retval += Util.size(mbrs);
retval += Global.BYTE_SIZE; // presence of join_rsp
if (join_rsp != null) retval += join_rsp.serializedSize();
retval += Global.BYTE_SIZE; // presence for my_digest
if (my_digest != null) retval += my_digest.serializedSize();
retval += Global.BYTE_SIZE; // presence for merge_id
if (merge_id != null) retval += merge_id.size();
retval += Global.BYTE_SIZE; // boolean useFlushIfPresent
return retval;
}
protected void sendDiscoveryResponse(
Address logical_addr,
PhysicalAddress physical_addr,
String logical_name,
final Address sender,
boolean coord) {
final PingData data =
new PingData(logical_addr, is_server, logical_name, physical_addr).coord(coord);
final Message rsp_msg =
new Message(sender)
.setFlag(Message.Flag.INTERNAL, Message.Flag.OOB, Message.Flag.DONT_BUNDLE)
.putHeader(this.id, new PingHeader(PingHeader.GET_MBRS_RSP))
.setBuffer(marshal(data));
if (stagger_timeout > 0) {
int view_size = view != null ? view.size() : 10;
int rank = Util.getRank(view, local_addr); // returns 0 if view or local_addr are null
long sleep_time =
rank == 0
? Util.random(stagger_timeout)
: stagger_timeout * rank / view_size - (stagger_timeout / view_size);
timer.schedule(
(Runnable)
() -> {
log.trace(
"%s: received GET_MBRS_REQ from %s, sending staggered response %s",
local_addr, sender, data);
down_prot.down(new Event(Event.MSG, rsp_msg));
},
sleep_time,
TimeUnit.MILLISECONDS);
return;
}
log.trace("%s: received GET_MBRS_REQ from %s, sending response %s", local_addr, sender, data);
down_prot.down(new Event(Event.MSG, rsp_msg));
}
public static void testLeftMembers2() {
final Address a = Util.createRandomAddress(),
b = Util.createRandomAddress(),
c = Util.createRandomAddress(),
d = Util.createRandomAddress();
List<Address> v1 = new ArrayList<>();
v1.add(a);
v1.add(b);
v1.add(c);
v1.add(d);
List<Address> v2 = new ArrayList<>();
v2.add(c);
v2.add(d);
v2.add(a);
v2.add(b);
View one = new View(new ViewId(a, 1), v1), two = new View(new ViewId(b, 2), v2);
List<Address> left = View.leftMembers(one, two);
System.out.println("left = " + left);
assert left != null;
assert left.isEmpty();
}
/**
* Tests thata there aren't unnecessary retransmissions caused by the retransmit task in NAKACK
*
* <p>https://issues.jboss.org/browse/JGRP-1539
*
* @author Bela Ban
* @since 3.3
*/
@Test(groups = Global.FUNCTIONAL, singleThreaded = true)
public class NAKACK2_RetransmissionTest {
protected static final short ID = ClassConfigurator.getProtocolId(NAKACK2.class);
protected static final Address A = Util.createRandomAddress("A"),
B = Util.createRandomAddress("B");
protected static final View view = View.create(A, 1, A, B);
protected NAKACK2 nak;
protected MockTransport transport;
protected MockProtocol receiver;
@BeforeMethod
protected void setup() throws Exception {
receiver = new MockProtocol();
nak = (NAKACK2) new NAKACK2().setValue("use_mcast_xmit", false);
transport = new MockTransport();
ProtocolStack stack = new ProtocolStack();
stack.addProtocols(transport, nak, receiver);
stack.init();
nak.down(new Event(Event.BECOME_SERVER));
nak.down(new Event(Event.SET_LOCAL_ADDRESS, A));
Digest digest = new Digest(view.getMembersRaw(), new long[] {0, 0, 0, 0});
nak.down(new Event(Event.SET_DIGEST, digest));
}
/**
* - Send a few messages such that missing messages are 5,7, 9-13, 18 - Kick off the
* retransmission task: no retransmit messages must be seen - Send some missing messages and
* create a few more gaps, such that the missing messages are 5, 10-12, 20-22, 25, 30 - Trigger a
* retransmit task run - The retransmit requests need to be: 5, 10-12 - Trigger a next run of the
* retransmit task. The retransmit messages need to be 5, 10-12, 20-22, 25, 30 - Make NAKACK2
* receive missing messages 5, 10-12 - Kick off another retransmission run: the missing messages
* are 20-22, 25, 30
*
* <p>- Receive all missing messages - On the last run of the retransmit task, no messages are
* retransmitted
*/
public void testRetransmission() {
injectMessages(1, 2, 3, 4, 6, 8, 14, 15, 16, 17, 19);
assertReceived(1, 2, 3, 4); // only messages 1-4 are delivered, there's a gap at 5
nak.triggerXmit();
// assertXmitRequests(5, 7, 9,10,11,12,13, 18);
assertXmitRequests(); // the first time, there will *not* be any retransmit requests !
injectMessages(7, 9, 13, 18, 23, 24, 26, 27, 28, 29, 31);
nak.triggerXmit();
assertXmitRequests(5, 10, 11, 12);
nak.triggerXmit();
assertXmitRequests(5, 10, 11, 12, 20, 21, 22, 25, 30);
injectMessages(5, 10, 11, 12);
nak.triggerXmit();
assertXmitRequests(20, 21, 22, 25, 30);
injectMessages(20, 21, 22, 25, 30);
nak.triggerXmit();
assertXmitRequests();
}
protected void injectMessages(long... seqnos) {
for (long seqno : seqnos) injectMessage(seqno);
}
/** Makes NAKACK2 receive a message with the given seqno */
protected void injectMessage(long seqno) {
Message msg = new Message(null, B, null);
NakAckHeader2 hdr = NakAckHeader2.createMessageHeader(seqno);
msg.putHeader(ID, hdr);
nak.up(new Event(Event.MSG, msg));
}
/**
* Asserts that the delivered messages are in the same order than the expected seqnos and then
* clears the list
*/
protected void assertReceived(long... seqnos) {
List<Long> msgs = receiver.getMsgs();
assert msgs.size() == seqnos.length
: "expected=" + Arrays.toString(seqnos) + ", received=" + msgs;
for (int i = 0; i < seqnos.length; i++)
assert seqnos[i] == msgs.get(i)
: "expected=" + Arrays.toString(seqnos) + ", received=" + msgs;
msgs.clear();
}
/** Asserts that the expected retransmission requests match the actual ones */
protected void assertXmitRequests(long... expected_seqnos) {
List<Long> actual_xmit_reqs = transport.getXmitRequests();
assert actual_xmit_reqs.size() == expected_seqnos.length
: "size mismatch: expected="
+ Arrays.toString(expected_seqnos)
+ ", received="
+ actual_xmit_reqs;
for (int i = 0; i < expected_seqnos.length; i++) {
assert expected_seqnos[i] == actual_xmit_reqs.get(i)
: "expected=" + Arrays.toString(expected_seqnos) + ", received=" + actual_xmit_reqs;
}
actual_xmit_reqs.clear();
}
/** Used to catch retransmit requests sent by NAKACK to the transport */
protected static class MockTransport extends TP {
protected final List<Long> xmit_requests = new LinkedList<>();
public List<Long> getXmitRequests() {
return xmit_requests;
}
public void clear() {
xmit_requests.clear();
}
public void init() throws Exception {}
public boolean supportsMulticasting() {
return true;
}
public void sendMulticast(AsciiString cluster_name, byte[] data, int offset, int length)
throws Exception {}
public void sendUnicast(PhysicalAddress dest, byte[] data, int offset, int length)
throws Exception {}
public String getInfo() {
return null;
}
protected PhysicalAddress getPhysicalAddress() {
return null;
}
public Object down(Event evt) {
switch (evt.getType()) {
case Event.MSG:
Message msg = (Message) evt.getArg();
NakAckHeader2 hdr = (NakAckHeader2) msg.getHeader(ID);
if (hdr == null) break;
if (hdr.getType() == NakAckHeader2.XMIT_REQ) {
SeqnoList seqnos = (SeqnoList) msg.getObject();
System.out.println("-- XMIT-REQ: request retransmission for " + seqnos);
for (Long seqno : seqnos) xmit_requests.add(seqno);
}
break;
}
return null;
}
}
protected static class MockProtocol extends Protocol {
protected final List<Long> msgs = new LinkedList<>();
public List<Long> getMsgs() {
return msgs;
}
public void clear() {
msgs.clear();
}
public Object up(Event evt) {
switch (evt.getType()) {
case Event.MSG:
Message msg = (Message) evt.getArg();
NakAckHeader2 hdr = (NakAckHeader2) msg.getHeader(ID);
if (hdr != null && hdr.getType() == NakAckHeader2.MSG) {
long seqno = hdr.getSeqno();
msgs.add(seqno);
System.out.println("-- received message #" + seqno + " from " + msg.getSrc());
}
break;
}
return null;
}
public void up(MessageBatch batch) {
for (Message msg : batch) {
NakAckHeader2 hdr = (NakAckHeader2) msg.getHeader(ID);
if (hdr != null && hdr.getType() == NakAckHeader2.MSG) {
long seqno = hdr.getSeqno();
msgs.add(seqno);
System.out.println("-- received message #" + seqno + " from " + msg.getSrc());
}
}
}
}
}
@ManagedAttribute(name = "view")
public String getViewAsString() {
View v = getView();
return v != null ? v.toString() : "n/a";
}
/**
* Callback method <br>
* Called by the ProtocolStack when a message is received.
*
* @param evt the event carrying the message from the protocol stack
*/
public Object up(Event evt) {
switch (evt.getType()) {
case Event.MSG:
Message msg = (Message) evt.getArg();
if (stats) {
received_msgs++;
received_bytes += msg.getLength();
}
// discard local messages (sent by myself to me)
if (discard_own_messages
&& local_addr != null
&& msg.getSrc() != null
&& local_addr.equals(msg.getSrc())) return null;
break;
case Event.VIEW_CHANGE:
View tmp = (View) evt.getArg();
if (tmp instanceof MergeView) my_view = new View(tmp.getViewId(), tmp.getMembers());
else my_view = tmp;
// Bela&Vladimir Oct 27th,2006 (JGroups 2.4): we need to set connected=true because a client
// can
// call channel.getView() in viewAccepted() callback invoked on this thread (see
// Event.VIEW_CHANGE handling below)
// not good: we are only connected when we returned from connect() - bela June 22 2007
// Changed: when a channel gets a view of which it is a member then it should be
// connected even if connect() hasn't returned yet ! (bela Noc 2010)
if (state != State.CONNECTED) state = State.CONNECTED;
break;
case Event.CONFIG:
Map<String, Object> cfg = (Map<String, Object>) evt.getArg();
if (cfg != null) {
if (cfg.containsKey("state_transfer")) {
state_transfer_supported = (Boolean) cfg.get("state_transfer");
}
if (cfg.containsKey("flush_supported")) {
flush_supported = (Boolean) cfg.get("flush_supported");
}
}
break;
case Event.GET_STATE_OK:
StateTransferResult result = (StateTransferResult) evt.getArg();
if (up_handler != null) {
try {
Object retval = up_handler.up(evt);
state_promise.setResult(new StateTransferResult());
return retval;
} catch (Throwable t) {
state_promise.setResult(new StateTransferResult(t));
}
}
if (receiver != null) {
try {
if (result.hasBuffer()) {
byte[] tmp_state = result.getBuffer();
ByteArrayInputStream input = new ByteArrayInputStream(tmp_state);
receiver.setState(input);
}
state_promise.setResult(result);
} catch (Throwable t) {
state_promise.setResult(new StateTransferResult(t));
}
}
break;
case Event.STATE_TRANSFER_INPUTSTREAM_CLOSED:
state_promise.setResult((StateTransferResult) evt.getArg());
break;
case Event.STATE_TRANSFER_INPUTSTREAM:
// Oct 13,2006 moved to down() when Event.STATE_TRANSFER_INPUTSTREAM_CLOSED is received
// state_promise.setResult(is != null? Boolean.TRUE : Boolean.FALSE);
if (up_handler != null) return up_handler.up(evt);
InputStream is = (InputStream) evt.getArg();
if (is != null && receiver != null) {
try {
receiver.setState(is);
} catch (Throwable t) {
throw new RuntimeException("failed calling setState() in state requester", t);
}
}
break;
case Event.STATE_TRANSFER_OUTPUTSTREAM:
if (receiver != null && evt.getArg() != null) {
try {
receiver.getState((OutputStream) evt.getArg());
} catch (Exception e) {
throw new RuntimeException("failed calling getState() in state provider", e);
}
}
break;
case Event.GET_LOCAL_ADDRESS:
return local_addr;
default:
break;
}
// If UpHandler is installed, pass all events to it and return (UpHandler is e.g. a building
// block)
if (up_handler != null) return up_handler.up(evt);
if (receiver != null) return invokeCallback(evt.getType(), evt.getArg());
return null;
}
@ManagedAttribute
public String getView() {
return view != null ? view.getViewId().toString() : "null";
}
/**
* An event was received from the layer below. Usually the current layer will want to examine the
* event type and - depending on its type - perform some computation (e.g. removing headers from a
* MSG event type, or updating the internal membership list when receiving a VIEW_CHANGE event).
* Finally the event is either a) discarded, or b) an event is sent down the stack using <code>
* PassDown</code> or c) the event (or another event) is sent up the stack using <code>PassUp
* </code>.
*
* <p>For the PING protocol, the Up operation does the following things. 1. If the event is a
* Event.MSG then PING will inspect the message header. If the header is null, PING simply passes
* up the event If the header is PingHeader.GET_MBRS_REQ then the PING protocol will PassDown a
* PingRequest message If the header is PingHeader.GET_MBRS_RSP we will add the message to the
* initial members vector and wake up any waiting threads. 2. If the event is Event.SET_LOCAL_ADDR
* we will simple set the local address of this protocol 3. For all other messages we simple pass
* it up to the protocol above
*
* @param evt - the event that has been sent from the layer below
*/
@SuppressWarnings("unchecked")
public Object up(Event evt) {
switch (evt.getType()) {
case Event.MSG:
Message msg = (Message) evt.getArg();
PingHeader hdr = (PingHeader) msg.getHeader(this.id);
if (hdr == null) return up_prot.up(evt);
PingData data = hdr.data;
Address logical_addr = data != null ? data.getAddress() : null;
if (is_leaving) return null; // prevents merging back a leaving member
// (https://issues.jboss.org/browse/JGRP-1336)
switch (hdr.type) {
case PingHeader.GET_MBRS_REQ: // return Rsp(local_addr, coord)
if (group_addr == null || hdr.cluster_name == null) {
if (log.isWarnEnabled())
log.warn(
"group_addr ("
+ group_addr
+ ") or cluster_name of header ("
+ hdr.cluster_name
+ ") is null; passing up discovery request from "
+ msg.getSrc()
+ ", but this should not"
+ " be the case");
} else {
if (!group_addr.equals(hdr.cluster_name)) {
if (log.isWarnEnabled())
log.warn(
"discarding discovery request for cluster '"
+ hdr.cluster_name
+ "' from "
+ msg.getSrc()
+ "; our cluster name is '"
+ group_addr
+ "'. "
+ "Please separate your clusters cleanly.");
return null;
}
}
// add physical address and logical name of the discovery sender (if available) to the
// cache
if (data != null) {
if (logical_addr == null) logical_addr = msg.getSrc();
Collection<PhysicalAddress> physical_addrs = data.getPhysicalAddrs();
PhysicalAddress physical_addr =
physical_addrs != null && !physical_addrs.isEmpty()
? physical_addrs.iterator().next()
: null;
if (logical_addr != null && physical_addr != null)
down(
new Event(
Event.SET_PHYSICAL_ADDRESS,
new Tuple<Address, PhysicalAddress>(logical_addr, physical_addr)));
if (logical_addr != null && data.getLogicalName() != null)
UUID.add(logical_addr, data.getLogicalName());
discoveryRequestReceived(msg.getSrc(), data.getLogicalName(), physical_addrs);
synchronized (ping_responses) {
for (Responses response : ping_responses) {
response.addResponse(data, false);
}
}
}
if (hdr.view_id != null) {
// If the discovery request is merge-triggered, and the ViewId shipped with it
// is the same as ours, we don't respond (JGRP-1315).
ViewId my_view_id = view != null ? view.getViewId() : null;
if (my_view_id != null && my_view_id.equals(hdr.view_id)) return null;
boolean send_discovery_rsp =
force_sending_discovery_rsps
|| is_coord
|| current_coord == null
|| current_coord.equals(msg.getSrc());
if (!send_discovery_rsp) {
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": suppressing merge response as I'm not a coordinator and the "
+ "discovery request was not sent by a coordinator");
return null;
}
}
if (isMergeRunning()) {
if (log.isTraceEnabled())
log.trace(
local_addr + ": suppressing merge response as a merge is already in progress");
return null;
}
List<PhysicalAddress> physical_addrs =
hdr.view_id != null
? null
: Arrays.asList(
(PhysicalAddress) down(new Event(Event.GET_PHYSICAL_ADDRESS, local_addr)));
sendDiscoveryResponse(
local_addr,
physical_addrs,
is_server,
hdr.view_id != null,
UUID.get(local_addr),
msg.getSrc());
return null;
case PingHeader.GET_MBRS_RSP: // add response to vector and notify waiting thread
// add physical address (if available) to transport's cache
if (data != null) {
Address response_sender = msg.getSrc();
if (logical_addr == null) logical_addr = msg.getSrc();
Collection<PhysicalAddress> addrs = data.getPhysicalAddrs();
PhysicalAddress physical_addr =
addrs != null && !addrs.isEmpty() ? addrs.iterator().next() : null;
if (logical_addr != null && physical_addr != null)
down(
new Event(
Event.SET_PHYSICAL_ADDRESS,
new Tuple<Address, PhysicalAddress>(logical_addr, physical_addr)));
if (logical_addr != null && data.getLogicalName() != null)
UUID.add(logical_addr, data.getLogicalName());
if (log.isTraceEnabled())
log.trace(
local_addr + ": received GET_MBRS_RSP from " + response_sender + ": " + data);
boolean overwrite = logical_addr != null && logical_addr.equals(response_sender);
synchronized (ping_responses) {
for (Responses response : ping_responses) {
response.addResponse(data, overwrite);
}
}
}
return null;
default:
if (log.isWarnEnabled())
log.warn("got PING header with unknown type (" + hdr.type + ')');
return null;
}
case Event.GET_PHYSICAL_ADDRESS:
try {
sendDiscoveryRequest(group_addr, null, null);
} catch (InterruptedIOException ie) {
if (log.isWarnEnabled()) {
log.warn("Discovery request for cluster " + group_addr + " interrupted");
}
Thread.currentThread().interrupt();
} catch (Exception ex) {
if (log.isErrorEnabled()) log.error("failed sending discovery request", ex);
}
return null;
case Event.FIND_INITIAL_MBRS: // sent by transport
return findInitialMembers(null);
}
return up_prot.up(evt);
}
public Object up(Event evt) {
Message msg;
SequencerHeader hdr;
switch (evt.getType()) {
case Event.MSG:
msg = (Message) evt.getArg();
if (msg.isFlagSet(Message.Flag.NO_TOTAL_ORDER) || msg.isFlagSet(Message.Flag.OOB)) break;
hdr = (SequencerHeader) msg.getHeader(this.id);
if (hdr == null) break; // pass up
switch (hdr.type) {
case SequencerHeader.FORWARD:
case SequencerHeader.FLUSH:
if (!is_coord) {
if (log.isErrorEnabled())
log.error(
local_addr + ": non-coord; dropping FORWARD request from " + msg.getSrc());
return null;
}
Address sender = msg.getSrc();
if (view != null && !view.containsMember(sender)) {
if (log.isErrorEnabled())
log.error(
local_addr
+ ": dropping FORWARD request from non-member "
+ sender
+ "; view="
+ view);
return null;
}
broadcast(
msg,
true,
msg.getSrc(),
hdr.seqno,
hdr.type == SequencerHeader.FLUSH); // do copy the message
received_forwards++;
break;
case SequencerHeader.BCAST:
deliver(msg, evt, hdr);
received_bcasts++;
break;
case SequencerHeader.WRAPPED_BCAST:
unwrapAndDeliver(
msg, hdr.flush_ack); // unwrap the original message (in the payload) and deliver it
received_bcasts++;
break;
}
return null;
case Event.VIEW_CHANGE:
Object retval = up_prot.up(evt);
handleViewChange((View) evt.getArg());
return retval;
case Event.TMP_VIEW:
handleTmpView((View) evt.getArg());
break;
}
return up_prot.up(evt);
}
public Object down(Event evt) {
switch (evt.getType()) {
case Event.MSG: // Add UnicastHeader, add to AckSenderWindow and pass down
Message msg = (Message) evt.getArg();
Address dst = msg.getDest();
/* only handle unicast messages */
if (dst == null || msg.isFlagSet(Message.NO_RELIABILITY)) break;
if (!running) {
if (log.isTraceEnabled())
log.trace("discarded message as start() has not yet been called, message: " + msg);
return null;
}
SenderEntry entry = send_table.get(dst);
if (entry == null) {
entry = new SenderEntry(getNewConnectionId());
SenderEntry existing = send_table.putIfAbsent(dst, entry);
if (existing != null) entry = existing;
else {
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": created connection to "
+ dst
+ " (conn_id="
+ entry.send_conn_id
+ ")");
if (cache != null && !members.contains(dst)) cache.add(dst);
}
}
short send_conn_id = entry.send_conn_id;
long seqno = entry.sent_msgs_seqno.getAndIncrement();
long sleep = 10;
while (running) {
try {
msg.putHeader(
this.id,
Unicast2Header.createDataHeader(seqno, send_conn_id, seqno == DEFAULT_FIRST_SEQNO));
entry.sent_msgs.add(seqno, msg); // add *including* UnicastHeader, adds to retransmitter
if (conn_expiry_timeout > 0) entry.update();
break;
} catch (Throwable t) {
if (!running) break;
if (log.isWarnEnabled()) log.warn("failed sending message", t);
Util.sleep(sleep);
sleep = Math.min(5000, sleep * 2);
}
}
if (log.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append(local_addr)
.append(" --> DATA(")
.append(dst)
.append(": #")
.append(seqno)
.append(", conn_id=")
.append(send_conn_id);
if (seqno == DEFAULT_FIRST_SEQNO) sb.append(", first");
sb.append(')');
log.trace(sb);
}
try {
down_prot.down(evt);
num_msgs_sent++;
} catch (Throwable t) {
log.warn("failed sending the message", t);
}
return null; // we already passed the msg down
case Event.VIEW_CHANGE: // remove connections to peers that are not members anymore !
View view = (View) evt.getArg();
List<Address> new_members = view.getMembers();
Set<Address> non_members = new HashSet<Address>(send_table.keySet());
non_members.addAll(recv_table.keySet());
members = new_members;
non_members.removeAll(new_members);
if (cache != null) cache.removeAll(new_members);
if (!non_members.isEmpty()) {
if (log.isTraceEnabled()) log.trace("removing non members " + non_members);
for (Address non_mbr : non_members) removeConnection(non_mbr);
}
break;
case Event.SET_LOCAL_ADDRESS:
local_addr = (Address) evt.getArg();
break;
}
return down_prot.down(evt); // Pass on to the layer below us
}
public void viewAccepted(View new_view) {
System.out.println("** view: " + new_view);
members.clear();
members.addAll(new_view.getMembers());
}
public ViewId getViewId() {
return view != null ? view.getViewId() : null;
}