/**
* Invokes a method in all members and expects responses from members contained in dests (or all
* members if dests is null).
*
* @param dests A list of addresses. If null, we'll wait for responses from all cluster members
* @param method_call The method (plus args) to be invoked
* @param options A collection of call options, e.g. sync versus async, timeout etc
* @param listener A FutureListener which will be registered (if non null) with the future
* <em>before</em> the call is invoked
* @return NotifyingFuture A future from which the results can be fetched
* @throws Exception If the sending of the message threw an exception. Note that <em>no</em>
* exception will be thrown if any of the target members threw an exception; such an exception
* will be in the Rsp element for the particular member in the RspList
*/
public <T> NotifyingFuture<RspList<T>> callRemoteMethodsWithFuture(
Collection<Address> dests,
MethodCall method_call,
RequestOptions options,
FutureListener<RspList<T>> listener)
throws Exception {
if (dests != null && dests.isEmpty()) { // don't send if dest list is empty
if (log.isTraceEnabled())
log.trace(
"destination list of %s() is empty: no need to send message", method_call.getName());
return new NullFuture<>(new RspList());
}
if (log.isTraceEnabled())
log.trace("dests=%s, method_call=%s, options=%s", dests, method_call, options);
Buffer buf =
req_marshaller != null
? req_marshaller.objectToBuffer(method_call)
: Util.objectToBuffer(method_call);
Message msg = new Message().setBuffer(buf);
NotifyingFuture<RspList<T>> retval = super.castMessageWithFuture(dests, msg, options, listener);
if (log.isTraceEnabled()) log.trace("responses: %s", retval);
return retval;
}
private static boolean containsCoordinatorResponse(Collection<PingData> rsps) {
if (rsps == null || rsps.isEmpty()) return false;
for (PingData rsp : rsps) {
if (rsp.isCoord()) return true;
}
return false;
}
public void sendDiscoveryRequest(String cluster_name, Promise promise, ViewId view_id)
throws Exception {
PingData data = null;
PhysicalAddress physical_addr =
(PhysicalAddress) down(new Event(Event.GET_PHYSICAL_ADDRESS, local_addr));
if (view_id == null) {
List<PhysicalAddress> physical_addrs = Arrays.asList(physical_addr);
data = new PingData(local_addr, null, false, UUID.get(local_addr), physical_addrs);
}
PingHeader hdr = new PingHeader(PingHeader.GET_MBRS_REQ, data, cluster_name);
hdr.view_id = view_id;
Collection<PhysicalAddress> cluster_members = fetchClusterMembers(cluster_name);
if (cluster_members == null) {
Message msg = new Message(null); // multicast msg
msg.setFlag(Message.OOB);
msg.putHeader(getId(), hdr);
sendMcastDiscoveryRequest(msg);
} else {
if (cluster_members.isEmpty()) { // if we don't find any members, return immediately
if (promise != null) promise.setResult(null);
} else {
for (final Address addr : cluster_members) {
if (addr.equals(physical_addr)) // no need to send the request to myself
continue;
final Message msg = new Message(addr, null, null);
msg.setFlag(Message.OOB);
msg.putHeader(this.id, hdr);
if (log.isTraceEnabled())
log.trace("[FIND_INITIAL_MBRS] sending discovery request to " + msg.getDest());
if (!sendDiscoveryRequestsInParallel()) {
down_prot.down(new Event(Event.MSG, msg));
} else {
timer.execute(
new Runnable() {
public void run() {
try {
down_prot.down(new Event(Event.MSG, msg));
} catch (Exception ex) {
if (log.isErrorEnabled())
log.error("failed sending discovery request to " + addr + ": " + ex);
}
}
});
}
}
}
}
}
/**
* Invokes a method in all members contained in dests (or all members if dests is null).
*
* @param dests A list of addresses. If null, the method will be invoked on all cluster members
* @param method_call The method (plus args) to be invoked
* @param options A collection of call options, e.g. sync versus async, timeout etc
* @return RspList A list of return values and flags (suspected, not received) per member
* @since 2.9
*/
public RspList callRemoteMethods(
Collection<Address> dests, MethodCall method_call, RequestOptions options) {
if (dests != null && dests.isEmpty()) { // don't send if dest list is empty
if (log.isTraceEnabled())
log.trace(
new StringBuilder("destination list of ")
.append(method_call.getName())
.append("() is empty: no need to send message"));
return RspList.EMPTY_RSP_LIST;
}
if (log.isTraceEnabled())
log.trace(
new StringBuilder("dests=")
.append(dests)
.append(", method_call=")
.append(method_call)
.append(", options=")
.append(options));
Object buf;
try {
buf =
req_marshaller != null
? req_marshaller.objectToBuffer(method_call)
: Util.objectToByteBuffer(method_call);
} catch (Exception e) {
// if(log.isErrorEnabled()) log.error("exception", e);
// we will change this in 3.0 to add the exception to the signature
// (see http://jira.jboss.com/jira/browse/JGRP-193). The reason for a RTE is that we cannot
// change the
// signature in 2.3, otherwise 2.3 would be *not* API compatible to prev releases
throw new RuntimeException("failure to marshal argument(s)", e);
}
Message msg = new Message();
if (buf instanceof Buffer) msg.setBuffer((Buffer) buf);
else msg.setBuffer((byte[]) buf);
msg.setFlag(options.getFlags());
if (options.getScope() > 0) msg.setScope(options.getScope());
RspList retval = super.castMessage(dests, msg, options);
if (log.isTraceEnabled()) log.trace("responses: " + retval);
return retval;
}
/**
* Invoked upon receiving a MERGE event from the MERGE layer. Starts the merge protocol. See
* description of protocol in DESIGN.
*
* @param views A List of <em>different</em> views detected by the merge protocol
*/
public void merge(Map<Address, View> views) {
if (isMergeInProgress()) {
if (log.isTraceEnabled())
log.trace(gms.local_addr + ": merge is already running (merge_id=" + merge_id + ")");
return;
}
// we need the merge *coordinators* not merge participants because not everyone can lead a merge
// !
Collection<Address> coords = Util.determineActualMergeCoords(views);
if (coords.isEmpty()) {
log.error(
gms.local_addr
+ ": unable to determine merge leader from "
+ views
+ "; not starting a merge");
return;
}
Membership tmp =
new Membership(coords); // establish a deterministic order, so that coords can elect leader
tmp.sort();
Address merge_leader = tmp.elementAt(0);
if (merge_leader.equals(gms.local_addr)) {
if (log.isDebugEnabled()) {
Collection<Address> merge_participants = Util.determineMergeParticipants(views);
log.debug(
"I ("
+ gms.local_addr
+ ") will be the leader. Starting the merge task for "
+ merge_participants.size()
+ " coords");
}
merge_task.start(views);
} else {
if (log.isTraceEnabled())
log.trace(
"I ("
+ gms.local_addr
+ ") am not the merge leader, "
+ "waiting for merge leader ("
+ merge_leader
+ ") to initiate merge");
}
}
/**
* An event was received from the layer below. Usually the current layer will want to examine the
* event type and - depending on its type - perform some computation (e.g. removing headers from a
* MSG event type, or updating the internal membership list when receiving a VIEW_CHANGE event).
* Finally the event is either a) discarded, or b) an event is sent down the stack using <code>
* PassDown</code> or c) the event (or another event) is sent up the stack using <code>PassUp
* </code>.
*
* <p>For the PING protocol, the Up operation does the following things. 1. If the event is a
* Event.MSG then PING will inspect the message header. If the header is null, PING simply passes
* up the event If the header is PingHeader.GET_MBRS_REQ then the PING protocol will PassDown a
* PingRequest message If the header is PingHeader.GET_MBRS_RSP we will add the message to the
* initial members vector and wake up any waiting threads. 2. If the event is Event.SET_LOCAL_ADDR
* we will simple set the local address of this protocol 3. For all other messages we simple pass
* it up to the protocol above
*
* @param evt - the event that has been sent from the layer below
*/
@SuppressWarnings("unchecked")
public Object up(Event evt) {
switch (evt.getType()) {
case Event.MSG:
Message msg = (Message) evt.getArg();
PingHeader hdr = (PingHeader) msg.getHeader(this.id);
if (hdr == null) return up_prot.up(evt);
PingData data = hdr.data;
Address logical_addr = data != null ? data.getAddress() : null;
if (is_leaving) return null; // prevents merging back a leaving member
// (https://issues.jboss.org/browse/JGRP-1336)
switch (hdr.type) {
case PingHeader.GET_MBRS_REQ: // return Rsp(local_addr, coord)
if (group_addr == null || hdr.cluster_name == null) {
if (log.isWarnEnabled())
log.warn(
"group_addr ("
+ group_addr
+ ") or cluster_name of header ("
+ hdr.cluster_name
+ ") is null; passing up discovery request from "
+ msg.getSrc()
+ ", but this should not"
+ " be the case");
} else {
if (!group_addr.equals(hdr.cluster_name)) {
if (log.isWarnEnabled())
log.warn(
"discarding discovery request for cluster '"
+ hdr.cluster_name
+ "' from "
+ msg.getSrc()
+ "; our cluster name is '"
+ group_addr
+ "'. "
+ "Please separate your clusters cleanly.");
return null;
}
}
// add physical address and logical name of the discovery sender (if available) to the
// cache
if (data != null) {
if (logical_addr == null) logical_addr = msg.getSrc();
Collection<PhysicalAddress> physical_addrs = data.getPhysicalAddrs();
PhysicalAddress physical_addr =
physical_addrs != null && !physical_addrs.isEmpty()
? physical_addrs.iterator().next()
: null;
if (logical_addr != null && physical_addr != null)
down(
new Event(
Event.SET_PHYSICAL_ADDRESS,
new Tuple<Address, PhysicalAddress>(logical_addr, physical_addr)));
if (logical_addr != null && data.getLogicalName() != null)
UUID.add(logical_addr, data.getLogicalName());
discoveryRequestReceived(msg.getSrc(), data.getLogicalName(), physical_addrs);
synchronized (ping_responses) {
for (Responses response : ping_responses) {
response.addResponse(data, false);
}
}
}
if (hdr.view_id != null) {
// If the discovery request is merge-triggered, and the ViewId shipped with it
// is the same as ours, we don't respond (JGRP-1315).
ViewId my_view_id = view != null ? view.getViewId() : null;
if (my_view_id != null && my_view_id.equals(hdr.view_id)) return null;
boolean send_discovery_rsp =
force_sending_discovery_rsps
|| is_coord
|| current_coord == null
|| current_coord.equals(msg.getSrc());
if (!send_discovery_rsp) {
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": suppressing merge response as I'm not a coordinator and the "
+ "discovery request was not sent by a coordinator");
return null;
}
}
if (isMergeRunning()) {
if (log.isTraceEnabled())
log.trace(
local_addr + ": suppressing merge response as a merge is already in progress");
return null;
}
List<PhysicalAddress> physical_addrs =
hdr.view_id != null
? null
: Arrays.asList(
(PhysicalAddress) down(new Event(Event.GET_PHYSICAL_ADDRESS, local_addr)));
sendDiscoveryResponse(
local_addr,
physical_addrs,
is_server,
hdr.view_id != null,
UUID.get(local_addr),
msg.getSrc());
return null;
case PingHeader.GET_MBRS_RSP: // add response to vector and notify waiting thread
// add physical address (if available) to transport's cache
if (data != null) {
Address response_sender = msg.getSrc();
if (logical_addr == null) logical_addr = msg.getSrc();
Collection<PhysicalAddress> addrs = data.getPhysicalAddrs();
PhysicalAddress physical_addr =
addrs != null && !addrs.isEmpty() ? addrs.iterator().next() : null;
if (logical_addr != null && physical_addr != null)
down(
new Event(
Event.SET_PHYSICAL_ADDRESS,
new Tuple<Address, PhysicalAddress>(logical_addr, physical_addr)));
if (logical_addr != null && data.getLogicalName() != null)
UUID.add(logical_addr, data.getLogicalName());
if (log.isTraceEnabled())
log.trace(
local_addr + ": received GET_MBRS_RSP from " + response_sender + ": " + data);
boolean overwrite = logical_addr != null && logical_addr.equals(response_sender);
synchronized (ping_responses) {
for (Responses response : ping_responses) {
response.addResponse(data, overwrite);
}
}
}
return null;
default:
if (log.isWarnEnabled())
log.warn("got PING header with unknown type (" + hdr.type + ')');
return null;
}
case Event.GET_PHYSICAL_ADDRESS:
try {
sendDiscoveryRequest(group_addr, null, null);
} catch (InterruptedIOException ie) {
if (log.isWarnEnabled()) {
log.warn("Discovery request for cluster " + group_addr + " interrupted");
}
Thread.currentThread().interrupt();
} catch (Exception ex) {
if (log.isErrorEnabled()) log.error("failed sending discovery request", ex);
}
return null;
case Event.FIND_INITIAL_MBRS: // sent by transport
return findInitialMembers(null);
}
return up_prot.up(evt);
}
/**
* Broadcasts the new view and digest, and waits for acks from all members in the list given as
* argument. If the list is null, we take the members who are part of new_view
*/
public void castViewChange(
View new_view, Digest digest, JoinRsp jr, Collection<Address> newMembers) {
if (log.isTraceEnabled())
log.trace(local_addr + ": mcasting view " + new_view + " (" + new_view.size() + " mbrs)\n");
// Send down a local TMP_VIEW event. This is needed by certain layers (e.g. NAKACK) to compute
// correct digest
// in case client's next request (e.g. getState()) reaches us *before* our own view change
// multicast.
// Check NAKACK's TMP_VIEW handling for details
down_prot.up(new Event(Event.TMP_VIEW, new_view));
down_prot.down(new Event(Event.TMP_VIEW, new_view));
List<Address> ackMembers = new ArrayList<Address>(new_view.getMembers());
if (newMembers != null && !newMembers.isEmpty()) ackMembers.removeAll(newMembers);
Message view_change_msg = new Message(); // bcast to all members
GmsHeader hdr = new GmsHeader(GmsHeader.VIEW, new_view);
hdr.my_digest = digest;
view_change_msg.putHeader(this.id, hdr);
// If we're the only member the VIEW is broadcast to, let's simply install the view directly,
// without
// sending the VIEW multicast ! Or else N-1 members drop the multicast anyway...
if (local_addr != null && ackMembers.size() == 1 && ackMembers.get(0).equals(local_addr)) {
// we need to add the message to the retransmit window (e.g. in NAKACK), so (1) it can be
// retransmitted and
// (2) we increment the seqno (otherwise, we'd return an incorrect digest)
down_prot.down(new Event(Event.ADD_TO_XMIT_TABLE, view_change_msg));
impl.handleViewChange(new_view, digest);
} else {
if (!ackMembers.isEmpty()) ack_collector.reset(ackMembers);
down_prot.down(new Event(Event.MSG, view_change_msg));
try {
if (!ackMembers.isEmpty()) {
ack_collector.waitForAllAcks(view_ack_collection_timeout);
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": received all "
+ ack_collector.expectedAcks()
+ " ACKs from members for view "
+ new_view.getVid());
}
} catch (TimeoutException e) {
if (log_collect_msgs && log.isWarnEnabled()) {
log.warn(
local_addr
+ ": failed to collect all ACKs (expected="
+ ack_collector.expectedAcks()
+ ") for view "
+ new_view.getViewId()
+ " after "
+ view_ack_collection_timeout
+ "ms, missing ACKs from "
+ ack_collector.printMissing());
}
}
}
if (jr != null && (newMembers != null && !newMembers.isEmpty())) {
ack_collector.reset(new ArrayList<Address>(newMembers));
for (Address joiner : newMembers) {
sendJoinResponse(jr, joiner);
}
try {
ack_collector.waitForAllAcks(view_ack_collection_timeout);
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": received all ACKs ("
+ ack_collector.expectedAcks()
+ ") from joiners for view "
+ new_view.getVid());
} catch (TimeoutException e) {
if (log_collect_msgs && log.isWarnEnabled()) {
log.warn(
local_addr
+ ": failed to collect all ACKs (expected="
+ ack_collector.expectedAcks()
+ ") for unicast view "
+ new_view
+ " after "
+ view_ack_collection_timeout
+ "ms, missing ACKs from "
+ ack_collector.printMissing());
}
}
}
}
public void handleMembershipChange(Collection<Request> requests) {
boolean joinAndStateTransferInitiated = false;
boolean useFlushIfPresent = gms.use_flush_if_present;
Collection<Address> new_mbrs = new LinkedHashSet<>(requests.size());
Collection<Address> suspected_mbrs = new LinkedHashSet<>(requests.size());
Collection<Address> leaving_mbrs = new LinkedHashSet<>(requests.size());
boolean self_leaving = false; // is the coord leaving
for (Request req : requests) {
switch (req.type) {
case Request.JOIN:
new_mbrs.add(req.mbr);
if (req.useFlushIfPresent) useFlushIfPresent = true;
break;
case Request.JOIN_WITH_STATE_TRANSFER:
new_mbrs.add(req.mbr);
joinAndStateTransferInitiated = true;
if (req.useFlushIfPresent) useFlushIfPresent = true;
break;
case Request.LEAVE:
if (req.suspected) suspected_mbrs.add(req.mbr);
else {
leaving_mbrs.add(req.mbr);
if (gms.local_addr != null && gms.local_addr.equals(req.mbr)) self_leaving = true;
}
break;
case Request.SUSPECT:
suspected_mbrs.add(req.mbr);
break;
}
}
new_mbrs.remove(gms.local_addr); // remove myself - cannot join myself (already joined)
if (gms.getViewId() == null) {
// we're probably not the coord anymore (we just left ourselves), let someone else do it
// (client will retry when it doesn't get a response)
log.debug(
"gms.view_id is null, I'm not the coordinator anymore (leaving=%b); "
+ "the new coordinator will handle the leave request",
self_leaving);
return;
}
List<Address> current_members = gms.members.getMembers();
leaving_mbrs.retainAll(
current_members); // remove all elements of leaving_mbrs which are not current members
if (suspected_mbrs.remove(gms.local_addr))
log.warn("I am the coord and I'm being suspected -- will probably leave shortly");
suspected_mbrs.retainAll(
current_members); // remove all elements of suspected_mbrs which are not current members
// for the members that have already joined, return the current digest and membership
for (Iterator<Address> it = new_mbrs.iterator(); it.hasNext(); ) {
Address mbr = it.next();
if (gms.members.contains(mbr)) { // already joined: return current digest and membership
log.trace(
"%s: %s already present; returning existing view %s", gms.local_addr, mbr, gms.view);
Tuple<View, Digest> tuple = gms.getViewAndDigest();
if (tuple != null) gms.sendJoinResponse(new JoinRsp(tuple.getVal1(), tuple.getVal2()), mbr);
else
log.warn(
"%s: did not find a digest matching view %s; dropping JOIN-RSP",
gms.local_addr, gms.view);
it
.remove(); // remove it anyway, even if we didn't find a digest matching the view
// (joiner will retry)
}
}
if (new_mbrs.isEmpty() && leaving_mbrs.isEmpty() && suspected_mbrs.isEmpty()) {
log.trace("%s: found no members to add or remove, will not create new view", gms.local_addr);
return;
}
View new_view = gms.getNextView(new_mbrs, leaving_mbrs, suspected_mbrs);
if (new_view.size() == 0
&& gms.local_addr != null
&& gms.local_addr.equals(new_view.getCreator())) {
if (self_leaving) gms.initState(); // in case connect() is called again
return;
}
log.trace(
"%s: joiners=%s, suspected=%s, leaving=%s, new view: %s",
gms.local_addr, new_mbrs, suspected_mbrs, leaving_mbrs, new_view);
JoinRsp join_rsp = null;
boolean hasJoiningMembers = !new_mbrs.isEmpty();
try {
boolean successfulFlush =
!useFlushIfPresent || !gms.flushProtocolInStack || gms.startFlush(new_view);
if (!successfulFlush && hasJoiningMembers) {
// Don't send a join response if the flush fails
// (http://jira.jboss.org/jira/browse/JGRP-759)
// The joiner should block until the previous FLUSH completed
sendLeaveResponses(leaving_mbrs); // we still have to send potential leave responses
// but let the joining client timeout and send another join request
return;
}
// we cannot garbage collect during joining a new member *if* we're the only member
// Example: {A}, B joins, after returning JoinRsp to B, A garbage collects messages higher
// than those
// in the digest returned to the client, so the client will *not* be able to ask for
// retransmission
// of those messages if he misses them
if (hasJoiningMembers) {
gms.getDownProtocol().down(new Event(Event.SUSPEND_STABLE, MAX_SUSPEND_TIMEOUT));
// create a new digest, which contains the new members, minus left members
MutableDigest join_digest =
new MutableDigest(new_view.getMembersRaw()).set(gms.getDigest());
for (Address member : new_mbrs)
join_digest.set(member, 0, 0); // ... and set the new members. their first seqno will be 1
// If the digest from NAKACK doesn't include all members of the view, we try once more; if
// it is still
// incomplete, we don't send a JoinRsp back to the joiner(s). This shouldn't be a problem as
// they will retry
if (join_digest.allSet() || join_digest.set(gms.getDigest()).allSet())
join_rsp = new JoinRsp(new_view, join_digest);
else
log.warn(
"%s: digest does not match view (missing seqnos for %s); dropping JOIN-RSP",
gms.local_addr, Arrays.toString(join_digest.getNonSetMembers()));
}
sendLeaveResponses(leaving_mbrs); // no-op if no leaving members
// we don't need to send the digest to existing members:
// https://issues.jboss.org/browse/JGRP-1317
gms.castViewChange(new_view, null, new_mbrs);
gms.sendJoinResponses(join_rsp, new_mbrs);
} finally {
if (hasJoiningMembers) gms.getDownProtocol().down(new Event(Event.RESUME_STABLE));
if (!joinAndStateTransferInitiated && useFlushIfPresent) gms.stopFlush();
if (self_leaving) gms.initState(); // in case connect() is called again
}
}
public static void testSet() {
SeqnoRange range = new SeqnoRange(10, 15);
range.set(11, 12, 13, 14);
System.out.println("range=" + print(range));
assert range.size() == 6;
assert range.getNumberOfReceivedMessages() == 4;
assert range.getNumberOfMissingMessages() == 2;
Collection<Range> xmits = range.getMessagesToRetransmit();
assert xmits.size() == 2;
Iterator<Range> it = xmits.iterator();
Range r = it.next();
assert r.low == 10 && r.high == 10;
r = it.next();
assert r.low == 15 && r.high == 15;
range = new SeqnoRange(10, 15);
range.set(10, 11, 12, 13, 14);
System.out.println("range=" + print(range));
assert range.size() == 6;
assert range.getNumberOfReceivedMessages() == 5;
assert range.getNumberOfMissingMessages() == 1;
xmits = range.getMessagesToRetransmit();
assert xmits.size() == 1;
it = xmits.iterator();
r = it.next();
assert r.low == 15 && r.high == 15;
range = new SeqnoRange(10, 15);
range.set(11, 12, 13, 14, 15);
System.out.println("range=" + print(range));
assert range.size() == 6;
assert range.getNumberOfReceivedMessages() == 5;
assert range.getNumberOfMissingMessages() == 1;
xmits = range.getMessagesToRetransmit();
assert xmits.size() == 1;
it = xmits.iterator();
r = it.next();
assert r.low == 10 && r.high == 10;
range = new SeqnoRange(10, 15);
range.set(10, 11, 12, 13, 14, 15);
System.out.println("range=" + print(range));
assert range.size() == 6;
assert range.getNumberOfReceivedMessages() == 6;
assert range.getNumberOfMissingMessages() == 0;
xmits = range.getMessagesToRetransmit();
assert xmits.isEmpty();
range = new SeqnoRange(10, 15);
range.set(11, 12, 14, 15);
System.out.println("range=" + print(range));
assert range.size() == 6;
assert range.getNumberOfReceivedMessages() == 4;
assert range.getNumberOfMissingMessages() == 2;
xmits = range.getMessagesToRetransmit();
assert xmits.size() == 2;
it = xmits.iterator();
r = it.next();
assert r.low == 10 && r.high == 10;
r = it.next();
assert r.low == 13 && r.high == 13;
range.set(13);
assert range.getNumberOfReceivedMessages() == 5;
assert range.getNumberOfMissingMessages() == 1;
xmits = range.getMessagesToRetransmit();
it = xmits.iterator();
r = it.next();
assert r.low == 10 && r.high == 10;
range.set(10);
System.out.println("range=" + print(range));
assert range.getNumberOfReceivedMessages() == 6;
assert range.getNumberOfMissingMessages() == 0;
xmits = range.getMessagesToRetransmit();
assert xmits.isEmpty();
}