public String toString() {
StringBuilder sb = new StringBuilder();
if (sent_msgs != null) sb.append(sent_msgs).append(", ");
sb.append("send_conn_id=" + send_conn_id);
sb.append(" (" + age() + " ms old)");
return sb.toString();
}
public String toString() {
StringBuilder sb = new StringBuilder();
if (received_msgs != null) sb.append(received_msgs).append(", ");
sb.append("recv_conn_id=" + recv_conn_id);
sb.append(" (" + age() + " ms old)");
return sb.toString();
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(type2Str(type)).append(", seqno=").append(seqno);
if (conn_id != 0) sb.append(", conn_id=").append(conn_id);
if (first) sb.append(", first");
return sb.toString();
}
protected void sendStableMessage(Address dest, short conn_id, long low, long high) {
Message stable_msg = new Message(dest, null, null);
Unicast2Header hdr = Unicast2Header.createStableHeader(conn_id, low, high);
stable_msg.putHeader(this.id, hdr);
stable_msg.setFlag(Message.OOB);
if (log.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append(local_addr)
.append(" --> STABLE(")
.append(dest)
.append(": ")
.append(low)
.append("-")
.append(high)
.append(", conn_id=")
.append(conn_id)
.append(")");
log.trace(sb.toString());
}
down_prot.down(new Event(Event.MSG, stable_msg));
ReceiverEntry entry = recv_table.get(dest);
NakReceiverWindow win = entry != null ? entry.received_msgs : null;
if (win != null) win.stable(win.getHighestDelivered());
}
/**
* We need to resend our first message with our conn_id
*
* @param sender
* @param seqno Resend messages in the range [lowest .. seqno]
*/
private void handleResendingOfFirstMessage(Address sender, long seqno) {
if (log.isTraceEnabled())
log.trace(local_addr + " <-- SEND_FIRST_SEQNO(" + sender + "," + seqno + ")");
SenderEntry entry = send_table.get(sender);
AckSenderWindow win = entry != null ? entry.sent_msgs : null;
if (win == null) {
if (log.isErrorEnabled())
log.error(local_addr + ": sender window for " + sender + " not found");
return;
}
long lowest = win.getLowest();
Message rsp = win.get(lowest);
if (rsp == null) return;
// We need to copy the UnicastHeader and put it back into the message because Message.copy()
// doesn't copy
// the headers and therefore we'd modify the original message in the sender retransmission
// window
// (https://jira.jboss.org/jira/browse/JGRP-965)
Message copy = rsp.copy();
Unicast2Header hdr = (Unicast2Header) copy.getHeader(this.id);
Unicast2Header newhdr = hdr.copy();
newhdr.first = true;
copy.putHeader(this.id, newhdr);
if (log.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append(local_addr)
.append(" --> DATA(")
.append(copy.getDest())
.append(": #")
.append(newhdr.seqno)
.append(", conn_id=")
.append(newhdr.conn_id);
if (newhdr.first) sb.append(", first");
sb.append(')');
log.trace(sb);
}
down_prot.down(new Event(Event.MSG, copy));
if (++lowest > seqno) return;
for (long i = lowest; i <= seqno; i++) {
rsp = win.get(i);
if (rsp != null) down_prot.down(new Event(Event.MSG, rsp));
}
}
@ManagedOperation
public String printConnections() {
StringBuilder sb = new StringBuilder();
if (!send_table.isEmpty()) {
sb.append("send connections:\n");
for (Map.Entry<Address, SenderEntry> entry : send_table.entrySet()) {
sb.append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
}
}
if (!recv_table.isEmpty()) {
sb.append("\nreceive connections:\n");
for (Map.Entry<Address, ReceiverEntry> entry : recv_table.entrySet()) {
sb.append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
}
}
return sb.toString();
}
@ManagedOperation(description = "Prints the contents of the send windows for all members")
public String printSendWindowMessages() {
StringBuilder ret = new StringBuilder(local_addr + ":\n");
for (Map.Entry<Address, SenderEntry> entry : send_table.entrySet()) {
Address addr = entry.getKey();
Table<Message> buf = entry.getValue().sent_msgs;
ret.append(addr).append(": ").append(buf.toString()).append('\n');
}
return ret.toString();
}
@ManagedOperation(description = "Returns the sizes of all NakReceiverWindow.RetransmitTables")
public String printRetransmitTableSizes() {
StringBuilder sb = new StringBuilder();
for (Map.Entry<Address, ReceiverEntry> entry : recv_table.entrySet()) {
NakReceiverWindow win = entry.getValue().received_msgs;
sb.append(entry.getKey() + ": ")
.append(win.getRetransmitTableSize())
.append(" (capacity=" + win.getRetransmitTableCapacity())
.append(", fill factor=" + win.getRetransmitTableFillFactor() + "%)\n");
}
return sb.toString();
}
@ManagedOperation
public String toString(boolean details) {
StringBuilder sb = new StringBuilder();
sb.append("local_addr=")
.append(local_addr)
.append('\n')
.append("cluster_name=")
.append(cluster_name)
.append('\n')
.append("my_view=")
.append(my_view)
.append('\n')
.append("state=")
.append(state)
.append('\n');
if (details) {
sb.append("discard_own_messages=").append(discard_own_messages).append('\n');
sb.append("state_transfer_supported=").append(state_transfer_supported).append('\n');
sb.append("props=").append(getProperties()).append('\n');
}
return sb.toString();
}
private void handleXmitRequest(Address sender, long low, long high) {
if (log.isTraceEnabled())
log.trace(
new StringBuilder()
.append(local_addr)
.append(" <-- XMIT(")
.append(sender)
.append(": #")
.append(low)
.append("-")
.append(high)
.append(')'));
SenderEntry entry = send_table.get(sender);
AckSenderWindow win = entry != null ? entry.sent_msgs : null;
if (win != null) {
for (long i = low; i <= high; i++) {
Message msg = win.get(i);
if (msg == null) {
if (log.isWarnEnabled() && !local_addr.equals(sender)) {
StringBuilder sb = new StringBuilder();
sb.append("(requester=").append(sender).append(", local_addr=").append(this.local_addr);
sb.append(") message ").append(sender).append("::").append(i);
sb.append(" not found in retransmission table of ")
.append(sender)
.append(":\n")
.append(win);
log.warn(sb.toString());
}
continue;
}
down_prot.down(new Event(Event.MSG, msg));
num_xmits++;
}
}
}
protected void handleXmitRequest(Address sender, SeqnoList missing) {
if (log.isTraceEnabled())
log.trace(
new StringBuilder()
.append(local_addr)
.append(" <-- XMIT(")
.append(sender)
.append(": #")
.append(missing)
.append(')'));
SenderEntry entry = send_table.get(sender);
xmit_reqs_received.addAndGet(missing.size());
Table<Message> win = entry != null ? entry.sent_msgs : null;
if (win != null) {
for (long seqno : missing) {
Message msg = win.get(seqno);
if (msg == null) {
if (log.isWarnEnabled() && !local_addr.equals(sender)) {
StringBuilder sb = new StringBuilder();
sb.append("(requester=").append(sender).append(", local_addr=").append(this.local_addr);
sb.append(") message ").append(sender).append("::").append(seqno);
sb.append(" not found in retransmission table of ")
.append(sender)
.append(":\n")
.append(win);
log.warn(sb.toString());
}
continue;
}
down_prot.down(new Event(Event.MSG, msg));
xmit_rsps_sent.incrementAndGet();
}
}
}
protected void sendStableMessage(Address dest, short conn_id, long hd, long hr) {
Message stable_msg = new Message(dest, null, null);
Unicast2Header hdr = Unicast2Header.createStableHeader(conn_id, hd, hr);
stable_msg.putHeader(this.id, hdr);
stable_msg.setFlag(Message.OOB);
if (log.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append(local_addr)
.append(" --> STABLE(")
.append(dest)
.append(": ")
.append(hd)
.append("-")
.append(hr)
.append(", conn_id=")
.append(conn_id)
.append(")");
log.trace(sb.toString());
}
down_prot.down(new Event(Event.MSG, stable_msg));
}
/**
* Check whether the hashtable contains an entry e for <code>sender</code> (create if not). If
* e.received_msgs is null and <code>first</code> is true: create a new AckReceiverWindow(seqno)
* and add message. Set e.received_msgs to the new window. Else just add the message.
*/
protected void handleDataReceived(
Address sender, long seqno, short conn_id, boolean first, Message msg, Event evt) {
if (log.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append(local_addr).append(" <-- DATA(").append(sender).append(": #").append(seqno);
if (conn_id != 0) sb.append(", conn_id=").append(conn_id);
if (first) sb.append(", first");
sb.append(')');
log.trace(sb);
}
ReceiverEntry entry;
NakReceiverWindow win;
recv_table_lock.lock();
try {
entry = recv_table.get(sender);
win = entry != null ? entry.received_msgs : null;
if (first) {
if (entry == null) {
entry = getOrCreateReceiverEntry(sender, seqno, conn_id);
win = entry.received_msgs;
} else { // entry != null && win != null
if (conn_id != entry.recv_conn_id) {
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": conn_id="
+ conn_id
+ " != "
+ entry.recv_conn_id
+ "; resetting receiver window");
ReceiverEntry entry2 = recv_table.remove(sender);
if (entry2 != null) entry2.received_msgs.destroy();
entry = getOrCreateReceiverEntry(sender, seqno, conn_id);
win = entry.received_msgs;
} else {;
}
}
} else { // entry == null && win == null OR entry != null && win == null OR entry != null &&
// win != null
if (win == null || entry.recv_conn_id != conn_id) {
recv_table_lock.unlock();
sendRequestForFirstSeqno(sender, seqno); // drops the message and returns (see below)
return;
}
}
} finally {
if (recv_table_lock.isHeldByCurrentThread()) recv_table_lock.unlock();
}
entry.update();
boolean added = win.add(seqno, msg); // win is guaranteed to be non-null if we get here
num_msgs_received++;
num_bytes_received += msg.getLength();
if (added) {
int len = msg.getLength();
if (len > 0) {
boolean send_stable_msg = false;
entry.lock();
try {
entry.received_bytes += len;
if (entry.received_bytes >= max_bytes) {
entry.received_bytes = 0;
send_stable_msg = true;
}
} finally {
entry.unlock();
}
if (send_stable_msg)
sendStableMessage(
sender, entry.recv_conn_id, win.getHighestDelivered(), win.getHighestReceived());
}
}
// An OOB message is passed up immediately. Later, when remove() is called, we discard it. This
// affects ordering !
// http://jira.jboss.com/jira/browse/JGRP-377
if (msg.isFlagSet(Message.OOB) && added) {
try {
up_prot.up(evt);
} catch (Throwable t) {
log.error("couldn't deliver OOB message " + msg, t);
}
}
final AtomicBoolean processing = win.getProcessing();
if (!processing.compareAndSet(false, true)) {
return;
}
// try to remove (from the AckReceiverWindow) as many messages as possible and pass them up
// Prevents concurrent passing up of messages by different threads
// (http://jira.jboss.com/jira/browse/JGRP-198);
// this is all the more important once we have a concurrent stack
// (http://jira.jboss.com/jira/browse/JGRP-181),
// where lots of threads can come up to this point concurrently, but only 1 is allowed to pass
// at a time
// We *can* deliver messages from *different* senders concurrently, e.g. reception of P1, Q1,
// P2, Q2 can result in
// delivery of P1, Q1, Q2, P2: FIFO (implemented by UNICAST) says messages need to be delivered
// only in the
// order in which they were sent by their senders
boolean released_processing = false;
try {
while (true) {
List<Message> msgs =
win.removeMany(processing, true, max_msg_batch_size); // remove my own messages
if (msgs == null || msgs.isEmpty()) {
released_processing = true;
return;
}
for (Message m : msgs) {
// discard OOB msg: it has already been delivered
// (http://jira.jboss.com/jira/browse/JGRP-377)
if (m.isFlagSet(Message.OOB)) continue;
try {
up_prot.up(new Event(Event.MSG, m));
} catch (Throwable t) {
log.error("couldn't deliver message " + m, t);
}
}
}
} finally {
// processing is always set in win.remove(processing) above and never here ! This code is just
// a
// 2nd line of defense should there be an exception before win.remove(processing) sets
// processing
if (!released_processing) processing.set(false);
}
}
public Object down(Event evt) {
switch (evt.getType()) {
case Event.MSG: // Add UnicastHeader, add to AckSenderWindow and pass down
Message msg = (Message) evt.getArg();
Address dst = msg.getDest();
/* only handle unicast messages */
if (dst == null || msg.isFlagSet(Message.NO_RELIABILITY)) break;
if (!started) {
if (log.isTraceEnabled())
log.trace("discarded message as start() has not yet been called, message: " + msg);
return null;
}
SenderEntry entry = send_table.get(dst);
if (entry == null) {
entry = new SenderEntry(getNewConnectionId());
SenderEntry existing = send_table.putIfAbsent(dst, entry);
if (existing != null) entry = existing;
else {
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": created connection to "
+ dst
+ " (conn_id="
+ entry.send_conn_id
+ ")");
if (cache != null && !members.contains(dst)) cache.add(dst);
}
}
long seqno = -2;
short send_conn_id = -1;
Unicast2Header hdr;
entry.lock(); // threads will only sync if they access the same entry
try {
seqno = entry.sent_msgs_seqno;
send_conn_id = entry.send_conn_id;
hdr = Unicast2Header.createDataHeader(seqno, send_conn_id, seqno == DEFAULT_FIRST_SEQNO);
msg.putHeader(this.id, hdr);
entry.sent_msgs.addToMessages(
seqno, msg); // add *including* UnicastHeader, adds to retransmitter
entry.sent_msgs_seqno++;
entry.update();
} finally {
entry.unlock();
}
if (log.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append(local_addr)
.append(" --> DATA(")
.append(dst)
.append(": #")
.append(seqno)
.append(", conn_id=")
.append(send_conn_id);
if (hdr.first) sb.append(", first");
sb.append(')');
log.trace(sb);
}
try {
down_prot.down(evt);
num_msgs_sent++;
num_bytes_sent += msg.getLength();
} catch (Throwable t) {
log.warn("failed sending the message", t);
}
return null; // we already passed the msg down
case Event.VIEW_CHANGE: // remove connections to peers that are not members anymore !
View view = (View) evt.getArg();
List<Address> new_members = view.getMembers();
Set<Address> non_members = new HashSet<Address>(send_table.keySet());
non_members.addAll(recv_table.keySet());
synchronized (members) {
members.clear();
if (new_members != null) members.addAll(new_members);
non_members.removeAll(members);
if (cache != null) {
cache.removeAll(members);
}
}
if (!non_members.isEmpty()) {
if (log.isTraceEnabled()) log.trace("removing non members " + non_members);
for (Address non_mbr : non_members) removeConnection(non_mbr);
}
break;
case Event.SET_LOCAL_ADDRESS:
local_addr = (Address) evt.getArg();
break;
}
return down_prot.down(evt); // Pass on to the layer below us
}
/**
* Check whether the hashmap contains an entry e for <code>sender</code> (create if not). If
* e.received_msgs is null and <code>first</code> is true: create a new AckReceiverWindow(seqno)
* and add message. Set e.received_msgs to the new window. Else just add the message.
*/
protected void handleDataReceived(
Address sender, long seqno, short conn_id, boolean first, Message msg, Event evt) {
if (log.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append(local_addr).append(" <-- DATA(").append(sender).append(": #").append(seqno);
if (conn_id != 0) sb.append(", conn_id=").append(conn_id);
if (first) sb.append(", first");
sb.append(')');
log.trace(sb);
}
ReceiverEntry entry = getReceiverEntry(sender, seqno, first, conn_id);
if (entry == null) return;
if (conn_expiry_timeout > 0) entry.update();
Table<Message> win = entry.received_msgs;
boolean added = win.add(seqno, msg); // win is guaranteed to be non-null if we get here
num_msgs_received++;
if (added) {
int len = msg.getLength();
if (len > 0 && entry.incrementStable(len))
sendStableMessage(
sender, entry.recv_conn_id, win.getHighestDelivered(), win.getHighestReceived());
}
// An OOB message is passed up immediately. Later, when remove() is called, we discard it. This
// affects ordering !
// http://jira.jboss.com/jira/browse/JGRP-377
if (msg.isFlagSet(Message.OOB) && added) {
try {
up_prot.up(evt);
} catch (Throwable t) {
log.error("couldn't deliver OOB message " + msg, t);
}
}
final AtomicBoolean processing = win.getProcessing();
if (!processing.compareAndSet(false, true)) {
return;
}
// Try to remove as many messages as possible and pass them up.
// Prevents concurrent passing up of messages by different threads
// (http://jira.jboss.com/jira/browse/JGRP-198);
// this is all the more important once we have a concurrent stack
// (http://jira.jboss.com/jira/browse/JGRP-181),
// where lots of threads can come up to this point concurrently, but only 1 is allowed to pass
// at a time
// We *can* deliver messages from *different* senders concurrently, e.g. reception of P1, Q1,
// P2, Q2 can result in
// delivery of P1, Q1, Q2, P2: FIFO (implemented by UNICAST) says messages need to be delivered
// only in the
// order in which they were sent by their senders
boolean released_processing = false;
try {
while (true) {
List<Message> msgs =
win.removeMany(processing, true, max_msg_batch_size); // remove my own messages
if (msgs == null || msgs.isEmpty()) {
released_processing = true;
return;
}
for (Message m : msgs) {
// discard OOB msg: it has already been delivered
// (http://jira.jboss.com/jira/browse/JGRP-377)
if (m.isFlagSet(Message.OOB)) continue;
try {
up_prot.up(new Event(Event.MSG, m));
} catch (Throwable t) {
log.error("couldn't deliver message " + m, t);
}
}
}
} finally {
// processing is always set in win.remove(processing) above and never here ! This code is just
// a
// 2nd line of defense should there be an exception before win.remove(processing) sets
// processing
if (!released_processing) processing.set(false);
}
}
public Object down(Event evt) {
switch (evt.getType()) {
case Event.MSG: // Add UnicastHeader, add to AckSenderWindow and pass down
Message msg = (Message) evt.getArg();
Address dst = msg.getDest();
/* only handle unicast messages */
if (dst == null || msg.isFlagSet(Message.NO_RELIABILITY)) break;
if (!running) {
if (log.isTraceEnabled())
log.trace("discarded message as start() has not yet been called, message: " + msg);
return null;
}
SenderEntry entry = send_table.get(dst);
if (entry == null) {
entry = new SenderEntry(getNewConnectionId());
SenderEntry existing = send_table.putIfAbsent(dst, entry);
if (existing != null) entry = existing;
else {
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": created connection to "
+ dst
+ " (conn_id="
+ entry.send_conn_id
+ ")");
if (cache != null && !members.contains(dst)) cache.add(dst);
}
}
short send_conn_id = entry.send_conn_id;
long seqno = entry.sent_msgs_seqno.getAndIncrement();
long sleep = 10;
while (running) {
try {
msg.putHeader(
this.id,
Unicast2Header.createDataHeader(seqno, send_conn_id, seqno == DEFAULT_FIRST_SEQNO));
entry.sent_msgs.add(seqno, msg); // add *including* UnicastHeader, adds to retransmitter
if (conn_expiry_timeout > 0) entry.update();
break;
} catch (Throwable t) {
if (!running) break;
if (log.isWarnEnabled()) log.warn("failed sending message", t);
Util.sleep(sleep);
sleep = Math.min(5000, sleep * 2);
}
}
if (log.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append(local_addr)
.append(" --> DATA(")
.append(dst)
.append(": #")
.append(seqno)
.append(", conn_id=")
.append(send_conn_id);
if (seqno == DEFAULT_FIRST_SEQNO) sb.append(", first");
sb.append(')');
log.trace(sb);
}
try {
down_prot.down(evt);
num_msgs_sent++;
} catch (Throwable t) {
log.warn("failed sending the message", t);
}
return null; // we already passed the msg down
case Event.VIEW_CHANGE: // remove connections to peers that are not members anymore !
View view = (View) evt.getArg();
List<Address> new_members = view.getMembers();
Set<Address> non_members = new HashSet<Address>(send_table.keySet());
non_members.addAll(recv_table.keySet());
members = new_members;
non_members.removeAll(new_members);
if (cache != null) cache.removeAll(new_members);
if (!non_members.isEmpty()) {
if (log.isTraceEnabled()) log.trace("removing non members " + non_members);
for (Address non_mbr : non_members) removeConnection(non_mbr);
}
break;
case Event.SET_LOCAL_ADDRESS:
local_addr = (Address) evt.getArg();
break;
}
return down_prot.down(evt); // Pass on to the layer below us
}