public String toString() { StringBuilder sb = new StringBuilder(); if (sent_msgs != null) sb.append(sent_msgs).append(", "); sb.append("send_conn_id=" + send_conn_id); sb.append(" (" + age() + " ms old)"); return sb.toString(); }
protected void sendStableMessage(Address dest, short conn_id, long low, long high) { Message stable_msg = new Message(dest, null, null); Unicast2Header hdr = Unicast2Header.createStableHeader(conn_id, low, high); stable_msg.putHeader(this.id, hdr); stable_msg.setFlag(Message.OOB); if (log.isTraceEnabled()) { StringBuilder sb = new StringBuilder(); sb.append(local_addr) .append(" --> STABLE(") .append(dest) .append(": ") .append(low) .append("-") .append(high) .append(", conn_id=") .append(conn_id) .append(")"); log.trace(sb.toString()); } down_prot.down(new Event(Event.MSG, stable_msg)); ReceiverEntry entry = recv_table.get(dest); NakReceiverWindow win = entry != null ? entry.received_msgs : null; if (win != null) win.stable(win.getHighestDelivered()); }
public String toString() { StringBuilder sb = new StringBuilder(); if (received_msgs != null) sb.append(received_msgs).append(", "); sb.append("recv_conn_id=" + recv_conn_id); sb.append(" (" + age() + " ms old)"); return sb.toString(); }
public String toString() { StringBuilder sb = new StringBuilder(); sb.append(type2Str(type)).append(", seqno=").append(seqno); if (conn_id != 0) sb.append(", conn_id=").append(conn_id); if (first) sb.append(", first"); return sb.toString(); }
@ManagedOperation(description = "Prints the contents of the send windows for all members") public String printSendWindowMessages() { StringBuilder ret = new StringBuilder(local_addr + ":\n"); for (Map.Entry<Address, SenderEntry> entry : send_table.entrySet()) { Address addr = entry.getKey(); Table<Message> buf = entry.getValue().sent_msgs; ret.append(addr).append(": ").append(buf.toString()).append('\n'); } return ret.toString(); }
@ManagedOperation(description = "Returns the sizes of all NakReceiverWindow.RetransmitTables") public String printRetransmitTableSizes() { StringBuilder sb = new StringBuilder(); for (Map.Entry<Address, ReceiverEntry> entry : recv_table.entrySet()) { NakReceiverWindow win = entry.getValue().received_msgs; sb.append(entry.getKey() + ": ") .append(win.getRetransmitTableSize()) .append(" (capacity=" + win.getRetransmitTableCapacity()) .append(", fill factor=" + win.getRetransmitTableFillFactor() + "%)\n"); } return sb.toString(); }
/** * We need to resend our first message with our conn_id * * @param sender * @param seqno Resend messages in the range [lowest .. seqno] */ private void handleResendingOfFirstMessage(Address sender, long seqno) { if (log.isTraceEnabled()) log.trace(local_addr + " <-- SEND_FIRST_SEQNO(" + sender + "," + seqno + ")"); SenderEntry entry = send_table.get(sender); AckSenderWindow win = entry != null ? entry.sent_msgs : null; if (win == null) { if (log.isErrorEnabled()) log.error(local_addr + ": sender window for " + sender + " not found"); return; } long lowest = win.getLowest(); Message rsp = win.get(lowest); if (rsp == null) return; // We need to copy the UnicastHeader and put it back into the message because Message.copy() // doesn't copy // the headers and therefore we'd modify the original message in the sender retransmission // window // (https://jira.jboss.org/jira/browse/JGRP-965) Message copy = rsp.copy(); Unicast2Header hdr = (Unicast2Header) copy.getHeader(this.id); Unicast2Header newhdr = hdr.copy(); newhdr.first = true; copy.putHeader(this.id, newhdr); if (log.isTraceEnabled()) { StringBuilder sb = new StringBuilder(); sb.append(local_addr) .append(" --> DATA(") .append(copy.getDest()) .append(": #") .append(newhdr.seqno) .append(", conn_id=") .append(newhdr.conn_id); if (newhdr.first) sb.append(", first"); sb.append(')'); log.trace(sb); } down_prot.down(new Event(Event.MSG, copy)); if (++lowest > seqno) return; for (long i = lowest; i <= seqno; i++) { rsp = win.get(i); if (rsp != null) down_prot.down(new Event(Event.MSG, rsp)); } }
protected void sendStableMessage(Address dest, short conn_id, long hd, long hr) { Message stable_msg = new Message(dest, null, null); Unicast2Header hdr = Unicast2Header.createStableHeader(conn_id, hd, hr); stable_msg.putHeader(this.id, hdr); stable_msg.setFlag(Message.OOB); if (log.isTraceEnabled()) { StringBuilder sb = new StringBuilder(); sb.append(local_addr) .append(" --> STABLE(") .append(dest) .append(": ") .append(hd) .append("-") .append(hr) .append(", conn_id=") .append(conn_id) .append(")"); log.trace(sb.toString()); } down_prot.down(new Event(Event.MSG, stable_msg)); }
private void handleXmitRequest(Address sender, long low, long high) { if (log.isTraceEnabled()) log.trace( new StringBuilder() .append(local_addr) .append(" <-- XMIT(") .append(sender) .append(": #") .append(low) .append("-") .append(high) .append(')')); SenderEntry entry = send_table.get(sender); AckSenderWindow win = entry != null ? entry.sent_msgs : null; if (win != null) { for (long i = low; i <= high; i++) { Message msg = win.get(i); if (msg == null) { if (log.isWarnEnabled() && !local_addr.equals(sender)) { StringBuilder sb = new StringBuilder(); sb.append("(requester=").append(sender).append(", local_addr=").append(this.local_addr); sb.append(") message ").append(sender).append("::").append(i); sb.append(" not found in retransmission table of ") .append(sender) .append(":\n") .append(win); log.warn(sb.toString()); } continue; } down_prot.down(new Event(Event.MSG, msg)); num_xmits++; } } }
protected void handleXmitRequest(Address sender, SeqnoList missing) { if (log.isTraceEnabled()) log.trace( new StringBuilder() .append(local_addr) .append(" <-- XMIT(") .append(sender) .append(": #") .append(missing) .append(')')); SenderEntry entry = send_table.get(sender); xmit_reqs_received.addAndGet(missing.size()); Table<Message> win = entry != null ? entry.sent_msgs : null; if (win != null) { for (long seqno : missing) { Message msg = win.get(seqno); if (msg == null) { if (log.isWarnEnabled() && !local_addr.equals(sender)) { StringBuilder sb = new StringBuilder(); sb.append("(requester=").append(sender).append(", local_addr=").append(this.local_addr); sb.append(") message ").append(sender).append("::").append(seqno); sb.append(" not found in retransmission table of ") .append(sender) .append(":\n") .append(win); log.warn(sb.toString()); } continue; } down_prot.down(new Event(Event.MSG, msg)); xmit_rsps_sent.incrementAndGet(); } } }
@ManagedOperation public String printConnections() { StringBuilder sb = new StringBuilder(); if (!send_table.isEmpty()) { sb.append("send connections:\n"); for (Map.Entry<Address, SenderEntry> entry : send_table.entrySet()) { sb.append(entry.getKey()).append(": ").append(entry.getValue()).append("\n"); } } if (!recv_table.isEmpty()) { sb.append("\nreceive connections:\n"); for (Map.Entry<Address, ReceiverEntry> entry : recv_table.entrySet()) { sb.append(entry.getKey()).append(": ").append(entry.getValue()).append("\n"); } } return sb.toString(); }
@ManagedOperation public String toString(boolean details) { StringBuilder sb = new StringBuilder(); sb.append("local_addr=") .append(local_addr) .append('\n') .append("cluster_name=") .append(cluster_name) .append('\n') .append("my_view=") .append(my_view) .append('\n') .append("state=") .append(state) .append('\n'); if (details) { sb.append("discard_own_messages=").append(discard_own_messages).append('\n'); sb.append("state_transfer_supported=").append(state_transfer_supported).append('\n'); sb.append("props=").append(getProperties()).append('\n'); } return sb.toString(); }
/** * Check whether the hashtable contains an entry e for <code>sender</code> (create if not). If * e.received_msgs is null and <code>first</code> is true: create a new AckReceiverWindow(seqno) * and add message. Set e.received_msgs to the new window. Else just add the message. */ protected void handleDataReceived( Address sender, long seqno, short conn_id, boolean first, Message msg, Event evt) { if (log.isTraceEnabled()) { StringBuilder sb = new StringBuilder(); sb.append(local_addr).append(" <-- DATA(").append(sender).append(": #").append(seqno); if (conn_id != 0) sb.append(", conn_id=").append(conn_id); if (first) sb.append(", first"); sb.append(')'); log.trace(sb); } ReceiverEntry entry; NakReceiverWindow win; recv_table_lock.lock(); try { entry = recv_table.get(sender); win = entry != null ? entry.received_msgs : null; if (first) { if (entry == null) { entry = getOrCreateReceiverEntry(sender, seqno, conn_id); win = entry.received_msgs; } else { // entry != null && win != null if (conn_id != entry.recv_conn_id) { if (log.isTraceEnabled()) log.trace( local_addr + ": conn_id=" + conn_id + " != " + entry.recv_conn_id + "; resetting receiver window"); ReceiverEntry entry2 = recv_table.remove(sender); if (entry2 != null) entry2.received_msgs.destroy(); entry = getOrCreateReceiverEntry(sender, seqno, conn_id); win = entry.received_msgs; } else {; } } } else { // entry == null && win == null OR entry != null && win == null OR entry != null && // win != null if (win == null || entry.recv_conn_id != conn_id) { recv_table_lock.unlock(); sendRequestForFirstSeqno(sender, seqno); // drops the message and returns (see below) return; } } } finally { if (recv_table_lock.isHeldByCurrentThread()) recv_table_lock.unlock(); } entry.update(); boolean added = win.add(seqno, msg); // win is guaranteed to be non-null if we get here num_msgs_received++; num_bytes_received += msg.getLength(); if (added) { int len = msg.getLength(); if (len > 0) { boolean send_stable_msg = false; entry.lock(); try { entry.received_bytes += len; if (entry.received_bytes >= max_bytes) { entry.received_bytes = 0; send_stable_msg = true; } } finally { entry.unlock(); } if (send_stable_msg) sendStableMessage( sender, entry.recv_conn_id, win.getHighestDelivered(), win.getHighestReceived()); } } // An OOB message is passed up immediately. Later, when remove() is called, we discard it. This // affects ordering ! // http://jira.jboss.com/jira/browse/JGRP-377 if (msg.isFlagSet(Message.OOB) && added) { try { up_prot.up(evt); } catch (Throwable t) { log.error("couldn't deliver OOB message " + msg, t); } } final AtomicBoolean processing = win.getProcessing(); if (!processing.compareAndSet(false, true)) { return; } // try to remove (from the AckReceiverWindow) as many messages as possible and pass them up // Prevents concurrent passing up of messages by different threads // (http://jira.jboss.com/jira/browse/JGRP-198); // this is all the more important once we have a concurrent stack // (http://jira.jboss.com/jira/browse/JGRP-181), // where lots of threads can come up to this point concurrently, but only 1 is allowed to pass // at a time // We *can* deliver messages from *different* senders concurrently, e.g. reception of P1, Q1, // P2, Q2 can result in // delivery of P1, Q1, Q2, P2: FIFO (implemented by UNICAST) says messages need to be delivered // only in the // order in which they were sent by their senders boolean released_processing = false; try { while (true) { List<Message> msgs = win.removeMany(processing, true, max_msg_batch_size); // remove my own messages if (msgs == null || msgs.isEmpty()) { released_processing = true; return; } for (Message m : msgs) { // discard OOB msg: it has already been delivered // (http://jira.jboss.com/jira/browse/JGRP-377) if (m.isFlagSet(Message.OOB)) continue; try { up_prot.up(new Event(Event.MSG, m)); } catch (Throwable t) { log.error("couldn't deliver message " + m, t); } } } } finally { // processing is always set in win.remove(processing) above and never here ! This code is just // a // 2nd line of defense should there be an exception before win.remove(processing) sets // processing if (!released_processing) processing.set(false); } }
public Object down(Event evt) { switch (evt.getType()) { case Event.MSG: // Add UnicastHeader, add to AckSenderWindow and pass down Message msg = (Message) evt.getArg(); Address dst = msg.getDest(); /* only handle unicast messages */ if (dst == null || msg.isFlagSet(Message.NO_RELIABILITY)) break; if (!started) { if (log.isTraceEnabled()) log.trace("discarded message as start() has not yet been called, message: " + msg); return null; } SenderEntry entry = send_table.get(dst); if (entry == null) { entry = new SenderEntry(getNewConnectionId()); SenderEntry existing = send_table.putIfAbsent(dst, entry); if (existing != null) entry = existing; else { if (log.isTraceEnabled()) log.trace( local_addr + ": created connection to " + dst + " (conn_id=" + entry.send_conn_id + ")"); if (cache != null && !members.contains(dst)) cache.add(dst); } } long seqno = -2; short send_conn_id = -1; Unicast2Header hdr; entry.lock(); // threads will only sync if they access the same entry try { seqno = entry.sent_msgs_seqno; send_conn_id = entry.send_conn_id; hdr = Unicast2Header.createDataHeader(seqno, send_conn_id, seqno == DEFAULT_FIRST_SEQNO); msg.putHeader(this.id, hdr); entry.sent_msgs.addToMessages( seqno, msg); // add *including* UnicastHeader, adds to retransmitter entry.sent_msgs_seqno++; entry.update(); } finally { entry.unlock(); } if (log.isTraceEnabled()) { StringBuilder sb = new StringBuilder(); sb.append(local_addr) .append(" --> DATA(") .append(dst) .append(": #") .append(seqno) .append(", conn_id=") .append(send_conn_id); if (hdr.first) sb.append(", first"); sb.append(')'); log.trace(sb); } try { down_prot.down(evt); num_msgs_sent++; num_bytes_sent += msg.getLength(); } catch (Throwable t) { log.warn("failed sending the message", t); } return null; // we already passed the msg down case Event.VIEW_CHANGE: // remove connections to peers that are not members anymore ! View view = (View) evt.getArg(); List<Address> new_members = view.getMembers(); Set<Address> non_members = new HashSet<Address>(send_table.keySet()); non_members.addAll(recv_table.keySet()); synchronized (members) { members.clear(); if (new_members != null) members.addAll(new_members); non_members.removeAll(members); if (cache != null) { cache.removeAll(members); } } if (!non_members.isEmpty()) { if (log.isTraceEnabled()) log.trace("removing non members " + non_members); for (Address non_mbr : non_members) removeConnection(non_mbr); } break; case Event.SET_LOCAL_ADDRESS: local_addr = (Address) evt.getArg(); break; } return down_prot.down(evt); // Pass on to the layer below us }
/** * Check whether the hashmap contains an entry e for <code>sender</code> (create if not). If * e.received_msgs is null and <code>first</code> is true: create a new AckReceiverWindow(seqno) * and add message. Set e.received_msgs to the new window. Else just add the message. */ protected void handleDataReceived( Address sender, long seqno, short conn_id, boolean first, Message msg, Event evt) { if (log.isTraceEnabled()) { StringBuilder sb = new StringBuilder(); sb.append(local_addr).append(" <-- DATA(").append(sender).append(": #").append(seqno); if (conn_id != 0) sb.append(", conn_id=").append(conn_id); if (first) sb.append(", first"); sb.append(')'); log.trace(sb); } ReceiverEntry entry = getReceiverEntry(sender, seqno, first, conn_id); if (entry == null) return; if (conn_expiry_timeout > 0) entry.update(); Table<Message> win = entry.received_msgs; boolean added = win.add(seqno, msg); // win is guaranteed to be non-null if we get here num_msgs_received++; if (added) { int len = msg.getLength(); if (len > 0 && entry.incrementStable(len)) sendStableMessage( sender, entry.recv_conn_id, win.getHighestDelivered(), win.getHighestReceived()); } // An OOB message is passed up immediately. Later, when remove() is called, we discard it. This // affects ordering ! // http://jira.jboss.com/jira/browse/JGRP-377 if (msg.isFlagSet(Message.OOB) && added) { try { up_prot.up(evt); } catch (Throwable t) { log.error("couldn't deliver OOB message " + msg, t); } } final AtomicBoolean processing = win.getProcessing(); if (!processing.compareAndSet(false, true)) { return; } // Try to remove as many messages as possible and pass them up. // Prevents concurrent passing up of messages by different threads // (http://jira.jboss.com/jira/browse/JGRP-198); // this is all the more important once we have a concurrent stack // (http://jira.jboss.com/jira/browse/JGRP-181), // where lots of threads can come up to this point concurrently, but only 1 is allowed to pass // at a time // We *can* deliver messages from *different* senders concurrently, e.g. reception of P1, Q1, // P2, Q2 can result in // delivery of P1, Q1, Q2, P2: FIFO (implemented by UNICAST) says messages need to be delivered // only in the // order in which they were sent by their senders boolean released_processing = false; try { while (true) { List<Message> msgs = win.removeMany(processing, true, max_msg_batch_size); // remove my own messages if (msgs == null || msgs.isEmpty()) { released_processing = true; return; } for (Message m : msgs) { // discard OOB msg: it has already been delivered // (http://jira.jboss.com/jira/browse/JGRP-377) if (m.isFlagSet(Message.OOB)) continue; try { up_prot.up(new Event(Event.MSG, m)); } catch (Throwable t) { log.error("couldn't deliver message " + m, t); } } } } finally { // processing is always set in win.remove(processing) above and never here ! This code is just // a // 2nd line of defense should there be an exception before win.remove(processing) sets // processing if (!released_processing) processing.set(false); } }
public Object down(Event evt) { switch (evt.getType()) { case Event.MSG: // Add UnicastHeader, add to AckSenderWindow and pass down Message msg = (Message) evt.getArg(); Address dst = msg.getDest(); /* only handle unicast messages */ if (dst == null || msg.isFlagSet(Message.NO_RELIABILITY)) break; if (!running) { if (log.isTraceEnabled()) log.trace("discarded message as start() has not yet been called, message: " + msg); return null; } SenderEntry entry = send_table.get(dst); if (entry == null) { entry = new SenderEntry(getNewConnectionId()); SenderEntry existing = send_table.putIfAbsent(dst, entry); if (existing != null) entry = existing; else { if (log.isTraceEnabled()) log.trace( local_addr + ": created connection to " + dst + " (conn_id=" + entry.send_conn_id + ")"); if (cache != null && !members.contains(dst)) cache.add(dst); } } short send_conn_id = entry.send_conn_id; long seqno = entry.sent_msgs_seqno.getAndIncrement(); long sleep = 10; while (running) { try { msg.putHeader( this.id, Unicast2Header.createDataHeader(seqno, send_conn_id, seqno == DEFAULT_FIRST_SEQNO)); entry.sent_msgs.add(seqno, msg); // add *including* UnicastHeader, adds to retransmitter if (conn_expiry_timeout > 0) entry.update(); break; } catch (Throwable t) { if (!running) break; if (log.isWarnEnabled()) log.warn("failed sending message", t); Util.sleep(sleep); sleep = Math.min(5000, sleep * 2); } } if (log.isTraceEnabled()) { StringBuilder sb = new StringBuilder(); sb.append(local_addr) .append(" --> DATA(") .append(dst) .append(": #") .append(seqno) .append(", conn_id=") .append(send_conn_id); if (seqno == DEFAULT_FIRST_SEQNO) sb.append(", first"); sb.append(')'); log.trace(sb); } try { down_prot.down(evt); num_msgs_sent++; } catch (Throwable t) { log.warn("failed sending the message", t); } return null; // we already passed the msg down case Event.VIEW_CHANGE: // remove connections to peers that are not members anymore ! View view = (View) evt.getArg(); List<Address> new_members = view.getMembers(); Set<Address> non_members = new HashSet<Address>(send_table.keySet()); non_members.addAll(recv_table.keySet()); members = new_members; non_members.removeAll(new_members); if (cache != null) cache.removeAll(new_members); if (!non_members.isEmpty()) { if (log.isTraceEnabled()) log.trace("removing non members " + non_members); for (Address non_mbr : non_members) removeConnection(non_mbr); } break; case Event.SET_LOCAL_ADDRESS: local_addr = (Address) evt.getArg(); break; } return down_prot.down(evt); // Pass on to the layer below us }