Exemple #1
0
  public void up(MessageBatch batch) {
    if (batch.dest() == null) { // not a unicast batch
      up_prot.up(batch);
      return;
    }

    int size = batch.size();
    Map<Short, List<Message>> msgs =
        new TreeMap<Short, List<Message>>(); // map of messages, keyed by conn-id
    for (Message msg : batch) {
      if (msg == null || msg.isFlagSet(Message.Flag.NO_RELIABILITY)) continue;
      UnicastHeader hdr = (UnicastHeader) msg.getHeader(id);
      if (hdr == null) continue;
      batch.remove(msg); // remove the message from the batch, so it won't be passed up the stack

      if (hdr.type != UnicastHeader.DATA) {
        try {
          handleUpEvent(msg.getSrc(), hdr);
        } catch (Throwable t) { // we cannot let an exception terminate the processing of this batch
          log.error(local_addr + ": failed handling event", t);
        }
        continue;
      }

      List<Message> list = msgs.get(hdr.conn_id);
      if (list == null) msgs.put(hdr.conn_id, list = new ArrayList<Message>(size));
      list.add(msg);
    }

    if (!msgs.isEmpty()) handleBatchReceived(batch.sender(), msgs); // process msgs:
    if (!batch.isEmpty()) up_prot.up(batch);
  }
Exemple #2
0
  protected int removeAndDeliver(final AtomicBoolean processing, Table<Message> win) {
    int retval = 0;
    boolean released_processing = false;
    try {
      while (true) {
        List<Message> list = win.removeMany(processing, true, max_msg_batch_size);
        if (list == null) {
          released_processing = true;
          return retval;
        }

        for (Message m : list) {
          retval++;
          // discard OOB msg: it has already been delivered
          // (http://jira.jboss.com/jira/browse/JGRP-377)
          if (m.isFlagSet(Message.OOB)) continue;
          try {
            up_prot.up(new Event(Event.MSG, m));
          } catch (Throwable t) {
            log.error("couldn't deliver message " + m, t);
          }
        }
      }
    } finally {
      // processing is always set in win.remove(processing) above and never here ! This code is just
      // a
      // 2nd line of defense should there be an exception before win.remove(processing) sets
      // processing
      if (!released_processing) processing.set(false);
    }
  }
Exemple #3
0
    public void receive(Message msg) {
      if (!msg.isFlagSet(Message.OOB)) {
        lock.lock();
        lock.unlock();
      }

      msgs.add((Integer) msg.getObject());
    }
Exemple #4
0
  protected void handleBatchReceived(Address sender, Map<Short, List<Message>> map) {
    for (Map.Entry<Short, List<Message>> element : map.entrySet()) {
      final List<Message> msg_list = element.getValue();
      if (log.isTraceEnabled()) {
        StringBuilder sb = new StringBuilder();
        sb.append(local_addr)
            .append(" <-- DATA(")
            .append(sender)
            .append(": " + printMessageList(msg_list))
            .append(')');
        log.trace(sb);
      }

      short conn_id = element.getKey();
      ReceiverEntry entry = null;
      for (Message msg : msg_list) {
        UnicastHeader hdr = (UnicastHeader) msg.getHeader(id);
        entry = getReceiverEntry(sender, hdr.seqno, hdr.first, conn_id);
        if (entry == null) continue;
        Table<Message> win = entry.received_msgs;
        boolean msg_added =
            win.add(hdr.seqno, msg); // win is guaranteed to be non-null if we get here
        num_msgs_received++;

        if (hdr.first && msg_added)
          sendAck(
              sender, hdr.seqno,
              conn_id); // send an ack immediately when we received the first message of a conn

        // An OOB message is passed up immediately. Later, when remove() is called, we discard it.
        // This affects ordering !
        // http://jira.jboss.com/jira/browse/JGRP-377
        if (msg.isFlagSet(Message.Flag.OOB) && msg_added) {
          try {
            up_prot.up(new Event(Event.MSG, msg));
          } catch (Throwable t) {
            log.error("couldn't deliver OOB message " + msg, t);
          }
        }
      }
      if (entry != null && conn_expiry_timeout > 0) entry.update();
    }

    ReceiverEntry entry = recv_table.get(sender);
    Table<Message> win = entry != null ? entry.received_msgs : null;
    if (win != null) {
      final AtomicBoolean processing = win.getProcessing();
      if (processing.compareAndSet(false, true)) {
        removeAndDeliver(processing, win, sender);
        sendAck(sender, win.getHighestDeliverable(), entry.recv_conn_id);
      }
    }
  }
Exemple #5
0
  /**
   * Check whether the hashtable contains an entry e for <code>sender</code> (create if not). If
   * e.received_msgs is null and <code>first</code> is true: create a new AckReceiverWindow(seqno)
   * and add message. Set e.received_msgs to the new window. Else just add the message.
   */
  protected void handleDataReceived(
      Address sender, long seqno, short conn_id, boolean first, Message msg, Event evt) {
    if (log.isTraceEnabled()) {
      StringBuilder sb = new StringBuilder();
      sb.append(local_addr).append(" <-- DATA(").append(sender).append(": #").append(seqno);
      if (conn_id != 0) sb.append(", conn_id=").append(conn_id);
      if (first) sb.append(", first");
      sb.append(')');
      log.trace(sb);
    }

    ReceiverEntry entry = getReceiverEntry(sender, seqno, first, conn_id);
    if (entry == null) return;
    if (conn_expiry_timeout > 0) entry.update();
    Table<Message> win = entry.received_msgs;
    boolean added = win.add(seqno, msg); // win is guaranteed to be non-null if we get here
    num_msgs_received++;

    // An OOB message is passed up immediately. Later, when remove() is called, we discard it. This
    // affects ordering !
    // http://jira.jboss.com/jira/browse/JGRP-377
    if (msg.isFlagSet(Message.OOB) && added) {
      try {
        up_prot.up(evt);
      } catch (Throwable t) {
        log.error("couldn't deliver OOB message " + msg, t);
      }
    }

    final AtomicBoolean processing = win.getProcessing();
    if (!processing.compareAndSet(false, true)) {
      return;
    }

    // try to remove (from the AckReceiverWindow) as many messages as possible as pass them up

    // Prevents concurrent passing up of messages by different threads
    // (http://jira.jboss.com/jira/browse/JGRP-198);
    // this is all the more important once we have a concurrent stack
    // (http://jira.jboss.com/jira/browse/JGRP-181),
    // where lots of threads can come up to this point concurrently, but only 1 is allowed to pass
    // at a time
    // We *can* deliver messages from *different* senders concurrently, e.g. reception of P1, Q1,
    // P2, Q2 can result in
    // delivery of P1, Q1, Q2, P2: FIFO (implemented by UNICAST) says messages need to be delivered
    // only in the
    // order in which they were sent by their senders
    int num_msgs = removeAndDeliver(processing, win);
    if (num_msgs > 0) {
      long hd = win.getHighestDelivered();
      sendAck(sender, hd);
    }
  }
Exemple #6
0
    public void receive(Message msg) {
      if (!msg.isFlagSet(Message.OOB)) {
        try {
          System.out.println(Thread.currentThread() + ": waiting on latch");
          latch.await(25000, TimeUnit.MILLISECONDS);
          System.out.println(Thread.currentThread() + ": DONE waiting on latch");
        } catch (InterruptedException e) {
          e.printStackTrace();
        }
      }

      msgs.add((Integer) msg.getObject());
    }
Exemple #7
0
  /**
   * Try to remove as many messages as possible from the table as pass them up. Prevents concurrent
   * passing up of messages by different threads (http://jira.jboss.com/jira/browse/JGRP-198); lots
   * of threads can come up to this point concurrently, but only 1 is allowed to pass at a time. We
   * *can* deliver messages from *different* senders concurrently, e.g. reception of P1, Q1, P2, Q2
   * can result in delivery of P1, Q1, Q2, P2: FIFO (implemented by UNICAST) says messages need to
   * be delivered in the order in which they were sent
   */
  protected int removeAndDeliver(
      final AtomicBoolean processing, Table<Message> win, Address sender) {
    int retval = 0;
    boolean released_processing = false;
    try {
      while (true) {
        List<Message> list = win.removeMany(processing, true, max_msg_batch_size);
        if (list == null) {
          released_processing = true;
          return retval;
        }

        MessageBatch batch = new MessageBatch(local_addr, sender, null, false, list);
        for (Message msg_to_deliver : batch) {
          // discard OOB msg: it has already been delivered
          // (http://jira.jboss.com/jira/browse/JGRP-377)
          if (msg_to_deliver.isFlagSet(Message.Flag.OOB)) batch.remove(msg_to_deliver);
        }

        try {
          if (log.isTraceEnabled()) {
            Message first = batch.first(), last = batch.last();
            StringBuilder sb = new StringBuilder(local_addr + ": delivering");
            if (first != null && last != null) {
              UnicastHeader hdr1 = (UnicastHeader) first.getHeader(id),
                  hdr2 = (UnicastHeader) last.getHeader(id);
              sb.append(" #").append(hdr1.seqno).append(" - #").append(hdr2.seqno);
            }
            sb.append(" (" + batch.size()).append(" messages)");
            log.trace(sb);
          }
          up_prot.up(batch);
        } catch (Throwable t) {
          log.error("failed to deliver batch " + batch, t);
        }
      }
    } finally {
      // processing is always set in win.remove(processing) above and never here ! This code is just
      // a
      // 2nd line of defense should there be an exception before win.remove(processing) sets
      // processing
      if (!released_processing) processing.set(false);
    }
  }
Exemple #8
0
  public Object up(Event evt) {
    Message msg;
    Address dst, src;
    Unicast2Header hdr;

    switch (evt.getType()) {
      case Event.MSG:
        msg = (Message) evt.getArg();
        dst = msg.getDest();

        if (dst == null || msg.isFlagSet(Message.NO_RELIABILITY)) // only handle unicast messages
        break; // pass up

        // changed from removeHeader(): we cannot remove the header because if we do loopback=true
        // at the
        // transport level, we will not have the header on retransmit ! (bela Aug 22 2006)
        hdr = (Unicast2Header) msg.getHeader(this.id);
        if (hdr == null) break;
        src = msg.getSrc();
        switch (hdr.type) {
          case Unicast2Header.DATA: // received regular message
            handleDataReceived(src, hdr.seqno, hdr.conn_id, hdr.first, msg, evt);
            return null; // we pass the deliverable message up in handleDataReceived()
          case Unicast2Header.XMIT_REQ: // received ACK for previously sent message
            handleXmitRequest(src, hdr.seqno, hdr.high_seqno);
            break;
          case Unicast2Header.SEND_FIRST_SEQNO:
            handleResendingOfFirstMessage(src, hdr.seqno);
            break;
          case Unicast2Header.STABLE:
            stable(msg.getSrc(), hdr.conn_id, hdr.seqno, hdr.high_seqno);
            break;
          default:
            log.error("UnicastHeader type " + hdr.type + " not known !");
            break;
        }
        return null;
    }

    return up_prot.up(evt); // Pass up to the layer above us
  }
Exemple #9
0
  public Object up(Event evt) {
    switch (evt.getType()) {
      case Event.MSG:
        Message msg = (Message) evt.getArg();
        if (msg.getDest() == null
            || msg.isFlagSet(Message.Flag.NO_RELIABILITY)) // only handle unicast messages
        break; // pass up

        UnicastHeader hdr = (UnicastHeader) msg.getHeader(this.id);
        if (hdr == null) break;
        Address sender = msg.getSrc();
        switch (hdr.type) {
          case UnicastHeader.DATA: // received regular message
            handleDataReceived(sender, hdr.seqno, hdr.conn_id, hdr.first, msg, evt);
            break;
          default:
            handleUpEvent(sender, hdr);
            break;
        }
        return null;
    }
    return up_prot.up(evt); // Pass up to the layer above us
  }
Exemple #10
0
  /**
   * Check whether the hashtable contains an entry e for <code>sender</code> (create if not). If
   * e.received_msgs is null and <code>first</code> is true: create a new AckReceiverWindow(seqno)
   * and add message. Set e.received_msgs to the new window. Else just add the message.
   */
  protected void handleDataReceived(
      Address sender, long seqno, short conn_id, boolean first, Message msg, Event evt) {
    if (log.isTraceEnabled()) {
      StringBuilder sb = new StringBuilder();
      sb.append(local_addr).append(" <-- DATA(").append(sender).append(": #").append(seqno);
      if (conn_id != 0) sb.append(", conn_id=").append(conn_id);
      if (first) sb.append(", first");
      sb.append(')');
      log.trace(sb);
    }

    ReceiverEntry entry;
    NakReceiverWindow win;

    recv_table_lock.lock();
    try {
      entry = recv_table.get(sender);
      win = entry != null ? entry.received_msgs : null;
      if (first) {
        if (entry == null) {
          entry = getOrCreateReceiverEntry(sender, seqno, conn_id);
          win = entry.received_msgs;
        } else { // entry != null && win != null
          if (conn_id != entry.recv_conn_id) {
            if (log.isTraceEnabled())
              log.trace(
                  local_addr
                      + ": conn_id="
                      + conn_id
                      + " != "
                      + entry.recv_conn_id
                      + "; resetting receiver window");

            ReceiverEntry entry2 = recv_table.remove(sender);
            if (entry2 != null) entry2.received_msgs.destroy();

            entry = getOrCreateReceiverEntry(sender, seqno, conn_id);
            win = entry.received_msgs;
          } else {;
          }
        }
      } else { // entry == null && win == null OR entry != null && win == null OR entry != null &&
               // win != null
        if (win == null || entry.recv_conn_id != conn_id) {
          recv_table_lock.unlock();
          sendRequestForFirstSeqno(sender, seqno); // drops the message and returns (see below)
          return;
        }
      }
    } finally {
      if (recv_table_lock.isHeldByCurrentThread()) recv_table_lock.unlock();
    }

    entry.update();
    boolean added = win.add(seqno, msg); // win is guaranteed to be non-null if we get here
    num_msgs_received++;
    num_bytes_received += msg.getLength();

    if (added) {
      int len = msg.getLength();
      if (len > 0) {
        boolean send_stable_msg = false;
        entry.lock();
        try {
          entry.received_bytes += len;
          if (entry.received_bytes >= max_bytes) {
            entry.received_bytes = 0;
            send_stable_msg = true;
          }
        } finally {
          entry.unlock();
        }

        if (send_stable_msg)
          sendStableMessage(
              sender, entry.recv_conn_id, win.getHighestDelivered(), win.getHighestReceived());
      }
    }

    // An OOB message is passed up immediately. Later, when remove() is called, we discard it. This
    // affects ordering !
    // http://jira.jboss.com/jira/browse/JGRP-377
    if (msg.isFlagSet(Message.OOB) && added) {
      try {
        up_prot.up(evt);
      } catch (Throwable t) {
        log.error("couldn't deliver OOB message " + msg, t);
      }
    }

    final AtomicBoolean processing = win.getProcessing();
    if (!processing.compareAndSet(false, true)) {
      return;
    }

    // try to remove (from the AckReceiverWindow) as many messages as possible and pass them up

    // Prevents concurrent passing up of messages by different threads
    // (http://jira.jboss.com/jira/browse/JGRP-198);
    // this is all the more important once we have a concurrent stack
    // (http://jira.jboss.com/jira/browse/JGRP-181),
    // where lots of threads can come up to this point concurrently, but only 1 is allowed to pass
    // at a time
    // We *can* deliver messages from *different* senders concurrently, e.g. reception of P1, Q1,
    // P2, Q2 can result in
    // delivery of P1, Q1, Q2, P2: FIFO (implemented by UNICAST) says messages need to be delivered
    // only in the
    // order in which they were sent by their senders
    boolean released_processing = false;
    try {
      while (true) {
        List<Message> msgs =
            win.removeMany(processing, true, max_msg_batch_size); // remove my own messages
        if (msgs == null || msgs.isEmpty()) {
          released_processing = true;
          return;
        }

        for (Message m : msgs) {
          // discard OOB msg: it has already been delivered
          // (http://jira.jboss.com/jira/browse/JGRP-377)
          if (m.isFlagSet(Message.OOB)) continue;
          try {
            up_prot.up(new Event(Event.MSG, m));
          } catch (Throwable t) {
            log.error("couldn't deliver message " + m, t);
          }
        }
      }
    } finally {
      // processing is always set in win.remove(processing) above and never here ! This code is just
      // a
      // 2nd line of defense should there be an exception before win.remove(processing) sets
      // processing
      if (!released_processing) processing.set(false);
    }
  }
Exemple #11
0
  public Object down(Event evt) {
    switch (evt.getType()) {
      case Event.MSG: // Add UnicastHeader, add to AckSenderWindow and pass down
        Message msg = (Message) evt.getArg();
        Address dst = msg.getDest();

        /* only handle unicast messages */
        if (dst == null || msg.isFlagSet(Message.NO_RELIABILITY)) break;

        if (!started) {
          if (log.isTraceEnabled())
            log.trace("discarded message as start() has not yet been called, message: " + msg);
          return null;
        }

        SenderEntry entry = send_table.get(dst);
        if (entry == null) {
          entry = new SenderEntry(getNewConnectionId());
          SenderEntry existing = send_table.putIfAbsent(dst, entry);
          if (existing != null) entry = existing;
          else {
            if (log.isTraceEnabled())
              log.trace(
                  local_addr
                      + ": created connection to "
                      + dst
                      + " (conn_id="
                      + entry.send_conn_id
                      + ")");
            if (cache != null && !members.contains(dst)) cache.add(dst);
          }
        }

        long seqno = -2;
        short send_conn_id = -1;
        Unicast2Header hdr;

        entry.lock(); // threads will only sync if they access the same entry
        try {
          seqno = entry.sent_msgs_seqno;
          send_conn_id = entry.send_conn_id;
          hdr = Unicast2Header.createDataHeader(seqno, send_conn_id, seqno == DEFAULT_FIRST_SEQNO);
          msg.putHeader(this.id, hdr);
          entry.sent_msgs.addToMessages(
              seqno, msg); // add *including* UnicastHeader, adds to retransmitter
          entry.sent_msgs_seqno++;
          entry.update();
        } finally {
          entry.unlock();
        }

        if (log.isTraceEnabled()) {
          StringBuilder sb = new StringBuilder();
          sb.append(local_addr)
              .append(" --> DATA(")
              .append(dst)
              .append(": #")
              .append(seqno)
              .append(", conn_id=")
              .append(send_conn_id);
          if (hdr.first) sb.append(", first");
          sb.append(')');
          log.trace(sb);
        }

        try {
          down_prot.down(evt);
          num_msgs_sent++;
          num_bytes_sent += msg.getLength();
        } catch (Throwable t) {
          log.warn("failed sending the message", t);
        }
        return null; // we already passed the msg down

      case Event.VIEW_CHANGE: // remove connections to peers that are not members anymore !
        View view = (View) evt.getArg();
        List<Address> new_members = view.getMembers();
        Set<Address> non_members = new HashSet<Address>(send_table.keySet());
        non_members.addAll(recv_table.keySet());

        synchronized (members) {
          members.clear();
          if (new_members != null) members.addAll(new_members);
          non_members.removeAll(members);
          if (cache != null) {
            cache.removeAll(members);
          }
        }

        if (!non_members.isEmpty()) {
          if (log.isTraceEnabled()) log.trace("removing non members " + non_members);
          for (Address non_mbr : non_members) removeConnection(non_mbr);
        }
        break;

      case Event.SET_LOCAL_ADDRESS:
        local_addr = (Address) evt.getArg();
        break;
    }

    return down_prot.down(evt); // Pass on to the layer below us
  }
Exemple #12
0
  /**
   * Check whether the hashmap contains an entry e for <code>sender</code> (create if not). If
   * e.received_msgs is null and <code>first</code> is true: create a new AckReceiverWindow(seqno)
   * and add message. Set e.received_msgs to the new window. Else just add the message.
   */
  protected void handleDataReceived(
      Address sender, long seqno, short conn_id, boolean first, Message msg, Event evt) {
    if (log.isTraceEnabled()) {
      StringBuilder sb = new StringBuilder();
      sb.append(local_addr).append(" <-- DATA(").append(sender).append(": #").append(seqno);
      if (conn_id != 0) sb.append(", conn_id=").append(conn_id);
      if (first) sb.append(", first");
      sb.append(')');
      log.trace(sb);
    }

    ReceiverEntry entry = getReceiverEntry(sender, seqno, first, conn_id);
    if (entry == null) return;
    if (conn_expiry_timeout > 0) entry.update();
    Table<Message> win = entry.received_msgs;
    boolean added = win.add(seqno, msg); // win is guaranteed to be non-null if we get here
    num_msgs_received++;

    if (added) {
      int len = msg.getLength();
      if (len > 0 && entry.incrementStable(len))
        sendStableMessage(
            sender, entry.recv_conn_id, win.getHighestDelivered(), win.getHighestReceived());
    }

    // An OOB message is passed up immediately. Later, when remove() is called, we discard it. This
    // affects ordering !
    // http://jira.jboss.com/jira/browse/JGRP-377
    if (msg.isFlagSet(Message.OOB) && added) {
      try {
        up_prot.up(evt);
      } catch (Throwable t) {
        log.error("couldn't deliver OOB message " + msg, t);
      }
    }

    final AtomicBoolean processing = win.getProcessing();
    if (!processing.compareAndSet(false, true)) {
      return;
    }

    // Try to remove as many messages as possible and pass them up.
    // Prevents concurrent passing up of messages by different threads
    // (http://jira.jboss.com/jira/browse/JGRP-198);
    // this is all the more important once we have a concurrent stack
    // (http://jira.jboss.com/jira/browse/JGRP-181),
    // where lots of threads can come up to this point concurrently, but only 1 is allowed to pass
    // at a time
    // We *can* deliver messages from *different* senders concurrently, e.g. reception of P1, Q1,
    // P2, Q2 can result in
    // delivery of P1, Q1, Q2, P2: FIFO (implemented by UNICAST) says messages need to be delivered
    // only in the
    // order in which they were sent by their senders
    boolean released_processing = false;
    try {
      while (true) {
        List<Message> msgs =
            win.removeMany(processing, true, max_msg_batch_size); // remove my own messages
        if (msgs == null || msgs.isEmpty()) {
          released_processing = true;
          return;
        }

        for (Message m : msgs) {
          // discard OOB msg: it has already been delivered
          // (http://jira.jboss.com/jira/browse/JGRP-377)
          if (m.isFlagSet(Message.OOB)) continue;
          try {
            up_prot.up(new Event(Event.MSG, m));
          } catch (Throwable t) {
            log.error("couldn't deliver message " + m, t);
          }
        }
      }
    } finally {
      // processing is always set in win.remove(processing) above and never here ! This code is just
      // a
      // 2nd line of defense should there be an exception before win.remove(processing) sets
      // processing
      if (!released_processing) processing.set(false);
    }
  }
Exemple #13
0
  public Object down(Event evt) {
    switch (evt.getType()) {
      case Event.MSG: // Add UnicastHeader, add to AckSenderWindow and pass down
        Message msg = (Message) evt.getArg();
        Address dst = msg.getDest();

        /* only handle unicast messages */
        if (dst == null || msg.isFlagSet(Message.NO_RELIABILITY)) break;

        if (!running) {
          if (log.isTraceEnabled())
            log.trace("discarded message as start() has not yet been called, message: " + msg);
          return null;
        }

        SenderEntry entry = send_table.get(dst);
        if (entry == null) {
          entry = new SenderEntry(getNewConnectionId());
          SenderEntry existing = send_table.putIfAbsent(dst, entry);
          if (existing != null) entry = existing;
          else {
            if (log.isTraceEnabled())
              log.trace(
                  local_addr
                      + ": created connection to "
                      + dst
                      + " (conn_id="
                      + entry.send_conn_id
                      + ")");
            if (cache != null && !members.contains(dst)) cache.add(dst);
          }
        }

        short send_conn_id = entry.send_conn_id;
        long seqno = entry.sent_msgs_seqno.getAndIncrement();
        long sleep = 10;
        while (running) {
          try {
            msg.putHeader(
                this.id,
                Unicast2Header.createDataHeader(seqno, send_conn_id, seqno == DEFAULT_FIRST_SEQNO));
            entry.sent_msgs.add(seqno, msg); // add *including* UnicastHeader, adds to retransmitter
            if (conn_expiry_timeout > 0) entry.update();
            break;
          } catch (Throwable t) {
            if (!running) break;
            if (log.isWarnEnabled()) log.warn("failed sending message", t);
            Util.sleep(sleep);
            sleep = Math.min(5000, sleep * 2);
          }
        }

        if (log.isTraceEnabled()) {
          StringBuilder sb = new StringBuilder();
          sb.append(local_addr)
              .append(" --> DATA(")
              .append(dst)
              .append(": #")
              .append(seqno)
              .append(", conn_id=")
              .append(send_conn_id);
          if (seqno == DEFAULT_FIRST_SEQNO) sb.append(", first");
          sb.append(')');
          log.trace(sb);
        }

        try {
          down_prot.down(evt);
          num_msgs_sent++;
        } catch (Throwable t) {
          log.warn("failed sending the message", t);
        }
        return null; // we already passed the msg down

      case Event.VIEW_CHANGE: // remove connections to peers that are not members anymore !
        View view = (View) evt.getArg();
        List<Address> new_members = view.getMembers();
        Set<Address> non_members = new HashSet<Address>(send_table.keySet());
        non_members.addAll(recv_table.keySet());

        members = new_members;
        non_members.removeAll(new_members);
        if (cache != null) cache.removeAll(new_members);

        if (!non_members.isEmpty()) {
          if (log.isTraceEnabled()) log.trace("removing non members " + non_members);
          for (Address non_mbr : non_members) removeConnection(non_mbr);
        }
        break;

      case Event.SET_LOCAL_ADDRESS:
        local_addr = (Address) evt.getArg();
        break;
    }

    return down_prot.down(evt); // Pass on to the layer below us
  }