Exemplo n.º 1
0
 private void startReceiver() {
   if (receiver == null || !receiver.isAlive()) {
     receiver = new Thread(Util.getGlobalThreadGroup(), this, "ReceiverThread");
     receiver.setDaemon(true);
     receiver.start();
     if (log.isTraceEnabled()) log.trace("receiver thread started");
   }
 }
Exemplo n.º 2
0
public class TCPConnectionMap {

  private final Mapper mapper;
  private final InetAddress bind_addr;
  private final Address local_addr; // bind_addr + port of srv_sock
  private final ThreadGroup thread_group =
      new ThreadGroup(Util.getGlobalThreadGroup(), "ConnectionMap");
  private final ServerSocket srv_sock;
  private Receiver receiver;
  private final long conn_expire_time;
  private final Log log = LogFactory.getLog(getClass());
  private int recv_buf_size = 120000;
  private int send_buf_size = 60000;
  private int send_queue_size = 0;
  private int sock_conn_timeout =
      1000; // max time in millis to wait for Socket.connect() to return
  private boolean tcp_nodelay = false;
  private int linger = -1;
  private final Thread acceptor;
  private final AtomicBoolean running = new AtomicBoolean(false);
  private volatile boolean use_send_queues = false;
  protected SocketFactory socket_factory = new DefaultSocketFactory();

  public TCPConnectionMap(
      String service_name,
      ThreadFactory f,
      SocketFactory socket_factory,
      Receiver r,
      InetAddress bind_addr,
      InetAddress external_addr,
      int srv_port,
      int max_port)
      throws Exception {
    this(service_name, f, socket_factory, r, bind_addr, external_addr, srv_port, max_port, 0, 0);
  }

  public TCPConnectionMap(
      String service_name,
      ThreadFactory f,
      Receiver r,
      InetAddress bind_addr,
      InetAddress external_addr,
      int srv_port,
      int max_port,
      long reaper_interval,
      long conn_expire_time)
      throws Exception {
    this(
        service_name,
        f,
        null,
        r,
        bind_addr,
        external_addr,
        srv_port,
        max_port,
        reaper_interval,
        conn_expire_time);
  }

  public TCPConnectionMap(
      String service_name,
      ThreadFactory f,
      SocketFactory socket_factory,
      Receiver r,
      InetAddress bind_addr,
      InetAddress external_addr,
      int srv_port,
      int max_port,
      long reaper_interval,
      long conn_expire_time)
      throws Exception {
    this.mapper = new Mapper(f, reaper_interval);
    this.receiver = r;
    this.bind_addr = bind_addr;
    this.conn_expire_time = conn_expire_time;
    if (socket_factory != null) this.socket_factory = socket_factory;
    this.srv_sock =
        Util.createServerSocket(this.socket_factory, service_name, bind_addr, srv_port, max_port);

    if (external_addr != null) local_addr = new IpAddress(external_addr, srv_sock.getLocalPort());
    else if (bind_addr != null) local_addr = new IpAddress(bind_addr, srv_sock.getLocalPort());
    else local_addr = new IpAddress(srv_sock.getLocalPort());

    acceptor = f.newThread(thread_group, new ConnectionAcceptor(), "ConnectionMap.Acceptor");
  }

  public Address getLocalAddress() {
    return local_addr;
  }

  public Receiver getReceiver() {
    return receiver;
  }

  public void setReceiver(Receiver receiver) {
    this.receiver = receiver;
  }

  public SocketFactory getSocketFactory() {
    return socket_factory;
  }

  public void setSocketFactory(SocketFactory socket_factory) {
    this.socket_factory = socket_factory;
  }

  public void addConnectionMapListener(
      AbstractConnectionMap.ConnectionMapListener<TCPConnection> l) {
    mapper.addConnectionMapListener(l);
  }

  public void removeConnectionMapListener(
      AbstractConnectionMap.ConnectionMapListener<TCPConnection> l) {
    mapper.removeConnectionMapListener(l);
  }

  /**
   * Calls the receiver callback. We do not serialize access to this method, and it may be called
   * concurrently by several Connection handler threads. Therefore the receiver needs to be
   * reentrant.
   */
  public void receive(Address sender, byte[] data, int offset, int length) {
    receiver.receive(sender, data, offset, length);
  }

  public void send(Address dest, byte[] data, int offset, int length) throws Exception {
    if (dest == null) {
      if (log.isErrorEnabled()) log.error("destination is null");
      return;
    }

    if (data == null) {
      log.warn("data is null; discarding packet");
      return;
    }

    if (!running.get()) {
      if (log.isDebugEnabled())
        log.debug("connection table is not running, discarding message to " + dest);
      return;
    }

    if (dest.equals(local_addr)) {
      receive(local_addr, data, offset, length);
      return;
    }

    // 1. Try to obtain correct Connection (or create one if not yet existent)
    TCPConnection conn;
    conn = mapper.getConnection(dest);

    // 2. Send the message using that connection
    if (conn != null) {
      try {
        conn.send(data, offset, length);
      } catch (Exception ex) {
        mapper.removeConnection(dest);
        throw ex;
      }
    }
  }

  public void start() throws Exception {
    if (running.compareAndSet(false, true)) {
      acceptor.start();
      mapper.start();
    }
  }

  public void stop() {
    if (running.compareAndSet(true, false)) {
      try {
        getSocketFactory().close(srv_sock);
      } catch (IOException e) {
      }
      Util.interruptAndWaitToDie(acceptor);
      mapper.stop();
    }
  }

  private void setSocketParameters(Socket client_sock) throws SocketException {
    if (log.isTraceEnabled())
      log.trace(
          "["
              + local_addr
              + "] accepted connection from "
              + client_sock.getInetAddress()
              + ":"
              + client_sock.getPort());
    try {
      client_sock.setSendBufferSize(send_buf_size);
    } catch (IllegalArgumentException ex) {
      if (log.isErrorEnabled())
        log.error("exception setting send buffer size to " + send_buf_size + " bytes", ex);
    }
    try {
      client_sock.setReceiveBufferSize(recv_buf_size);
    } catch (IllegalArgumentException ex) {
      if (log.isErrorEnabled())
        log.error("exception setting receive buffer size to " + send_buf_size + " bytes", ex);
    }

    client_sock.setKeepAlive(true);
    client_sock.setTcpNoDelay(tcp_nodelay);
    if (linger > 0) client_sock.setSoLinger(true, linger);
    else client_sock.setSoLinger(false, -1);
  }

  /** Used for message reception. */
  public interface Receiver {
    void receive(Address sender, byte[] data, int offset, int length);
  }

  private class ConnectionAcceptor implements Runnable {

    /**
     * Acceptor thread. Continuously accept new connections. Create a new thread for each new
     * connection and put it in conns. When the thread should stop, it is interrupted by the thread
     * creator.
     */
    public void run() {
      while (!srv_sock.isClosed() && !Thread.currentThread().isInterrupted()) {
        TCPConnection conn = null;
        Socket client_sock = null;
        try {
          client_sock = srv_sock.accept();
          conn = new TCPConnection(client_sock);
          Address peer_addr = conn.getPeerAddress();
          mapper.getLock().lock();
          try {
            boolean currentConnectionOpen = mapper.hasOpenConnection(peer_addr);
            boolean replaceWithNewConnection = false;
            if (currentConnectionOpen) {
              replaceWithNewConnection = peer_addr.compareTo(local_addr) > 0;
            }
            if (!currentConnectionOpen || replaceWithNewConnection) {
              mapper.removeConnection(peer_addr);
              mapper.addConnection(peer_addr, conn);
              conn.start(mapper.getThreadFactory()); // starts handler thread on this socket
            } else {
              Util.close(conn);
            }
          } finally {
            mapper.getLock().unlock();
          }
        } catch (SocketException se) {
          boolean threadExiting = srv_sock.isClosed() || Thread.currentThread().isInterrupted();
          if (threadExiting) {
            break;
          } else {
            if (log.isWarnEnabled()) log.warn("Could not accept connection from peer ", se);
            Util.close(conn);
            Util.close(client_sock);
          }
        } catch (Exception ex) {
          if (log.isWarnEnabled()) log.warn("Could not read accept connection from peer " + ex);
          Util.close(conn);
          Util.close(client_sock);
        }
      }
      if (log.isTraceEnabled()) log.trace(Thread.currentThread().getName() + " terminated");
    }
  }

  public void setReceiveBufferSize(int recv_buf_size) {
    this.recv_buf_size = recv_buf_size;
  }

  public void setSocketConnectionTimeout(int sock_conn_timeout) {
    this.sock_conn_timeout = sock_conn_timeout;
  }

  public void setSendBufferSize(int send_buf_size) {
    this.send_buf_size = send_buf_size;
  }

  public void setLinger(int linger) {
    this.linger = linger;
  }

  public void setTcpNodelay(boolean tcp_nodelay) {
    this.tcp_nodelay = tcp_nodelay;
  }

  public void setSendQueueSize(int send_queue_size) {
    this.send_queue_size = send_queue_size;
  }

  public void setUseSendQueues(boolean use_send_queues) {
    this.use_send_queues = use_send_queues;
  }

  public int getNumConnections() {
    return mapper.getNumConnections();
  }

  public boolean connectionEstablishedTo(Address addr) {
    return mapper.connectionEstablishedTo(addr);
  }

  public String printConnections() {
    return mapper.printConnections();
  }

  public void retainAll(Collection<Address> members) {
    mapper.retainAll(members);
  }

  public long getConnectionExpiryTimeout() {
    return conn_expire_time;
  }

  public int getSenderQueueSize() {
    return send_queue_size;
  }

  public String toString() {
    StringBuilder ret = new StringBuilder();
    ret.append("local_addr=" + local_addr).append("\n");
    ret.append("connections (" + mapper.size() + "):\n");
    ret.append(mapper.toString());
    ret.append('\n');
    return ret.toString();
  }

  public class TCPConnection implements Connection {

    private final Socket sock; // socket to/from peer (result of srv_sock.accept() or new Socket())
    private final Lock send_lock = new ReentrantLock(); // serialize send()
    private final Log log = LogFactory.getLog(getClass());
    private final byte[] cookie = {'b', 'e', 'l', 'a'};
    private final DataOutputStream out;
    private final DataInputStream in;
    private final Address peer_addr; // address of the 'other end' of the connection
    private final int peer_addr_read_timeout =
        2000; // max time in milliseconds to block on reading peer address
    private long last_access =
        System.currentTimeMillis(); // last time a message was sent or received
    private Sender sender;
    private ConnectionPeerReceiver connectionPeerReceiver;
    private AtomicBoolean active = new AtomicBoolean(false);

    TCPConnection(Address peer_addr) throws Exception {
      if (peer_addr == null)
        throw new IllegalArgumentException("Invalid parameter peer_addr=" + peer_addr);
      SocketAddress destAddr =
          new InetSocketAddress(
              ((IpAddress) peer_addr).getIpAddress(), ((IpAddress) peer_addr).getPort());
      this.sock = new Socket();
      this.sock.bind(new InetSocketAddress(bind_addr, 0));
      Util.connect(this.sock, destAddr, sock_conn_timeout);
      setSocketParameters(sock);
      this.out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream()));
      this.in = new DataInputStream(new BufferedInputStream(sock.getInputStream()));
      sendLocalAddress(getLocalAddress());
      this.peer_addr = peer_addr;
    }

    TCPConnection(Socket s) throws Exception {
      if (s == null) throw new IllegalArgumentException("Invalid parameter s=" + s);
      setSocketParameters(s);
      this.out = new DataOutputStream(new BufferedOutputStream(s.getOutputStream()));
      this.in = new DataInputStream(new BufferedInputStream(s.getInputStream()));
      this.peer_addr = readPeerAddress(s);
      this.sock = s;
    }

    private Address getPeerAddress() {
      return peer_addr;
    }

    private void updateLastAccessed() {
      last_access = System.currentTimeMillis();
    }

    private void start(ThreadFactory f) {
      // only start once....
      if (active.compareAndSet(false, true)) {
        connectionPeerReceiver = new ConnectionPeerReceiver(f);
        connectionPeerReceiver.start();

        if (isSenderUsed()) {
          sender = new Sender(f, getSenderQueueSize());
          sender.start();
        }
      }
    }

    private boolean isSenderUsed() {
      return getSenderQueueSize() > 0 && use_send_queues;
    }

    private String getSockAddress() {
      StringBuilder sb = new StringBuilder();
      if (sock != null) {
        sb.append(sock.getLocalAddress().getHostAddress()).append(':').append(sock.getLocalPort());
        sb.append(" - ")
            .append(sock.getInetAddress().getHostAddress())
            .append(':')
            .append(sock.getPort());
      }
      return sb.toString();
    }

    /**
     * @param data Guaranteed to be non null
     * @param offset
     * @param length
     */
    private void send(byte[] data, int offset, int length) throws Exception {
      if (isSenderUsed()) {
        // we need to copy the byte[] buffer here because the original buffer might get
        // changed meanwhile
        byte[] tmp = new byte[length];
        System.arraycopy(data, offset, tmp, 0, length);
        sender.addToQueue(tmp);
      } else {
        _send(data, offset, length, true);
      }
    }

    /**
     * Sends data using the 'out' output stream of the socket
     *
     * @param data
     * @param offset
     * @param length
     * @param acquire_lock
     * @throws Exception
     */
    private void _send(byte[] data, int offset, int length, boolean acquire_lock) throws Exception {
      if (acquire_lock) send_lock.lock();

      try {
        doSend(data, offset, length);
        updateLastAccessed();
      } catch (InterruptedException iex) {
        Thread.currentThread().interrupt(); // set interrupt flag again
      } finally {
        if (acquire_lock) send_lock.unlock();
      }
    }

    private void doSend(byte[] data, int offset, int length) throws Exception {
      // we're using 'double-writes', sending the buffer to the destination in 2 pieces. this would
      // ensure that, if the peer closed the connection while we were idle, we would get an
      // exception.
      // this won't happen if we use a single write (see Stevens, ch. 5.13).

      out.writeInt(length); // write the length of the data buffer first
      Util.doubleWrite(data, offset, length, out);
      out.flush(); // may not be very efficient (but safe)
    }

    /**
     * Reads the peer's address. First a cookie has to be sent which has to match my own cookie,
     * otherwise the connection will be refused
     */
    private Address readPeerAddress(Socket client_sock) throws Exception {
      int timeout = client_sock.getSoTimeout();
      client_sock.setSoTimeout(peer_addr_read_timeout);

      try {
        // read the cookie first
        byte[] input_cookie = new byte[cookie.length];
        in.readFully(input_cookie, 0, input_cookie.length);
        if (!matchCookie(input_cookie))
          throw new SocketException(
              "ConnectionMap.Connection.readPeerAddress(): cookie read by "
                  + getLocalAddress()
                  + " does not match own cookie; terminating connection");
        // then read the version
        short version = in.readShort();

        if (!Version.isBinaryCompatible(version)) {
          if (log.isWarnEnabled())
            log.warn(
                new StringBuilder("packet from ")
                    .append(client_sock.getInetAddress())
                    .append(':')
                    .append(client_sock.getPort())
                    .append(" has different version (")
                    .append(Version.print(version))
                    .append(") from ours (")
                    .append(Version.printVersion())
                    .append("). This may cause problems")
                    .toString());
        }
        Address client_peer_addr = new IpAddress();
        client_peer_addr.readFrom(in);

        updateLastAccessed();
        return client_peer_addr;
      } finally {
        client_sock.setSoTimeout(timeout);
      }
    }

    /**
     * Send the cookie first, then the our port number. If the cookie doesn't match the receiver's
     * cookie, the receiver will reject the connection and close it.
     *
     * @throws IOException
     */
    private void sendLocalAddress(Address local_addr) throws IOException {
      // write the cookie
      out.write(cookie, 0, cookie.length);

      // write the version
      out.writeShort(Version.version);
      local_addr.writeTo(out);
      out.flush(); // needed ?
      updateLastAccessed();
    }

    private boolean matchCookie(byte[] input) {
      if (input == null || input.length < cookie.length) return false;
      for (int i = 0; i < cookie.length; i++) if (cookie[i] != input[i]) return false;
      return true;
    }

    private class ConnectionPeerReceiver implements Runnable {
      private Thread recv;
      private final AtomicBoolean receiving = new AtomicBoolean(false);

      public ConnectionPeerReceiver(ThreadFactory f) {
        recv = f.newThread(this, "Connection.Receiver [" + getSockAddress() + "]");
      }

      public void start() {
        if (receiving.compareAndSet(false, true)) {
          recv.start();
        }
      }

      public void stop() {
        if (receiving.compareAndSet(true, false)) {
          recv.interrupt();
        }
      }

      public boolean isRunning() {
        return receiving.get();
      }

      public boolean canRun() {
        return isRunning() && isConnected();
      }

      public void run() {
        try {
          while (!Thread.currentThread().isInterrupted() && canRun()) {
            try {
              int len = in.readInt();
              byte[] buf = new byte[len];
              in.readFully(buf, 0, len);
              updateLastAccessed();
              receiver.receive(peer_addr, buf, 0, len);
            } catch (OutOfMemoryError mem_ex) {
              break; // continue;
            } catch (IOException io_ex) {
              break;
            } catch (Throwable e) {
            }
          }
        } finally {
          Util.close(TCPConnection.this);
        }
      }
    }

    private class Sender implements Runnable {

      final BlockingQueue<byte[]> send_queue;
      final Thread runner;
      private final AtomicBoolean running = new AtomicBoolean(false);

      public Sender(ThreadFactory tf, int send_queue_size) {
        this.runner = tf.newThread(this, "Connection.Sender [" + getSockAddress() + "]");
        this.send_queue = new LinkedBlockingQueue<byte[]>(send_queue_size);
      }

      public void addToQueue(byte[] data) throws Exception {
        if (canRun()) send_queue.add(data);
      }

      public void start() {
        if (running.compareAndSet(false, true)) {
          runner.start();
        }
      }

      public void stop() {
        if (running.compareAndSet(true, false)) {
          runner.interrupt();
        }
      }

      public boolean isRunning() {
        return running.get();
      }

      public boolean canRun() {
        return isRunning() && isConnected();
      }

      public void run() {
        try {
          while (!Thread.currentThread().isInterrupted() && canRun()) {
            byte[] data = null;
            try {
              data = send_queue.take();
            } catch (InterruptedException e) {
              // Thread.currentThread().interrupt();
              break;
            }

            if (data != null) {
              try {
                _send(data, 0, data.length, false);
              } catch (Throwable ignored) {
              }
            }
          }
        } finally {
          Util.close(TCPConnection.this);
        }
        if (log.isTraceEnabled())
          log.trace("TCPConnection.Sender thread terminated at " + local_addr);
      }
    }

    public String toString() {
      StringBuilder ret = new StringBuilder();
      InetAddress local = null, remote = null;
      String local_str, remote_str;

      Socket tmp_sock = sock;
      if (tmp_sock == null) ret.append("<null socket>");
      else {
        // since the sock variable gets set to null we want to make
        // make sure we make it through here without a nullpointer exception
        local = tmp_sock.getLocalAddress();
        remote = tmp_sock.getInetAddress();
        local_str = local != null ? Util.shortName(local) : "<null>";
        remote_str = remote != null ? Util.shortName(remote) : "<null>";
        ret.append(
            '<'
                + local_str
                + ':'
                + tmp_sock.getLocalPort()
                + " --> "
                + remote_str
                + ':'
                + tmp_sock.getPort()
                + "> ("
                + ((System.currentTimeMillis() - last_access) / 1000)
                + " secs old)");
      }
      tmp_sock = null;

      return ret.toString();
    }

    public boolean isExpired(long now) {
      return getConnectionExpiryTimeout() > 0 && now - last_access >= getConnectionExpiryTimeout();
    }

    public boolean isConnected() {
      return !sock.isClosed() && sock.isConnected();
    }

    public boolean isOpen() {
      return isConnected()
          && (!isSenderUsed() || sender.isRunning())
          && (connectionPeerReceiver != null && connectionPeerReceiver.isRunning());
    }

    public void close() throws IOException {
      // can close even if start was never called...
      send_lock.lock();
      try {
        connectionPeerReceiver.stop();
        if (isSenderUsed()) {
          sender.stop();
        }
        Util.close(sock);
        Util.close(out);
        Util.close(in);
      } finally {
        send_lock.unlock();
      }
      mapper.notifyConnectionClosed(peer_addr);
    }
  }

  private class Mapper extends AbstractConnectionMap<TCPConnection> {

    public Mapper(ThreadFactory factory) {
      super(factory);
    }

    public Mapper(ThreadFactory factory, long reaper_interval) {
      super(factory, reaper_interval);
    }

    public TCPConnection getConnection(Address dest) throws Exception {
      TCPConnection conn = null;
      getLock().lock();
      try {
        conn = conns.get(dest);
        if (conn != null && conn.isOpen()) return conn;
        try {
          conn = new TCPConnection(dest);
          conn.start(getThreadFactory());
          addConnection(dest, conn);
          if (log.isTraceEnabled()) log.trace("created socket to " + dest);
        } catch (Exception ex) {
          if (log.isTraceEnabled()) log.trace("failed creating connection to " + dest);
        }
      } finally {
        getLock().unlock();
      }
      return conn;
    }

    public boolean connectionEstablishedTo(Address address) {
      lock.lock();
      try {
        TCPConnection conn = conns.get(address);
        return conn != null && conn.isConnected();
      } finally {
        lock.unlock();
      }
    }

    public int size() {
      return conns.size();
    }

    public String toString() {
      StringBuilder sb = new StringBuilder();

      getLock().lock();
      try {
        for (Map.Entry<Address, TCPConnection> entry : conns.entrySet()) {
          sb.append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
        }
        return sb.toString();
      } finally {
        getLock().unlock();
      }
    }
  }
}