Ejemplo n.º 1
0
    /**
     * writes all the entries that have changed, to the buffer which will later be written to TCP/IP
     *
     * @param modificationIterator a record of which entries have modification
     * @param selectionKey
     */
    void entriesToBuffer(
        @NotNull final ModificationIterator modificationIterator,
        @NotNull final SelectionKey selectionKey)
        throws InterruptedException, IOException {

      final SocketChannel socketChannel = (SocketChannel) selectionKey.channel();
      final Attached attached = (Attached) selectionKey.attachment();

      // this can occur when new SHM's are added to a cluster
      final boolean handShakingComplete = attached.isHandShakingComplete();

      for (; ; ) {

        final boolean wasDataRead = modificationIterator.nextEntry(entryCallback, 0);

        if (!wasDataRead) {

          // if we have no more data to write to the socket then we will
          // un-register OP_WRITE on the selector, until more data becomes available
          if (in.position() == 0 && handShakingComplete) disableWrite(socketChannel, attached);

          return;
        }

        // we've filled up the buffer lets give another channel a chance to send some data
        if (in.remaining() <= maxEntrySizeBytes) return;

        // if we have space in the buffer to write more data and we just wrote data into the
        // buffer then let try and write some more
      }
    }
Ejemplo n.º 2
0
  /**
   * check to see if we have lost connection with the remote node and if we have attempts a
   * reconnect.
   *
   * @param key the key relating to the heartbeat that we are checking
   * @param approxTimeOutTime the approximate time in milliseconds
   * @throws ConnectException
   */
  private void heartbeatCheckHasReceived(
      @NotNull final SelectionKey key, final long approxTimeOutTime) throws ConnectException {

    final Attached attached = (Attached) key.attachment();

    // we wont attempt to reconnect the server socket
    if (attached.isServer || !attached.isHandShakingComplete()) return;

    final SocketChannel channel = (SocketChannel) key.channel();

    if (approxTimeOutTime
        > attached.entryReader.lastHeartBeatReceived + attached.remoteHeartbeatInterval) {
      if (LOG.isDebugEnabled())
        LOG.debug(
            "lost connection, attempting to reconnect. "
                + "missed heartbeat from identifier="
                + attached.remoteIdentifier);
      try {
        channel.socket().close();
        channel.close();
        activeKeys.clear(attached.remoteIdentifier);
        closeables.remove(channel);
      } catch (IOException e) {
        LOG.debug("", e);
      }

      attached.connector.connectLater();
    }
  }
Ejemplo n.º 3
0
  /**
   * checks that we receive heartbeats and send out heart beats.
   *
   * @param approxTime the approximate time in milliseconds
   */
  void heartBeatMonitor(long approxTime) {
    for (int i = activeKeys.nextSetBit(0); i >= 0; i = activeKeys.nextSetBit(i + 1)) {
      try {
        final SelectionKey key = selectionKeysStore[i];
        if (!key.isValid() || !key.channel().isOpen()) {
          activeKeys.clear(i);
          continue;
        }
        try {
          sendHeartbeatIfRequired(approxTime, key);
        } catch (Exception e) {
          if (LOG.isDebugEnabled()) LOG.debug("", e);
        }

        try {
          heartbeatCheckHasReceived(key, approxTime);
        } catch (Exception e) {
          if (LOG.isDebugEnabled()) LOG.debug("", e);
        }

      } catch (Exception e) {
        if (LOG.isDebugEnabled()) LOG.debug("", e);
      }
    }
  }
Ejemplo n.º 4
0
 public void applyUpdates() {
   if (wasChanged.getAndSet(false)) {
     for (int i = changeOfOpWriteRequired.nextSetBit(0);
         i >= 0;
         i = changeOfOpWriteRequired.nextSetBit(i + 1)) {
       changeOfOpWriteRequired.clear(i);
       final SelectionKey key = selectionKeys[i];
       try {
         key.interestOps(key.interestOps() | op);
       } catch (Exception e) {
         LOG.debug("", e);
       }
     }
   }
 }
Ejemplo n.º 5
0
  /** called when the selector receives a OP_WRITE message */
  private void onWrite(@NotNull final SelectionKey key, final long approxTime)
      throws InterruptedException, IOException {
    final SocketChannel socketChannel = (SocketChannel) key.channel();
    final Attached attached = (Attached) key.attachment();

    if (attached.remoteModificationIterator != null)
      attached.entryWriter.entriesToBuffer(attached.remoteModificationIterator, key);

    try {
      int bytesJustWritten = attached.entryWriter.writeBufferToSocket(socketChannel, approxTime);

      contemplateThrottleWrites(bytesJustWritten);

    } catch (IOException e) {
      quietClose(key, e);
      if (!attached.isServer) attached.connector.connectLater();
      throw e;
    }
  }
Ejemplo n.º 6
0
    /**
     * removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE will
     * get added back in as soon as we have data to write
     *
     * @param socketChannel the socketChannel we wish to stop writing to
     * @param attached data associated with the socketChannels key
     */
    public synchronized void disableWrite(
        @NotNull final SocketChannel socketChannel, @NotNull final Attached attached) {
      try {
        SelectionKey key = socketChannel.keyFor(selector);
        if (key != null) {
          if (attached.isHandShakingComplete() && selector.isOpen()) {
            if (LOG.isDebugEnabled())
              LOG.debug(
                  "Disabling OP_WRITE to remoteIdentifier="
                      + attached.remoteIdentifier
                      + ", localIdentifier="
                      + localIdentifier);
            key.interestOps(key.interestOps() & ~OP_WRITE);
          }
        }

      } catch (Exception e) {
        LOG.error("", e);
      }
    }
Ejemplo n.º 7
0
  /** called when the selector receives a OP_CONNECT message */
  private void onConnect(@NotNull final SelectionKey key) throws IOException, InterruptedException {

    final SocketChannel channel = (SocketChannel) key.channel();
    final Attached attached = (Attached) key.attachment();

    try {
      if (!channel.finishConnect()) {
        return;
      }
    } catch (SocketException e) {
      quietClose(key, e);
      attached.connector.connect();
      throw e;
    }

    attached.connector.setSuccessfullyConnected();

    if (LOG.isDebugEnabled())
      LOG.debug(
          "successfully connected to {}, local-id={}",
          channel.socket().getInetAddress(),
          localIdentifier);

    channel.configureBlocking(false);
    channel.socket().setTcpNoDelay(true);
    channel.socket().setSoTimeout(0);
    channel.socket().setSoLinger(false, 0);

    attached.entryReader = new TcpSocketChannelEntryReader();
    attached.entryWriter = new TcpSocketChannelEntryWriter();

    key.interestOps(OP_WRITE | OP_READ);

    throttle(channel);

    // register it with the selector and store the ModificationIterator for this key
    attached.entryWriter.identifierToBuffer(localIdentifier);
  }
Ejemplo n.º 8
0
  /** called when the selector receives a OP_READ message */
  private void onRead(final SelectionKey key, final long approxTime)
      throws IOException, InterruptedException {

    final SocketChannel socketChannel = (SocketChannel) key.channel();
    final Attached attached = (Attached) key.attachment();

    try {
      if (attached.entryReader.readSocketToBuffer(socketChannel) <= 0) return;

    } catch (IOException e) {
      if (!attached.isServer) attached.connector.connectLater();
      throw e;
    }

    if (LOG.isDebugEnabled()) LOG.debug("heartbeat or data received.");

    attached.entryReader.lastHeartBeatReceived = approxTime;

    if (attached.isHandShakingComplete()) {
      attached.entryReader.entriesFromBuffer();
    } else {
      doHandShaking(key);
    }
  }
Ejemplo n.º 9
0
  /**
   * check to see if its time to send a heartbeat, and send one if required
   *
   * @param approxTime the current time ( approximately )
   * @param key nio selection key
   */
  private void sendHeartbeatIfRequired(final long approxTime, @NotNull final SelectionKey key) {

    final Attached attachment = (Attached) key.attachment();

    if (attachment.isHandShakingComplete()
        && attachment.entryWriter.lastSentTime + heartBeatInterval < approxTime) {

      attachment.entryWriter.lastSentTime = approxTime;
      attachment.entryWriter.writeHeartbeatToBuffer();

      enableOpWrite(key);

      if (LOG.isDebugEnabled()) LOG.debug("sending heartbeat");
    }
  }
Ejemplo n.º 10
0
  /** called when the selector receives a OP_ACCEPT message */
  private void onAccept(@NotNull final SelectionKey key) throws IOException {

    final ServerSocketChannel server = (ServerSocketChannel) key.channel();
    final SocketChannel channel = server.accept();
    channel.configureBlocking(false);
    channel.socket().setReuseAddress(true);
    channel.socket().setTcpNoDelay(true);
    channel.socket().setSoTimeout(0);
    channel.socket().setSoLinger(false, 0);

    final Attached attached = new Attached();
    channel.register(selector, OP_WRITE | OP_READ, attached);

    throttle(channel);

    attached.entryReader = new TcpSocketChannelEntryReader();
    attached.entryWriter = new TcpSocketChannelEntryWriter();

    attached.isServer = true;
    attached.entryWriter.identifierToBuffer(localIdentifier);
  }
Ejemplo n.º 11
0
  /**
   * used to exchange identifiers and timestamps and heartbeat intervals between the server and
   * client
   *
   * @param key the SelectionKey relating to the this cha
   * @throws java.io.IOException
   * @throws InterruptedException
   */
  private void doHandShaking(@NotNull final SelectionKey key)
      throws IOException, InterruptedException {
    final Attached attached = (Attached) key.attachment();
    final TcpSocketChannelEntryWriter writer = attached.entryWriter;
    final TcpSocketChannelEntryReader reader = attached.entryReader;

    if (attached.remoteIdentifier == Byte.MIN_VALUE) {

      final byte remoteIdentifier = reader.identifierFromBuffer();

      if (remoteIdentifier == Byte.MIN_VALUE) return;

      attached.remoteIdentifier = remoteIdentifier;

      // we use the as iterating the activeKeys via the bitset wont create and Objects
      // but if we use the selector.keys() this will.
      selectionKeysStore[remoteIdentifier] = key;
      activeKeys.set(remoteIdentifier);

      if (LOG.isDebugEnabled()) {
        LOG.debug(
            "server-connection id={}, remoteIdentifier={}", localIdentifier, remoteIdentifier);
      }

      if (remoteIdentifier == localIdentifier) {
        throw new IllegalStateException(
            "Where are connecting to a remote "
                + "map with the same "
                + "identifier as this map, identifier="
                + localIdentifier
                + ", "
                + "please change either this maps identifier or the remote one");
      }

      attached.remoteModificationIterator =
          replica.acquireModificationIterator(remoteIdentifier, attached);

      writer.writeRemoteBootstrapTimestamp(replica.lastModificationTime(remoteIdentifier));

      // tell the remote node, what are heartbeat interval is
      writer.writeRemoteHeartbeatInterval(heartBeatInterval);
    }

    if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) {
      attached.remoteBootstrapTimestamp = reader.remoteBootstrapTimestamp();
      if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) return;
    }

    if (!attached.hasRemoteHeartbeatInterval) {

      long value = reader.remoteHeartbeatIntervalFromBuffer();

      if (value == Long.MIN_VALUE) return;

      if (value < 0) {
        LOG.error("value=" + value);
      }

      // we add a 10% safety margin to the timeout time due to latency fluctuations on the network,
      // in other words we wont consider a connection to have
      // timed out, unless the heartbeat interval has exceeded 25% of the expected time.
      attached.remoteHeartbeatInterval = (long) (value * 1.25);

      // we have to make our selector poll interval at least as short as the minimum selector
      // timeout
      selectorTimeout = Math.min(selectorTimeout, value);

      if (selectorTimeout < 0) LOG.info("");

      attached.hasRemoteHeartbeatInterval = true;

      // now we're finished we can get on with reading the entries
      attached.setHandShakingComplete();
      attached.remoteModificationIterator.dirtyEntries(attached.remoteBootstrapTimestamp);
      reader.entriesFromBuffer();
    }
  }
Ejemplo n.º 12
0
 /**
  * closes and only logs the exception at debug
  *
  * @param key the SelectionKey
  * @param e the Exception that caused the issue
  */
 private void quietClose(@NotNull final SelectionKey key, @NotNull final Exception e) {
   if (LOG.isDebugEnabled()) LOG.debug("", e);
   closeEarlyAndQuietly(key.channel());
 }
Ejemplo n.º 13
0
 private void enableOpWrite(SelectionKey key) {
   int ops = key.interestOps();
   if ((ops & (OP_CONNECT | OP_ACCEPT)) == 0) key.interestOps(ops | OP_WRITE);
 }
Ejemplo n.º 14
0
  @Override
  void process() throws IOException {
    try {
      final Details serverDetails = new Details(serverInetSocketAddress, localIdentifier);
      connectorBySocket.put(serverInetSocketAddress, new ServerConnector(serverDetails));

      for (InetSocketAddress client : endpoints) {
        final Details clientDetails = new Details(client, localIdentifier);
        connectorBySocket.put(client, new ClientConnector(clientDetails));
      }

      for (AbstractConnector connector : connectorBySocket.values()) {
        connector.connect();
      }

      while (selector.isOpen()) {

        registerPendingRegistrations();

        final int nSelectedKeys = selector.select(selectorTimeout);

        // its less resource intensive to set this less frequently and use an approximation
        final long approxTime = System.currentTimeMillis();

        checkThrottleInterval();

        // check that we have sent and received heartbeats
        heartBeatMonitor(approxTime);

        // set the OP_WRITE when data is ready to send
        opWriteUpdater.applyUpdates();

        if (nSelectedKeys == 0) continue; // go back and check pendingRegistrations

        final Set<SelectionKey> selectionKeys = selector.selectedKeys();
        for (final SelectionKey key : selectionKeys) {
          try {

            if (!key.isValid()) continue;

            if (key.isAcceptable()) onAccept(key);

            if (key.isConnectable()) onConnect(key);

            if (key.isReadable()) onRead(key, approxTime);

            if (key.isWritable()) onWrite(key, approxTime);

          } catch (CancelledKeyException e) {
            quietClose(key, e);
          } catch (ClosedSelectorException e) {
            quietClose(key, e);
          } catch (IOException e) {
            quietClose(key, e);
          } catch (InterruptedException e) {
            quietClose(key, e);
          } catch (Exception e) {
            LOG.info("", e);
            closeEarlyAndQuietly(key.channel());
          }
        }
        selectionKeys.clear();
      }
    } catch (CancelledKeyException e) {
      if (LOG.isDebugEnabled()) LOG.debug("", e);
    } catch (ClosedSelectorException e) {
      if (LOG.isDebugEnabled()) LOG.debug("", e);
    } catch (ClosedChannelException e) {
      if (LOG.isDebugEnabled()) LOG.debug("", e);
    } catch (ConnectException e) {
      if (LOG.isDebugEnabled()) LOG.debug("", e);
    } catch (Exception e) {
      LOG.error("", e);
    } finally {
      if (selector != null)
        try {
          selector.close();
        } catch (IOException e) {
          if (LOG.isDebugEnabled()) LOG.debug("", e);
        }
      close();
    }
  }