/** * check to see if we have lost connection with the remote node and if we have attempts a * reconnect. * * @param key the key relating to the heartbeat that we are checking * @param approxTimeOutTime the approximate time in milliseconds * @throws ConnectException */ private void heartbeatCheckHasReceived( @NotNull final SelectionKey key, final long approxTimeOutTime) throws ConnectException { final Attached attached = (Attached) key.attachment(); // we wont attempt to reconnect the server socket if (attached.isServer || !attached.isHandShakingComplete()) return; final SocketChannel channel = (SocketChannel) key.channel(); if (approxTimeOutTime > attached.entryReader.lastHeartBeatReceived + attached.remoteHeartbeatInterval) { if (LOG.isDebugEnabled()) LOG.debug( "lost connection, attempting to reconnect. " + "missed heartbeat from identifier=" + attached.remoteIdentifier); try { channel.socket().close(); channel.close(); activeKeys.clear(attached.remoteIdentifier); closeables.remove(channel); } catch (IOException e) { LOG.debug("", e); } attached.connector.connectLater(); } }
/** * writes all the entries that have changed, to the buffer which will later be written to TCP/IP * * @param modificationIterator a record of which entries have modification * @param selectionKey */ void entriesToBuffer( @NotNull final ModificationIterator modificationIterator, @NotNull final SelectionKey selectionKey) throws InterruptedException, IOException { final SocketChannel socketChannel = (SocketChannel) selectionKey.channel(); final Attached attached = (Attached) selectionKey.attachment(); // this can occur when new SHM's are added to a cluster final boolean handShakingComplete = attached.isHandShakingComplete(); for (; ; ) { final boolean wasDataRead = modificationIterator.nextEntry(entryCallback, 0); if (!wasDataRead) { // if we have no more data to write to the socket then we will // un-register OP_WRITE on the selector, until more data becomes available if (in.position() == 0 && handShakingComplete) disableWrite(socketChannel, attached); return; } // we've filled up the buffer lets give another channel a chance to send some data if (in.remaining() <= maxEntrySizeBytes) return; // if we have space in the buffer to write more data and we just wrote data into the // buffer then let try and write some more } }
/** * check to see if its time to send a heartbeat, and send one if required * * @param approxTime the current time ( approximately ) * @param key nio selection key */ private void sendHeartbeatIfRequired(final long approxTime, @NotNull final SelectionKey key) { final Attached attachment = (Attached) key.attachment(); if (attachment.isHandShakingComplete() && attachment.entryWriter.lastSentTime + heartBeatInterval < approxTime) { attachment.entryWriter.lastSentTime = approxTime; attachment.entryWriter.writeHeartbeatToBuffer(); enableOpWrite(key); if (LOG.isDebugEnabled()) LOG.debug("sending heartbeat"); } }
/** called when the selector receives a OP_ACCEPT message */ private void onAccept(@NotNull final SelectionKey key) throws IOException { final ServerSocketChannel server = (ServerSocketChannel) key.channel(); final SocketChannel channel = server.accept(); channel.configureBlocking(false); channel.socket().setReuseAddress(true); channel.socket().setTcpNoDelay(true); channel.socket().setSoTimeout(0); channel.socket().setSoLinger(false, 0); final Attached attached = new Attached(); channel.register(selector, OP_WRITE | OP_READ, attached); throttle(channel); attached.entryReader = new TcpSocketChannelEntryReader(); attached.entryWriter = new TcpSocketChannelEntryWriter(); attached.isServer = true; attached.entryWriter.identifierToBuffer(localIdentifier); }
/** called when the selector receives a OP_CONNECT message */ private void onConnect(@NotNull final SelectionKey key) throws IOException, InterruptedException { final SocketChannel channel = (SocketChannel) key.channel(); final Attached attached = (Attached) key.attachment(); try { if (!channel.finishConnect()) { return; } } catch (SocketException e) { quietClose(key, e); attached.connector.connect(); throw e; } attached.connector.setSuccessfullyConnected(); if (LOG.isDebugEnabled()) LOG.debug( "successfully connected to {}, local-id={}", channel.socket().getInetAddress(), localIdentifier); channel.configureBlocking(false); channel.socket().setTcpNoDelay(true); channel.socket().setSoTimeout(0); channel.socket().setSoLinger(false, 0); attached.entryReader = new TcpSocketChannelEntryReader(); attached.entryWriter = new TcpSocketChannelEntryWriter(); key.interestOps(OP_WRITE | OP_READ); throttle(channel); // register it with the selector and store the ModificationIterator for this key attached.entryWriter.identifierToBuffer(localIdentifier); }
/** called when the selector receives a OP_READ message */ private void onRead(final SelectionKey key, final long approxTime) throws IOException, InterruptedException { final SocketChannel socketChannel = (SocketChannel) key.channel(); final Attached attached = (Attached) key.attachment(); try { if (attached.entryReader.readSocketToBuffer(socketChannel) <= 0) return; } catch (IOException e) { if (!attached.isServer) attached.connector.connectLater(); throw e; } if (LOG.isDebugEnabled()) LOG.debug("heartbeat or data received."); attached.entryReader.lastHeartBeatReceived = approxTime; if (attached.isHandShakingComplete()) { attached.entryReader.entriesFromBuffer(); } else { doHandShaking(key); } }
/** * removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE will * get added back in as soon as we have data to write * * @param socketChannel the socketChannel we wish to stop writing to * @param attached data associated with the socketChannels key */ public synchronized void disableWrite( @NotNull final SocketChannel socketChannel, @NotNull final Attached attached) { try { SelectionKey key = socketChannel.keyFor(selector); if (key != null) { if (attached.isHandShakingComplete() && selector.isOpen()) { if (LOG.isDebugEnabled()) LOG.debug( "Disabling OP_WRITE to remoteIdentifier=" + attached.remoteIdentifier + ", localIdentifier=" + localIdentifier); key.interestOps(key.interestOps() & ~OP_WRITE); } } } catch (Exception e) { LOG.error("", e); } }
/** * used to exchange identifiers and timestamps and heartbeat intervals between the server and * client * * @param key the SelectionKey relating to the this cha * @throws java.io.IOException * @throws InterruptedException */ private void doHandShaking(@NotNull final SelectionKey key) throws IOException, InterruptedException { final Attached attached = (Attached) key.attachment(); final TcpSocketChannelEntryWriter writer = attached.entryWriter; final TcpSocketChannelEntryReader reader = attached.entryReader; if (attached.remoteIdentifier == Byte.MIN_VALUE) { final byte remoteIdentifier = reader.identifierFromBuffer(); if (remoteIdentifier == Byte.MIN_VALUE) return; attached.remoteIdentifier = remoteIdentifier; // we use the as iterating the activeKeys via the bitset wont create and Objects // but if we use the selector.keys() this will. selectionKeysStore[remoteIdentifier] = key; activeKeys.set(remoteIdentifier); if (LOG.isDebugEnabled()) { LOG.debug( "server-connection id={}, remoteIdentifier={}", localIdentifier, remoteIdentifier); } if (remoteIdentifier == localIdentifier) { throw new IllegalStateException( "Where are connecting to a remote " + "map with the same " + "identifier as this map, identifier=" + localIdentifier + ", " + "please change either this maps identifier or the remote one"); } attached.remoteModificationIterator = replica.acquireModificationIterator(remoteIdentifier, attached); writer.writeRemoteBootstrapTimestamp(replica.lastModificationTime(remoteIdentifier)); // tell the remote node, what are heartbeat interval is writer.writeRemoteHeartbeatInterval(heartBeatInterval); } if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) { attached.remoteBootstrapTimestamp = reader.remoteBootstrapTimestamp(); if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) return; } if (!attached.hasRemoteHeartbeatInterval) { long value = reader.remoteHeartbeatIntervalFromBuffer(); if (value == Long.MIN_VALUE) return; if (value < 0) { LOG.error("value=" + value); } // we add a 10% safety margin to the timeout time due to latency fluctuations on the network, // in other words we wont consider a connection to have // timed out, unless the heartbeat interval has exceeded 25% of the expected time. attached.remoteHeartbeatInterval = (long) (value * 1.25); // we have to make our selector poll interval at least as short as the minimum selector // timeout selectorTimeout = Math.min(selectorTimeout, value); if (selectorTimeout < 0) LOG.info(""); attached.hasRemoteHeartbeatInterval = true; // now we're finished we can get on with reading the entries attached.setHandShakingComplete(); attached.remoteModificationIterator.dirtyEntries(attached.remoteBootstrapTimestamp); reader.entriesFromBuffer(); } }