/**
* writes all the entries that have changed, to the buffer which will later be written to TCP/IP
*
* @param modificationIterator a record of which entries have modification
* @param selectionKey
*/
void entriesToBuffer(
@NotNull final ModificationIterator modificationIterator,
@NotNull final SelectionKey selectionKey)
throws InterruptedException, IOException {
final SocketChannel socketChannel = (SocketChannel) selectionKey.channel();
final Attached attached = (Attached) selectionKey.attachment();
// this can occur when new SHM's are added to a cluster
final boolean handShakingComplete = attached.isHandShakingComplete();
for (; ; ) {
final boolean wasDataRead = modificationIterator.nextEntry(entryCallback, 0);
if (!wasDataRead) {
// if we have no more data to write to the socket then we will
// un-register OP_WRITE on the selector, until more data becomes available
if (in.position() == 0 && handShakingComplete) disableWrite(socketChannel, attached);
return;
}
// we've filled up the buffer lets give another channel a chance to send some data
if (in.remaining() <= maxEntrySizeBytes) return;
// if we have space in the buffer to write more data and we just wrote data into the
// buffer then let try and write some more
}
}
/**
* check to see if we have lost connection with the remote node and if we have attempts a
* reconnect.
*
* @param key the key relating to the heartbeat that we are checking
* @param approxTimeOutTime the approximate time in milliseconds
* @throws ConnectException
*/
private void heartbeatCheckHasReceived(
@NotNull final SelectionKey key, final long approxTimeOutTime) throws ConnectException {
final Attached attached = (Attached) key.attachment();
// we wont attempt to reconnect the server socket
if (attached.isServer || !attached.isHandShakingComplete()) return;
final SocketChannel channel = (SocketChannel) key.channel();
if (approxTimeOutTime
> attached.entryReader.lastHeartBeatReceived + attached.remoteHeartbeatInterval) {
if (LOG.isDebugEnabled())
LOG.debug(
"lost connection, attempting to reconnect. "
+ "missed heartbeat from identifier="
+ attached.remoteIdentifier);
try {
channel.socket().close();
channel.close();
activeKeys.clear(attached.remoteIdentifier);
closeables.remove(channel);
} catch (IOException e) {
LOG.debug("", e);
}
attached.connector.connectLater();
}
}
/**
* checks that we receive heartbeats and send out heart beats.
*
* @param approxTime the approximate time in milliseconds
*/
void heartBeatMonitor(long approxTime) {
for (int i = activeKeys.nextSetBit(0); i >= 0; i = activeKeys.nextSetBit(i + 1)) {
try {
final SelectionKey key = selectionKeysStore[i];
if (!key.isValid() || !key.channel().isOpen()) {
activeKeys.clear(i);
continue;
}
try {
sendHeartbeatIfRequired(approxTime, key);
} catch (Exception e) {
if (LOG.isDebugEnabled()) LOG.debug("", e);
}
try {
heartbeatCheckHasReceived(key, approxTime);
} catch (Exception e) {
if (LOG.isDebugEnabled()) LOG.debug("", e);
}
} catch (Exception e) {
if (LOG.isDebugEnabled()) LOG.debug("", e);
}
}
}
public void applyUpdates() {
if (wasChanged.getAndSet(false)) {
for (int i = changeOfOpWriteRequired.nextSetBit(0);
i >= 0;
i = changeOfOpWriteRequired.nextSetBit(i + 1)) {
changeOfOpWriteRequired.clear(i);
final SelectionKey key = selectionKeys[i];
try {
key.interestOps(key.interestOps() | op);
} catch (Exception e) {
LOG.debug("", e);
}
}
}
}
/** called when the selector receives a OP_WRITE message */
private void onWrite(@NotNull final SelectionKey key, final long approxTime)
throws InterruptedException, IOException {
final SocketChannel socketChannel = (SocketChannel) key.channel();
final Attached attached = (Attached) key.attachment();
if (attached.remoteModificationIterator != null)
attached.entryWriter.entriesToBuffer(attached.remoteModificationIterator, key);
try {
int bytesJustWritten = attached.entryWriter.writeBufferToSocket(socketChannel, approxTime);
contemplateThrottleWrites(bytesJustWritten);
} catch (IOException e) {
quietClose(key, e);
if (!attached.isServer) attached.connector.connectLater();
throw e;
}
}
/**
* removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE will
* get added back in as soon as we have data to write
*
* @param socketChannel the socketChannel we wish to stop writing to
* @param attached data associated with the socketChannels key
*/
public synchronized void disableWrite(
@NotNull final SocketChannel socketChannel, @NotNull final Attached attached) {
try {
SelectionKey key = socketChannel.keyFor(selector);
if (key != null) {
if (attached.isHandShakingComplete() && selector.isOpen()) {
if (LOG.isDebugEnabled())
LOG.debug(
"Disabling OP_WRITE to remoteIdentifier="
+ attached.remoteIdentifier
+ ", localIdentifier="
+ localIdentifier);
key.interestOps(key.interestOps() & ~OP_WRITE);
}
}
} catch (Exception e) {
LOG.error("", e);
}
}
/** called when the selector receives a OP_CONNECT message */
private void onConnect(@NotNull final SelectionKey key) throws IOException, InterruptedException {
final SocketChannel channel = (SocketChannel) key.channel();
final Attached attached = (Attached) key.attachment();
try {
if (!channel.finishConnect()) {
return;
}
} catch (SocketException e) {
quietClose(key, e);
attached.connector.connect();
throw e;
}
attached.connector.setSuccessfullyConnected();
if (LOG.isDebugEnabled())
LOG.debug(
"successfully connected to {}, local-id={}",
channel.socket().getInetAddress(),
localIdentifier);
channel.configureBlocking(false);
channel.socket().setTcpNoDelay(true);
channel.socket().setSoTimeout(0);
channel.socket().setSoLinger(false, 0);
attached.entryReader = new TcpSocketChannelEntryReader();
attached.entryWriter = new TcpSocketChannelEntryWriter();
key.interestOps(OP_WRITE | OP_READ);
throttle(channel);
// register it with the selector and store the ModificationIterator for this key
attached.entryWriter.identifierToBuffer(localIdentifier);
}
/** called when the selector receives a OP_READ message */
private void onRead(final SelectionKey key, final long approxTime)
throws IOException, InterruptedException {
final SocketChannel socketChannel = (SocketChannel) key.channel();
final Attached attached = (Attached) key.attachment();
try {
if (attached.entryReader.readSocketToBuffer(socketChannel) <= 0) return;
} catch (IOException e) {
if (!attached.isServer) attached.connector.connectLater();
throw e;
}
if (LOG.isDebugEnabled()) LOG.debug("heartbeat or data received.");
attached.entryReader.lastHeartBeatReceived = approxTime;
if (attached.isHandShakingComplete()) {
attached.entryReader.entriesFromBuffer();
} else {
doHandShaking(key);
}
}
/**
* check to see if its time to send a heartbeat, and send one if required
*
* @param approxTime the current time ( approximately )
* @param key nio selection key
*/
private void sendHeartbeatIfRequired(final long approxTime, @NotNull final SelectionKey key) {
final Attached attachment = (Attached) key.attachment();
if (attachment.isHandShakingComplete()
&& attachment.entryWriter.lastSentTime + heartBeatInterval < approxTime) {
attachment.entryWriter.lastSentTime = approxTime;
attachment.entryWriter.writeHeartbeatToBuffer();
enableOpWrite(key);
if (LOG.isDebugEnabled()) LOG.debug("sending heartbeat");
}
}
/** called when the selector receives a OP_ACCEPT message */
private void onAccept(@NotNull final SelectionKey key) throws IOException {
final ServerSocketChannel server = (ServerSocketChannel) key.channel();
final SocketChannel channel = server.accept();
channel.configureBlocking(false);
channel.socket().setReuseAddress(true);
channel.socket().setTcpNoDelay(true);
channel.socket().setSoTimeout(0);
channel.socket().setSoLinger(false, 0);
final Attached attached = new Attached();
channel.register(selector, OP_WRITE | OP_READ, attached);
throttle(channel);
attached.entryReader = new TcpSocketChannelEntryReader();
attached.entryWriter = new TcpSocketChannelEntryWriter();
attached.isServer = true;
attached.entryWriter.identifierToBuffer(localIdentifier);
}
/**
* used to exchange identifiers and timestamps and heartbeat intervals between the server and
* client
*
* @param key the SelectionKey relating to the this cha
* @throws java.io.IOException
* @throws InterruptedException
*/
private void doHandShaking(@NotNull final SelectionKey key)
throws IOException, InterruptedException {
final Attached attached = (Attached) key.attachment();
final TcpSocketChannelEntryWriter writer = attached.entryWriter;
final TcpSocketChannelEntryReader reader = attached.entryReader;
if (attached.remoteIdentifier == Byte.MIN_VALUE) {
final byte remoteIdentifier = reader.identifierFromBuffer();
if (remoteIdentifier == Byte.MIN_VALUE) return;
attached.remoteIdentifier = remoteIdentifier;
// we use the as iterating the activeKeys via the bitset wont create and Objects
// but if we use the selector.keys() this will.
selectionKeysStore[remoteIdentifier] = key;
activeKeys.set(remoteIdentifier);
if (LOG.isDebugEnabled()) {
LOG.debug(
"server-connection id={}, remoteIdentifier={}", localIdentifier, remoteIdentifier);
}
if (remoteIdentifier == localIdentifier) {
throw new IllegalStateException(
"Where are connecting to a remote "
+ "map with the same "
+ "identifier as this map, identifier="
+ localIdentifier
+ ", "
+ "please change either this maps identifier or the remote one");
}
attached.remoteModificationIterator =
replica.acquireModificationIterator(remoteIdentifier, attached);
writer.writeRemoteBootstrapTimestamp(replica.lastModificationTime(remoteIdentifier));
// tell the remote node, what are heartbeat interval is
writer.writeRemoteHeartbeatInterval(heartBeatInterval);
}
if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) {
attached.remoteBootstrapTimestamp = reader.remoteBootstrapTimestamp();
if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) return;
}
if (!attached.hasRemoteHeartbeatInterval) {
long value = reader.remoteHeartbeatIntervalFromBuffer();
if (value == Long.MIN_VALUE) return;
if (value < 0) {
LOG.error("value=" + value);
}
// we add a 10% safety margin to the timeout time due to latency fluctuations on the network,
// in other words we wont consider a connection to have
// timed out, unless the heartbeat interval has exceeded 25% of the expected time.
attached.remoteHeartbeatInterval = (long) (value * 1.25);
// we have to make our selector poll interval at least as short as the minimum selector
// timeout
selectorTimeout = Math.min(selectorTimeout, value);
if (selectorTimeout < 0) LOG.info("");
attached.hasRemoteHeartbeatInterval = true;
// now we're finished we can get on with reading the entries
attached.setHandShakingComplete();
attached.remoteModificationIterator.dirtyEntries(attached.remoteBootstrapTimestamp);
reader.entriesFromBuffer();
}
}
/**
* closes and only logs the exception at debug
*
* @param key the SelectionKey
* @param e the Exception that caused the issue
*/
private void quietClose(@NotNull final SelectionKey key, @NotNull final Exception e) {
if (LOG.isDebugEnabled()) LOG.debug("", e);
closeEarlyAndQuietly(key.channel());
}
private void enableOpWrite(SelectionKey key) {
int ops = key.interestOps();
if ((ops & (OP_CONNECT | OP_ACCEPT)) == 0) key.interestOps(ops | OP_WRITE);
}
@Override
void process() throws IOException {
try {
final Details serverDetails = new Details(serverInetSocketAddress, localIdentifier);
connectorBySocket.put(serverInetSocketAddress, new ServerConnector(serverDetails));
for (InetSocketAddress client : endpoints) {
final Details clientDetails = new Details(client, localIdentifier);
connectorBySocket.put(client, new ClientConnector(clientDetails));
}
for (AbstractConnector connector : connectorBySocket.values()) {
connector.connect();
}
while (selector.isOpen()) {
registerPendingRegistrations();
final int nSelectedKeys = selector.select(selectorTimeout);
// its less resource intensive to set this less frequently and use an approximation
final long approxTime = System.currentTimeMillis();
checkThrottleInterval();
// check that we have sent and received heartbeats
heartBeatMonitor(approxTime);
// set the OP_WRITE when data is ready to send
opWriteUpdater.applyUpdates();
if (nSelectedKeys == 0) continue; // go back and check pendingRegistrations
final Set<SelectionKey> selectionKeys = selector.selectedKeys();
for (final SelectionKey key : selectionKeys) {
try {
if (!key.isValid()) continue;
if (key.isAcceptable()) onAccept(key);
if (key.isConnectable()) onConnect(key);
if (key.isReadable()) onRead(key, approxTime);
if (key.isWritable()) onWrite(key, approxTime);
} catch (CancelledKeyException e) {
quietClose(key, e);
} catch (ClosedSelectorException e) {
quietClose(key, e);
} catch (IOException e) {
quietClose(key, e);
} catch (InterruptedException e) {
quietClose(key, e);
} catch (Exception e) {
LOG.info("", e);
closeEarlyAndQuietly(key.channel());
}
}
selectionKeys.clear();
}
} catch (CancelledKeyException e) {
if (LOG.isDebugEnabled()) LOG.debug("", e);
} catch (ClosedSelectorException e) {
if (LOG.isDebugEnabled()) LOG.debug("", e);
} catch (ClosedChannelException e) {
if (LOG.isDebugEnabled()) LOG.debug("", e);
} catch (ConnectException e) {
if (LOG.isDebugEnabled()) LOG.debug("", e);
} catch (Exception e) {
LOG.error("", e);
} finally {
if (selector != null)
try {
selector.close();
} catch (IOException e) {
if (LOG.isDebugEnabled()) LOG.debug("", e);
}
close();
}
}