/** Checks consistency after all operations. */
private void consistencyCheck() {
if (CONSISTENCY_CHECK) {
assert lock.writeLock().isHeldByCurrentThread();
if (node2part == null) return;
for (Map.Entry<UUID, GridDhtPartitionMap> e : node2part.entrySet()) {
for (Integer p : e.getValue().keySet()) {
Set<UUID> nodeIds = part2node.get(p);
assert nodeIds != null
: "Failed consistency check [part=" + p + ", nodeId=" + e.getKey() + ']';
assert nodeIds.contains(e.getKey())
: "Failed consistency check [part="
+ p
+ ", nodeId="
+ e.getKey()
+ ", nodeIds="
+ nodeIds
+ ']';
}
}
for (Map.Entry<Integer, Set<UUID>> e : part2node.entrySet()) {
for (UUID nodeId : e.getValue()) {
GridDhtPartitionMap map = node2part.get(nodeId);
assert map != null
: "Failed consistency check [part=" + e.getKey() + ", nodeId=" + nodeId + ']';
assert map.containsKey(e.getKey())
: "Failed consistency check [part=" + e.getKey() + ", nodeId=" + nodeId + ']';
}
}
}
}
/**
* a DatagramChannel has data ready - process all the pending packets, whether its for a
* rdpserversocket or rdpconnection.
*/
void processActiveChannel(DatagramChannel dc) throws ClosedChannelException {
RDPPacket packet;
int count = 0;
// read in the packet
try {
Set<RDPConnection> needsAckConnections = new HashSet<RDPConnection>();
while ((packet = RDPServer.receivePacket(dc)) != null) {
if (Log.loggingNet)
Log.net(
"RDPServer.processActiveChannel: Starting iteration with count of "
+ count
+ " packets");
// see if there is a connection already for this packet
InetAddress remoteAddr = packet.getInetAddress();
int remotePort = packet.getPort();
int localPort = dc.socket().getLocalPort();
ConnectionInfo conInfo = new ConnectionInfo(remoteAddr, remotePort, localPort);
RDPConnection con = RDPServer.getConnection(dc, conInfo);
if (con != null) {
if (Log.loggingNet)
Log.net("RDPServer.processActiveChannel: found an existing connection: " + con);
count++;
if (processExistingConnection(con, packet)) needsAckConnections.add(con);
// Prevent this from blocking getActiveChannels by
// putting an upper bound on the number of packets
// processed
if (count >= 20) break;
continue;
} else {
Log.net("RDPServer.processActiveChannel: did not find an existing connection");
}
// there is no connection,
// see if there is a socket listening for new connection
RDPServerSocket rdpSocket = RDPServer.getRDPSocket(dc);
if (rdpSocket != null) {
count++;
processNewConnection(rdpSocket, packet);
return;
}
return;
}
// Finally, send out the acks
for (RDPConnection con : needsAckConnections) {
RDPPacket replyPacket = new RDPPacket(con);
con.sendPacketImmediate(replyPacket, false);
}
} catch (ClosedChannelException ex) {
Log.error("RDPServer.processActiveChannel: ClosedChannel " + dc.socket());
throw ex;
} finally {
if (Log.loggingNet)
Log.net("RDPServer.processActiveChannel: Returning after processing " + count + " packets");
}
}
static Set<RDPConnection> getAllConnections() {
lock.lock();
try {
Set<RDPConnection> allCon = new HashSet<RDPConnection>();
Iterator<Map<ConnectionInfo, RDPConnection>> iter = allConMap.values().iterator();
while (iter.hasNext()) {
Map<ConnectionInfo, RDPConnection> dcMap = iter.next();
allCon.addAll(dcMap.values());
}
return allCon;
} finally {
lock.unlock();
}
}
private boolean trashConnection(Connection connection) {
// First, make sure we don't go below core connections
for (; ; ) {
int opened = open.get();
if (opened <= options().getCoreConnectionsPerHost(hostDistance)) return false;
if (open.compareAndSet(opened, opened - 1)) break;
}
trash.add(connection);
connections.remove(connection);
if (connection.inFlight.get() == 0 && trash.remove(connection)) close(connection);
return true;
}
public void returnConnection(Connection connection) {
int inFlight = connection.inFlight.decrementAndGet();
if (connection.isDefunct()) {
if (host.getMonitor().signalConnectionFailure(connection.lastException())) shutdown();
else replace(connection);
} else {
if (trash.contains(connection) && inFlight == 0) {
if (trash.remove(connection)) close(connection);
return;
}
if (connections.size() > options().getCoreConnectionsPerHost(hostDistance)
&& inFlight <= options().getMinSimultaneousRequestsPerConnectionThreshold(hostDistance)) {
trashConnection(connection);
} else {
signalAvailableConnection();
}
}
}
/** @param nodeId Node to remove. */
private void removeNode(UUID nodeId) {
assert nodeId != null;
assert lock.writeLock().isHeldByCurrentThread();
ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx.shared(), topVer);
assert oldest != null;
ClusterNode loc = cctx.localNode();
if (node2part != null) {
if (oldest.equals(loc) && !node2part.nodeId().equals(loc.id())) {
updateSeq.setIfGreater(node2part.updateSequence());
node2part =
new GridDhtPartitionFullMap(
loc.id(), loc.order(), updateSeq.incrementAndGet(), node2part, false);
} else node2part = new GridDhtPartitionFullMap(node2part, node2part.updateSequence());
part2node = new HashMap<>(part2node);
GridDhtPartitionMap parts = node2part.remove(nodeId);
if (parts != null) {
for (Integer p : parts.keySet()) {
Set<UUID> nodeIds = part2node.get(p);
if (nodeIds != null) {
nodeIds.remove(nodeId);
if (nodeIds.isEmpty()) part2node.remove(p);
}
}
}
consistencyCheck();
}
}
/** starts the server listens to incoming packets */
public void run() {
try {
while (true) {
if (Log.loggingNet) Log.net("In RDPServer.run: starting new iteration");
try {
Set<DatagramChannel> activeChannels = getActiveChannels();
activeChannelCalls++;
Iterator<DatagramChannel> iter = activeChannels.iterator();
while (iter.hasNext()) {
DatagramChannel dc = iter.next();
if (Log.loggingNet) Log.net("In RDPServer.run: about to call processActiveChannel");
processActiveChannel(dc);
if (Log.loggingNet) Log.net("In RDPServer.run: returned from processActiveChannel");
}
} catch (ClosedChannelException ex) {
// ignore
} catch (Exception e) {
Log.exception("RDPServer.run caught exception", e);
}
}
} finally {
Log.warn("RDPServer.run: thread exiting");
}
}
/** returns a list of rdpserversockets */
Set<DatagramChannel> getActiveChannels() throws InterruptedException, java.io.IOException {
lock.lock();
try {
while (channelMap.isEmpty()) {
channelMapNotEmpty.await();
}
} finally {
lock.unlock();
}
Set<SelectionKey> readyKeys = null;
do {
lock.lock();
try {
if (!newChannelSet.isEmpty()) {
if (Log.loggingNet) Log.net("RDPServer.getActiveChannels: newChannelSet is not null");
Iterator<DatagramChannel> iter = newChannelSet.iterator();
while (iter.hasNext()) {
DatagramChannel newDC = iter.next();
iter.remove();
newDC.register(selector, SelectionKey.OP_READ);
}
}
} finally {
lock.unlock();
}
int numReady = selector.select(); // this is a blocking call - thread safe
selectCalls++;
if (numReady == 0) {
if (Log.loggingNet) Log.net("RDPServer.getActiveChannels: selector returned 0");
continue;
}
readyKeys = selector.selectedKeys();
if (Log.loggingNet)
Log.net(
"RDPServer.getActiveChannels: called select - # of ready keys = "
+ readyKeys.size()
+ " == "
+ numReady);
} while (readyKeys == null || readyKeys.isEmpty());
lock.lock();
try {
// get a datagramchannel that is ready
Set<DatagramChannel> activeChannels = new HashSet<DatagramChannel>();
Iterator<SelectionKey> iter = readyKeys.iterator();
while (iter.hasNext()) {
SelectionKey key = iter.next();
if (Log.loggingNet)
Log.net(
"RDPServer.getActiveChannels: matched selectionkey: "
+ key
+ ", isAcceptable="
+ key.isAcceptable()
+ ", isReadable="
+ key.isReadable()
+ ", isValid="
+ key.isValid()
+ ", isWritable="
+ key.isWritable());
iter.remove(); // remove from the selected key list
if (!key.isReadable() || !key.isValid()) {
Log.error(
"RDPServer.getActiveChannels: Throwing exception: RDPServer: not readable or invalid");
throw new MVRuntimeException("RDPServer: not readable or invalid");
}
DatagramChannel dc = (DatagramChannel) key.channel();
activeChannels.add(dc);
}
if (Log.loggingNet)
Log.net(
"RDPServer.getActiveChannels: returning " + activeChannels.size() + " active channels");
return activeChannels;
} finally {
lock.unlock();
}
}
/** {@inheritDoc} */
@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
@Nullable
@Override
public GridDhtPartitionMap update(
@Nullable GridDhtPartitionExchangeId exchId, GridDhtPartitionFullMap partMap) {
if (log.isDebugEnabled())
log.debug(
"Updating full partition map [exchId=" + exchId + ", parts=" + fullMapString() + ']');
assert partMap != null;
lock.writeLock().lock();
try {
if (stopping) return null;
if (exchId != null && lastExchangeId != null && lastExchangeId.compareTo(exchId) >= 0) {
if (log.isDebugEnabled())
log.debug(
"Stale exchange id for full partition map update (will ignore) [lastExchId="
+ lastExchangeId
+ ", exchId="
+ exchId
+ ']');
return null;
}
if (node2part != null && node2part.compareTo(partMap) >= 0) {
if (log.isDebugEnabled())
log.debug(
"Stale partition map for full partition map update (will ignore) [lastExchId="
+ lastExchangeId
+ ", exchId="
+ exchId
+ ", curMap="
+ node2part
+ ", newMap="
+ partMap
+ ']');
return null;
}
long updateSeq = this.updateSeq.incrementAndGet();
if (exchId != null) lastExchangeId = exchId;
if (node2part != null) {
for (GridDhtPartitionMap part : node2part.values()) {
GridDhtPartitionMap newPart = partMap.get(part.nodeId());
// If for some nodes current partition has a newer map,
// then we keep the newer value.
if (newPart != null && newPart.updateSequence() < part.updateSequence()) {
if (log.isDebugEnabled())
log.debug(
"Overriding partition map in full update map [exchId="
+ exchId
+ ", curPart="
+ mapString(part)
+ ", newPart="
+ mapString(newPart)
+ ']');
partMap.put(part.nodeId(), part);
}
}
for (Iterator<UUID> it = partMap.keySet().iterator(); it.hasNext(); ) {
UUID nodeId = it.next();
if (!cctx.discovery().alive(nodeId)) {
if (log.isDebugEnabled())
log.debug(
"Removing left node from full map update [nodeId="
+ nodeId
+ ", partMap="
+ partMap
+ ']');
it.remove();
}
}
}
node2part = partMap;
Map<Integer, Set<UUID>> p2n = new HashMap<>(cctx.affinity().partitions(), 1.0f);
for (Map.Entry<UUID, GridDhtPartitionMap> e : partMap.entrySet()) {
for (Integer p : e.getValue().keySet()) {
Set<UUID> ids = p2n.get(p);
if (ids == null)
// Initialize HashSet to size 3 in anticipation that there won't be
// more than 3 nodes per partitions.
p2n.put(p, ids = U.newHashSet(3));
ids.add(e.getKey());
}
}
part2node = p2n;
boolean changed = checkEvictions(updateSeq);
consistencyCheck();
if (log.isDebugEnabled()) log.debug("Partition map after full update: " + fullMapString());
return changed ? localPartitionMap() : null;
} finally {
lock.writeLock().unlock();
}
}
/**
* Updates value for single partition.
*
* @param p Partition.
* @param nodeId Node ID.
* @param state State.
* @param updateSeq Update sequence.
*/
@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
private void updateLocal(int p, UUID nodeId, GridDhtPartitionState state, long updateSeq) {
assert lock.isWriteLockedByCurrentThread();
assert nodeId.equals(cctx.nodeId());
// In case if node joins, get topology at the time of joining node.
ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx.shared(), topVer);
assert oldest != null;
// If this node became the oldest node.
if (oldest.id().equals(cctx.nodeId())) {
long seq = node2part.updateSequence();
if (seq != updateSeq) {
if (seq > updateSeq) {
if (this.updateSeq.get() < seq) {
// Update global counter if necessary.
boolean b = this.updateSeq.compareAndSet(this.updateSeq.get(), seq + 1);
assert b
: "Invalid update sequence [updateSeq="
+ updateSeq
+ ", seq="
+ seq
+ ", curUpdateSeq="
+ this.updateSeq.get()
+ ", node2part="
+ node2part.toFullString()
+ ']';
updateSeq = seq + 1;
} else updateSeq = seq;
}
node2part.updateSequence(updateSeq);
}
}
GridDhtPartitionMap map = node2part.get(nodeId);
if (map == null)
node2part.put(
nodeId,
map =
new GridDhtPartitionMap(
nodeId,
updateSeq,
Collections.<Integer, GridDhtPartitionState>emptyMap(),
false));
map.updateSequence(updateSeq);
map.put(p, state);
Set<UUID> ids = part2node.get(p);
if (ids == null) part2node.put(p, ids = U.newHashSet(3));
ids.add(nodeId);
}
/** {@inheritDoc} */
@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
@Nullable
@Override
public GridDhtPartitionMap update(
@Nullable GridDhtPartitionExchangeId exchId, GridDhtPartitionMap parts) {
if (log.isDebugEnabled())
log.debug(
"Updating single partition map [exchId=" + exchId + ", parts=" + mapString(parts) + ']');
if (!cctx.discovery().alive(parts.nodeId())) {
if (log.isDebugEnabled())
log.debug(
"Received partition update for non-existing node (will ignore) [exchId="
+ exchId
+ ", parts="
+ parts
+ ']');
return null;
}
lock.writeLock().lock();
try {
if (stopping) return null;
if (lastExchangeId != null && exchId != null && lastExchangeId.compareTo(exchId) > 0) {
if (log.isDebugEnabled())
log.debug(
"Stale exchange id for single partition map update (will ignore) [lastExchId="
+ lastExchangeId
+ ", exchId="
+ exchId
+ ']');
return null;
}
if (exchId != null) lastExchangeId = exchId;
if (node2part == null)
// Create invalid partition map.
node2part = new GridDhtPartitionFullMap();
GridDhtPartitionMap cur = node2part.get(parts.nodeId());
if (cur != null && cur.updateSequence() >= parts.updateSequence()) {
if (log.isDebugEnabled())
log.debug(
"Stale update sequence for single partition map update (will ignore) [exchId="
+ exchId
+ ", curSeq="
+ cur.updateSequence()
+ ", newSeq="
+ parts.updateSequence()
+ ']');
return null;
}
long updateSeq = this.updateSeq.incrementAndGet();
node2part = new GridDhtPartitionFullMap(node2part, updateSeq);
boolean changed = false;
if (cur == null || !cur.equals(parts)) changed = true;
node2part.put(parts.nodeId(), parts);
part2node = new HashMap<>(part2node);
// Add new mappings.
for (Integer p : parts.keySet()) {
Set<UUID> ids = part2node.get(p);
if (ids == null)
// Initialize HashSet to size 3 in anticipation that there won't be
// more than 3 nodes per partition.
part2node.put(p, ids = U.newHashSet(3));
changed |= ids.add(parts.nodeId());
}
// Remove obsolete mappings.
if (cur != null) {
for (Integer p : F.view(cur.keySet(), F0.notIn(parts.keySet()))) {
Set<UUID> ids = part2node.get(p);
if (ids != null) changed |= ids.remove(parts.nodeId());
}
}
changed |= checkEvictions(updateSeq);
consistencyCheck();
if (log.isDebugEnabled()) log.debug("Partition map after single update: " + fullMapString());
return changed ? localPartitionMap() : null;
} finally {
lock.writeLock().unlock();
}
}