public void removeMember(MemberImpl member) {
checkServiceThread();
logger.log(Level.FINEST, "ClusterManager removing " + member);
mapMembers.remove(member.getAddress());
lsMembers.remove(member);
memberMasterConfirmationTimes.remove(member);
if (!member.isLiteMember()) {
dataMemberCount.decrement();
}
}
List<Edge> detectDeadlock() {
Collection<Map<String, MapLockState>> collection =
(Collection<Map<String, MapLockState>>) callOnAllMembers(new LockInformationCallable());
List<Vertex> graph = new ArrayList<Vertex>();
for (Map<String, MapLockState> mapLockStateMap : collection) {
for (MapLockState map : mapLockStateMap.values()) {
for (Object key : map.getLockOwners().keySet()) {
Vertex owner = new Vertex(map.getLockOwners().get(key));
Vertex requester = new Vertex(map.getLockRequested().get(key));
int index = graph.indexOf(owner);
if (index >= 0) {
owner = graph.get(index);
} else {
graph.add(owner);
}
index = graph.indexOf(requester);
if (index >= 0) {
requester = graph.get(index);
} else {
graph.add(requester);
}
Edge edge = new Edge();
edge.from = requester;
edge.to = owner;
edge.key = key;
edge.mapName = map.getMapName();
edge.globalLock = map.isGlobalLock();
owner.addIncoming(edge);
requester.addOutgoing(edge);
}
}
}
List<Edge> list = new ArrayList<Edge>();
if (graph != null && graph.size() > 0) {
try {
graph.get(0).visit(list);
} catch (RuntimeException e) {
}
}
return list;
}
public void stop() {
if (setJoins != null) {
setJoins.clear();
}
timeToStartJoin = 0;
if (lsMembers != null) {
lsMembers.clear();
}
dataMemberCount.reset();
if (mapMembers != null) {
mapMembers.clear();
}
if (mapCalls != null) {
mapCalls.clear();
}
if (memberMasterConfirmationTimes != null) {
memberMasterConfirmationTimes.clear();
}
}
public Member addMember(boolean checkServiceThread, MemberImpl member) {
if (checkServiceThread) {
checkServiceThread();
}
logger.log(Level.FINEST, "ClusterManager adding " + member);
if (lsMembers.contains(member)) {
for (MemberImpl m : lsMembers) {
if (m.equals(member)) {
member = m;
}
}
mapMembers.put(member.getAddress(), member);
} else {
lsMembers.add(member);
mapMembers.put(member.getAddress(), member);
if (!member.isLiteMember()) {
dataMemberCount.increment();
}
}
memberMasterConfirmationTimes.put(member, Clock.currentTimeMillis());
return member;
}
void updateMembers(Collection<MemberInfo> lsMemberInfos) {
checkServiceThread();
final Map<Address, MemberImpl> mapOldMembers = new HashMap<Address, MemberImpl>();
for (MemberImpl member : lsMembers) {
mapOldMembers.put(member.getAddress(), member);
}
if (mapOldMembers.size() == lsMemberInfos.size()) {
boolean same = true;
for (MemberInfo memberInfo : lsMemberInfos) {
MemberImpl member = mapOldMembers.get(memberInfo.getAddress());
if (member == null || !member.getUuid().equals(memberInfo.uuid)) {
same = false;
break;
}
}
if (same) {
logger.log(Level.FINEST, "No need to process member update...");
return;
}
}
logger.log(Level.FINEST, "Updating Members");
lsMembers.clear();
dataMemberCount.reset();
mapMembers.clear();
memberMasterConfirmationTimes.clear();
for (MemberInfo memberInfo : lsMemberInfos) {
MemberImpl member = mapOldMembers.get(memberInfo.address);
if (member == null) {
member = addMember(memberInfo.address, memberInfo.nodeType, memberInfo.uuid);
} else {
addMember(member);
}
member.didRead();
}
if (!lsMembers.contains(thisMember)) {
throw new RuntimeException("Member list doesn't contain local member!");
}
heartBeater();
node.getClusterImpl().setMembers(lsMembers);
node.setJoined();
logger.log(Level.INFO, this.toString());
}
// Will be called just before this node becomes the master
private void resetMemberMasterConfirmations() {
checkServiceThread();
for (MemberImpl member : lsMembers) {
memberMasterConfirmationTimes.put(member, Clock.currentTimeMillis());
}
}
public final void heartBeater() {
if (!node.joined() || !node.isActive()) return;
long now = Clock.currentTimeMillis();
if (isMaster()) {
List<Address> lsDeadAddresses = null;
for (MemberImpl memberImpl : lsMembers) {
final Address address = memberImpl.getAddress();
if (!thisAddress.equals(address)) {
try {
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null && conn.live()) {
if ((now - memberImpl.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
conn = null;
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because of timeout since last read");
lsDeadAddresses.add(address);
} else if ((now - memberImpl.getLastRead()) >= 5000
&& (now - memberImpl.getLastPing()) >= 5000) {
ping(memberImpl);
}
if ((now - memberImpl.getLastWrite()) > 500) {
sendHeartbeat(conn);
}
Long lastConfirmation = memberMasterConfirmationTimes.get(memberImpl);
if (lastConfirmation == null
|| (now - lastConfirmation > MAX_NO_MASTER_CONFIRMATION_MILLIS)) {
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because it has not sent a master confirmation recently");
lsDeadAddresses.add(address);
}
} else if (conn == null && (now - memberImpl.getLastRead()) > 5000) {
logMissingConnection(address);
memberImpl.didRead();
}
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);
}
}
}
if (lsDeadAddresses != null) {
for (Address address : lsDeadAddresses) {
logger.log(Level.FINEST, "No heartbeat should remove " + address);
doRemoveAddress(address);
}
}
} else {
// send heartbeat to master
Address masterAddress = getMasterAddress();
if (masterAddress != null) {
final Connection connMaster = node.connectionManager.getOrConnect(masterAddress);
MemberImpl masterMember = getMember(masterAddress);
boolean removed = false;
if (masterMember != null) {
if ((now - masterMember.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
logger.log(
Level.WARNING, "Master node has timed out its heartbeat and will be removed");
doRemoveAddress(masterAddress);
removed = true;
} else if ((now - masterMember.getLastRead()) >= 5000
&& (now - masterMember.getLastPing()) >= 5000) {
ping(masterMember);
}
}
if (!removed) {
sendHeartbeat(connMaster);
}
}
for (MemberImpl member : lsMembers) {
if (!member.localMember()) {
Address address = member.getAddress();
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null) {
sendHeartbeat(conn);
} else {
logger.log(Level.FINEST, "Could not connect to " + address + " to send heartbeat");
}
}
}
}
}
private void createRuntimeProps(MemberStateImpl memberState) {
Runtime runtime = Runtime.getRuntime();
ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean();
RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
ClassLoadingMXBean clMxBean = ManagementFactory.getClassLoadingMXBean();
MemoryMXBean memoryMxBean = ManagementFactory.getMemoryMXBean();
MemoryUsage heapMemory = memoryMxBean.getHeapMemoryUsage();
MemoryUsage nonHeapMemory = memoryMxBean.getNonHeapMemoryUsage();
Map<String, Long> map = new HashMap<String, Long>();
map.put(
"runtime.availableProcessors", Integer.valueOf(runtime.availableProcessors()).longValue());
map.put("date.startTime", runtimeMxBean.getStartTime());
map.put("seconds.upTime", runtimeMxBean.getUptime());
map.put("memory.maxMemory", runtime.maxMemory());
map.put("memory.freeMemory", runtime.freeMemory());
map.put("memory.totalMemory", runtime.totalMemory());
map.put("memory.heapMemoryMax", heapMemory.getMax());
map.put("memory.heapMemoryUsed", heapMemory.getUsed());
map.put("memory.nonHeapMemoryMax", nonHeapMemory.getMax());
map.put("memory.nonHeapMemoryUsed", nonHeapMemory.getUsed());
map.put("runtime.totalLoadedClassCount", clMxBean.getTotalLoadedClassCount());
map.put(
"runtime.loadedClassCount", Integer.valueOf(clMxBean.getLoadedClassCount()).longValue());
map.put("runtime.unloadedClassCount", clMxBean.getUnloadedClassCount());
map.put("runtime.totalStartedThreadCount", threadMxBean.getTotalStartedThreadCount());
map.put("runtime.threadCount", Integer.valueOf(threadMxBean.getThreadCount()).longValue());
map.put(
"runtime.peakThreadCount", Integer.valueOf(threadMxBean.getPeakThreadCount()).longValue());
map.put(
"runtime.daemonThreadCount",
Integer.valueOf(threadMxBean.getDaemonThreadCount()).longValue());
memberState.setRuntimeProps(map);
}