@Override
public void doRun() {
Collection<MemberInfo> lsMemberInfos = membersUpdate.getMemberInfos();
List<Address> newMemberList = new ArrayList<Address>(lsMemberInfos.size());
for (final MemberInfo memberInfo : lsMemberInfos) {
newMemberList.add(memberInfo.address);
}
doCall(membersUpdate, newMemberList, true);
systemLogService.logJoin("JoinRunnable update members done.");
doCall(new SyncProcess(), newMemberList, false);
systemLogService.logJoin("JoinRunnable sync done.");
doCall(new ConnectionCheckCall(), newMemberList, false);
systemLogService.logJoin("JoinRunnable connection check done.");
}
void doCall(AbstractRemotelyCallable callable, List<Address> targets, boolean ignoreThis) {
List<AsyncRemotelyBooleanOp> calls = new ArrayList<AsyncRemotelyBooleanOp>(targets.size());
for (final Address target : targets) {
boolean skip = ignoreThis && thisAddress.equals(target);
if (!skip) {
AsyncRemotelyBooleanOp op = new AsyncRemotelyBooleanOp(callable, target, false);
op.execute();
calls.add(op);
}
}
for (AsyncRemotelyBooleanOp call : calls) {
if (!call.getResultAsBoolean(5)) {
targets.remove(call.getTarget());
}
}
}
public void finalizeJoin() {
Set<Member> members = node.getClusterImpl().getMembers();
List<AsyncRemotelyBooleanOp> calls = new ArrayList<AsyncRemotelyBooleanOp>();
for (Member m : members) {
MemberImpl member = (MemberImpl) m;
if (!member.localMember()) {
AsyncRemotelyBooleanOp op =
new AsyncRemotelyBooleanOp(new FinalizeJoin(), member.getAddress(), false);
op.execute();
calls.add(op);
}
}
for (AsyncRemotelyBooleanOp call : calls) {
call.getResultAsBoolean();
}
}
List<Edge> detectDeadlock() {
Collection<Map<String, MapLockState>> collection =
(Collection<Map<String, MapLockState>>) callOnAllMembers(new LockInformationCallable());
List<Vertex> graph = new ArrayList<Vertex>();
for (Map<String, MapLockState> mapLockStateMap : collection) {
for (MapLockState map : mapLockStateMap.values()) {
for (Object key : map.getLockOwners().keySet()) {
Vertex owner = new Vertex(map.getLockOwners().get(key));
Vertex requester = new Vertex(map.getLockRequested().get(key));
int index = graph.indexOf(owner);
if (index >= 0) {
owner = graph.get(index);
} else {
graph.add(owner);
}
index = graph.indexOf(requester);
if (index >= 0) {
requester = graph.get(index);
} else {
graph.add(requester);
}
Edge edge = new Edge();
edge.from = requester;
edge.to = owner;
edge.key = key;
edge.mapName = map.getMapName();
edge.globalLock = map.isGlobalLock();
owner.addIncoming(edge);
requester.addOutgoing(edge);
}
}
}
List<Edge> list = new ArrayList<Edge>();
if (graph != null && graph.size() > 0) {
try {
graph.get(0).visit(list);
} catch (RuntimeException e) {
}
}
return list;
}
public final void heartBeater() {
if (!node.joined() || !node.isActive()) return;
long now = Clock.currentTimeMillis();
if (isMaster()) {
List<Address> lsDeadAddresses = null;
for (MemberImpl memberImpl : lsMembers) {
final Address address = memberImpl.getAddress();
if (!thisAddress.equals(address)) {
try {
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null && conn.live()) {
if ((now - memberImpl.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
conn = null;
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because of timeout since last read");
lsDeadAddresses.add(address);
} else if ((now - memberImpl.getLastRead()) >= 5000
&& (now - memberImpl.getLastPing()) >= 5000) {
ping(memberImpl);
}
if ((now - memberImpl.getLastWrite()) > 500) {
sendHeartbeat(conn);
}
Long lastConfirmation = memberMasterConfirmationTimes.get(memberImpl);
if (lastConfirmation == null
|| (now - lastConfirmation > MAX_NO_MASTER_CONFIRMATION_MILLIS)) {
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because it has not sent a master confirmation recently");
lsDeadAddresses.add(address);
}
} else if (conn == null && (now - memberImpl.getLastRead()) > 5000) {
logMissingConnection(address);
memberImpl.didRead();
}
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);
}
}
}
if (lsDeadAddresses != null) {
for (Address address : lsDeadAddresses) {
logger.log(Level.FINEST, "No heartbeat should remove " + address);
doRemoveAddress(address);
}
}
} else {
// send heartbeat to master
Address masterAddress = getMasterAddress();
if (masterAddress != null) {
final Connection connMaster = node.connectionManager.getOrConnect(masterAddress);
MemberImpl masterMember = getMember(masterAddress);
boolean removed = false;
if (masterMember != null) {
if ((now - masterMember.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
logger.log(
Level.WARNING, "Master node has timed out its heartbeat and will be removed");
doRemoveAddress(masterAddress);
removed = true;
} else if ((now - masterMember.getLastRead()) >= 5000
&& (now - masterMember.getLastPing()) >= 5000) {
ping(masterMember);
}
}
if (!removed) {
sendHeartbeat(connMaster);
}
}
for (MemberImpl member : lsMembers) {
if (!member.localMember()) {
Address address = member.getAddress();
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null) {
sendHeartbeat(conn);
} else {
logger.log(Level.FINEST, "Could not connect to " + address + " to send heartbeat");
}
}
}
}
}