private void sendRemoveMemberToOthers(final Address deadAddress) {
for (MemberImpl member : lsMembers) {
Address address = member.getAddress();
if (!thisAddress.equals(address) && !address.equals(deadAddress)) {
sendProcessableTo(new MemberRemover(deadAddress), address);
}
}
}
public void removeMember(MemberImpl member) {
checkServiceThread();
logger.log(Level.FINEST, "ClusterManager removing " + member);
mapMembers.remove(member.getAddress());
lsMembers.remove(member);
memberMasterConfirmationTimes.remove(member);
if (!member.isLiteMember()) {
dataMemberCount.decrement();
}
}
public Member addMember(boolean checkServiceThread, MemberImpl member) {
if (checkServiceThread) {
checkServiceThread();
}
logger.log(Level.FINEST, "ClusterManager adding " + member);
if (lsMembers.contains(member)) {
for (MemberImpl m : lsMembers) {
if (m.equals(member)) {
member = m;
}
}
mapMembers.put(member.getAddress(), member);
} else {
lsMembers.add(member);
mapMembers.put(member.getAddress(), member);
if (!member.isLiteMember()) {
dataMemberCount.increment();
}
}
memberMasterConfirmationTimes.put(member, Clock.currentTimeMillis());
return member;
}
public void finalizeJoin() {
Set<Member> members = node.getClusterImpl().getMembers();
List<AsyncRemotelyBooleanOp> calls = new ArrayList<AsyncRemotelyBooleanOp>();
for (Member m : members) {
MemberImpl member = (MemberImpl) m;
if (!member.localMember()) {
AsyncRemotelyBooleanOp op =
new AsyncRemotelyBooleanOp(new FinalizeJoin(), member.getAddress(), false);
op.execute();
calls.add(op);
}
}
for (AsyncRemotelyBooleanOp call : calls) {
call.getResultAsBoolean();
}
}
void doRemoveAddress(Address deadAddress, boolean destroyConnection) {
checkServiceThread();
logger.log(Level.INFO, "Removing Address " + deadAddress);
if (!node.joined()) {
node.failedConnection(deadAddress);
return;
}
if (deadAddress.equals(thisAddress)) return;
if (deadAddress.equals(getMasterAddress())) {
if (node.joined()) {
MemberImpl newMaster = getNextMemberAfter(deadAddress, false, 1);
if (newMaster != null) node.setMasterAddress(newMaster.getAddress());
else node.setMasterAddress(null);
} else {
node.setMasterAddress(null);
}
logger.log(Level.FINEST, "Now Master " + node.getMasterAddress());
}
if (isMaster()) {
setJoins.remove(new MemberInfo(deadAddress));
resetMemberMasterConfirmations();
}
final Connection conn = node.connectionManager.getConnection(deadAddress);
if (destroyConnection && conn != null) {
node.connectionManager.destroyConnection(conn);
}
MemberImpl deadMember = getMember(deadAddress);
if (deadMember != null) {
removeMember(deadMember);
node.getClusterImpl().setMembers(lsMembers);
node.concurrentMapManager.syncForDead(deadMember);
node.blockingQueueManager.syncForDead(deadAddress);
node.listenerManager.syncForDead(deadAddress);
node.topicManager.syncForDead(deadAddress);
// node.getClusterImpl().setMembers(lsMembers); // shifted up to get members in syncForDead
// methods
disconnectExistingCalls(deadAddress);
if (isMaster()) {
logger.log(Level.FINEST, deadAddress + " is dead. Sending remove to all other members.");
sendRemoveMemberToOthers(deadAddress);
}
logger.log(Level.INFO, this.toString());
}
}
void updateMembers(Collection<MemberInfo> lsMemberInfos) {
checkServiceThread();
final Map<Address, MemberImpl> mapOldMembers = new HashMap<Address, MemberImpl>();
for (MemberImpl member : lsMembers) {
mapOldMembers.put(member.getAddress(), member);
}
if (mapOldMembers.size() == lsMemberInfos.size()) {
boolean same = true;
for (MemberInfo memberInfo : lsMemberInfos) {
MemberImpl member = mapOldMembers.get(memberInfo.getAddress());
if (member == null || !member.getUuid().equals(memberInfo.uuid)) {
same = false;
break;
}
}
if (same) {
logger.log(Level.FINEST, "No need to process member update...");
return;
}
}
logger.log(Level.FINEST, "Updating Members");
lsMembers.clear();
dataMemberCount.reset();
mapMembers.clear();
memberMasterConfirmationTimes.clear();
for (MemberInfo memberInfo : lsMemberInfos) {
MemberImpl member = mapOldMembers.get(memberInfo.address);
if (member == null) {
member = addMember(memberInfo.address, memberInfo.nodeType, memberInfo.uuid);
} else {
addMember(member);
}
member.didRead();
}
if (!lsMembers.contains(thisMember)) {
throw new RuntimeException("Member list doesn't contain local member!");
}
heartBeater();
node.getClusterImpl().setMembers(lsMembers);
node.setJoined();
logger.log(Level.INFO, this.toString());
}
public TimedMemberState getTimedMemberState() {
if (running.get()) {
final MemberStateImpl memberState = new MemberStateImpl();
createMemberState(memberState);
GroupConfig groupConfig = factory.getConfig().getGroupConfig();
TimedMemberState timedMemberState = new TimedMemberState();
timedMemberState.setMaster(factory.node.isMaster());
if (timedMemberState.getMaster()) {
timedMemberState.setMemberList(new ArrayList<String>());
Set<Member> memberSet = factory.getCluster().getMembers();
for (Member member : memberSet) {
MemberImpl memberImpl = (MemberImpl) member;
Address address = memberImpl.getAddress();
timedMemberState.getMemberList().add(address.getHost() + ":" + address.getPort());
}
}
timedMemberState.setMemberState(memberState);
timedMemberState.setClusterName(groupConfig.getName());
timedMemberState.setInstanceNames(getLongInstanceNames());
return timedMemberState;
}
return null;
}
void handleJoinRequest(JoinRequest joinRequest) {
final long now = Clock.currentTimeMillis();
String msg =
"Handling join from "
+ joinRequest.address
+ ", inProgress: "
+ joinInProgress
+ (timeToStartJoin > 0 ? ", timeToStart: " + (timeToStartJoin - now) : "");
logger.log(Level.FINEST, msg);
boolean validJoinRequest;
try {
validJoinRequest = node.validateJoinRequest(joinRequest);
} catch (Exception e) {
validJoinRequest = false;
}
final Connection conn = joinRequest.getConnection();
if (validJoinRequest) {
final MemberImpl member = getMember(joinRequest.address);
if (member != null) {
if (joinRequest.getUuid().equals(member.getUuid())) {
String message = "Ignoring join request, member already exists.. => " + joinRequest;
logger.log(Level.FINEST, message);
// send members update back to node trying to join again...
final long clusterTime = node.getClusterImpl().getClusterTime();
sendProcessableTo(new MembersUpdateCall(lsMembers, clusterTime), conn);
sendProcessableTo(new SyncProcess(), conn);
return;
}
// If this node is master then remove old member and process join request.
// If requesting address is equal to master node's address, that means master node
// somehow disconnected and wants to join back.
// So drop old member and process join request if this node becomes master.
if (isMaster() || member.getAddress().equals(getMasterAddress())) {
logger.log(
Level.WARNING,
"New join request has been received from an existing endpoint! => "
+ member
+ " Removing old member and processing join request...");
// If existing connection of endpoint is different from current connection
// destroy it, otherwise keep it.
// final Connection existingConnection =
// node.connectionManager.getConnection(joinRequest.address);
// final boolean destroyExistingConnection = existingConnection !=
// conn;
doRemoveAddress(member.getAddress(), false);
}
}
if (!node.getConfig().getNetworkConfig().getJoin().getMulticastConfig().isEnabled()) {
if (node.isActive() && node.joined() && node.getMasterAddress() != null && !isMaster()) {
sendProcessableTo(new Master(node.getMasterAddress()), conn);
}
}
if (isMaster() && node.joined() && node.isActive()) {
final MemberInfo newMemberInfo =
new MemberInfo(joinRequest.address, joinRequest.nodeType, joinRequest.getUuid());
if (node.securityContext != null && !setJoins.contains(newMemberInfo)) {
final ILogger securityLogger = node.loggingService.getLogger("com.hazelcast.security");
final Credentials cr = joinRequest.getCredentials();
if (cr == null) {
securityLogger.log(
Level.SEVERE,
"Expecting security credentials "
+ "but credentials could not be found in JoinRequest!");
sendAuthFail(conn);
return;
} else {
try {
LoginContext lc = node.securityContext.createMemberLoginContext(cr);
lc.login();
} catch (LoginException e) {
securityLogger.log(
Level.SEVERE,
"Authentication has failed for "
+ cr.getPrincipal()
+ '@'
+ cr.getEndpoint()
+ " => ("
+ e.getMessage()
+ ")");
securityLogger.log(Level.FINEST, e.getMessage(), e);
sendAuthFail(conn);
return;
}
}
}
if (joinRequest.to != null && !joinRequest.to.equals(thisAddress)) {
sendProcessableTo(new Master(node.getMasterAddress()), conn);
return;
}
if (!joinInProgress) {
if (firstJoinRequest != 0
&& now - firstJoinRequest >= MAX_WAIT_SECONDS_BEFORE_JOIN * 1000) {
startJoin();
} else {
if (setJoins.add(newMemberInfo)) {
sendProcessableTo(new Master(node.getMasterAddress()), conn);
if (firstJoinRequest == 0) {
firstJoinRequest = now;
}
if (now - firstJoinRequest < MAX_WAIT_SECONDS_BEFORE_JOIN * 1000) {
timeToStartJoin = now + WAIT_MILLIS_BEFORE_JOIN;
}
}
if (now > timeToStartJoin) {
startJoin();
}
}
}
}
} else {
conn.close();
}
}
public final void heartBeater() {
if (!node.joined() || !node.isActive()) return;
long now = Clock.currentTimeMillis();
if (isMaster()) {
List<Address> lsDeadAddresses = null;
for (MemberImpl memberImpl : lsMembers) {
final Address address = memberImpl.getAddress();
if (!thisAddress.equals(address)) {
try {
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null && conn.live()) {
if ((now - memberImpl.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
conn = null;
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because of timeout since last read");
lsDeadAddresses.add(address);
} else if ((now - memberImpl.getLastRead()) >= 5000
&& (now - memberImpl.getLastPing()) >= 5000) {
ping(memberImpl);
}
if ((now - memberImpl.getLastWrite()) > 500) {
sendHeartbeat(conn);
}
Long lastConfirmation = memberMasterConfirmationTimes.get(memberImpl);
if (lastConfirmation == null
|| (now - lastConfirmation > MAX_NO_MASTER_CONFIRMATION_MILLIS)) {
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because it has not sent a master confirmation recently");
lsDeadAddresses.add(address);
}
} else if (conn == null && (now - memberImpl.getLastRead()) > 5000) {
logMissingConnection(address);
memberImpl.didRead();
}
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);
}
}
}
if (lsDeadAddresses != null) {
for (Address address : lsDeadAddresses) {
logger.log(Level.FINEST, "No heartbeat should remove " + address);
doRemoveAddress(address);
}
}
} else {
// send heartbeat to master
Address masterAddress = getMasterAddress();
if (masterAddress != null) {
final Connection connMaster = node.connectionManager.getOrConnect(masterAddress);
MemberImpl masterMember = getMember(masterAddress);
boolean removed = false;
if (masterMember != null) {
if ((now - masterMember.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
logger.log(
Level.WARNING, "Master node has timed out its heartbeat and will be removed");
doRemoveAddress(masterAddress);
removed = true;
} else if ((now - masterMember.getLastRead()) >= 5000
&& (now - masterMember.getLastPing()) >= 5000) {
ping(masterMember);
}
}
if (!removed) {
sendHeartbeat(connMaster);
}
}
for (MemberImpl member : lsMembers) {
if (!member.localMember()) {
Address address = member.getAddress();
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null) {
sendHeartbeat(conn);
} else {
logger.log(Level.FINEST, "Could not connect to " + address + " to send heartbeat");
}
}
}
}
}