private void sendRemoveMemberToOthers(final Address deadAddress) {
for (MemberImpl member : lsMembers) {
Address address = member.getAddress();
if (!thisAddress.equals(address) && !address.equals(deadAddress)) {
sendProcessableTo(new MemberRemover(deadAddress), address);
}
}
}
public void removeMember(MemberImpl member) {
checkServiceThread();
logger.log(Level.FINEST, "ClusterManager removing " + member);
mapMembers.remove(member.getAddress());
lsMembers.remove(member);
memberMasterConfirmationTimes.remove(member);
if (!member.isLiteMember()) {
dataMemberCount.decrement();
}
}
@Override
public void setDomain(byte[] domain) {
this.domain = domain;
if (localMember != null) {
localMember.setDomain(domain);
localMember.getData(true, true);
try {
if (impl != null) impl.send(false);
} catch (Exception x) {
log.error(sm.getString("McastService.domain"), x);
}
}
}
@Override
public void setPayload(byte[] payload) {
this.payload = payload;
if (localMember != null) {
localMember.setPayload(payload);
localMember.getData(true, true);
try {
if (impl != null) impl.send(false);
} catch (Exception x) {
log.error(sm.getString("McastService.payload"), x);
}
}
}
public boolean compareMembers(MemberImpl impl1, MemberImpl impl2) {
boolean result = true;
result = result && Arrays.equals(impl1.getHost(), impl2.getHost());
result = result && Arrays.equals(impl1.getPayload(), impl2.getPayload());
result = result && Arrays.equals(impl1.getUniqueId(), impl2.getUniqueId());
result = result && impl1.getPort() == impl2.getPort();
return result;
}
private void ping(final MemberImpl memberImpl) {
memberImpl.didPing();
if (!ICMP_ENABLED) return;
node.executorManager.executeNow(
new Runnable() {
public void run() {
try {
final Address address = memberImpl.getAddress();
logger.log(Level.WARNING, thisAddress + " will ping " + address);
for (int i = 0; i < 5; i++) {
try {
if (address.getInetAddress().isReachable(null, ICMP_TTL, ICMP_TIMEOUT)) {
logger.log(Level.INFO, thisAddress + " pings successfully. Target: " + address);
return;
}
} catch (ConnectException ignored) {
// no route to host
// means we cannot connect anymore
}
}
logger.log(Level.WARNING, thisAddress + " couldn't ping " + address);
// not reachable.
enqueueAndReturn(
new Processable() {
public void process() {
doRemoveAddress(address);
}
});
} catch (Throwable ignored) {
}
}
});
}
public void finalizeJoin() {
Set<Member> members = node.getClusterImpl().getMembers();
List<AsyncRemotelyBooleanOp> calls = new ArrayList<AsyncRemotelyBooleanOp>();
for (Member m : members) {
MemberImpl member = (MemberImpl) m;
if (!member.localMember()) {
AsyncRemotelyBooleanOp op =
new AsyncRemotelyBooleanOp(new FinalizeJoin(), member.getAddress(), false);
op.execute();
calls.add(op);
}
}
for (AsyncRemotelyBooleanOp call : calls) {
call.getResultAsBoolean();
}
}
public void onRestart() {
joined.set(false);
joiner.reset();
final String uuid = UuidUtil.createMemberUuid(address);
logger.finest("Generated new UUID for local member: " + uuid);
localMember.setUuid(uuid);
}
public JoinMessage createSplitBrainJoinMessage() {
return new JoinMessage(
Packet.VERSION,
buildInfo.getBuildNumber(),
address,
localMember.getUuid(),
createConfigCheck(),
clusterService.getMemberAddresses());
}
void doRemoveAddress(Address deadAddress, boolean destroyConnection) {
checkServiceThread();
logger.log(Level.INFO, "Removing Address " + deadAddress);
if (!node.joined()) {
node.failedConnection(deadAddress);
return;
}
if (deadAddress.equals(thisAddress)) return;
if (deadAddress.equals(getMasterAddress())) {
if (node.joined()) {
MemberImpl newMaster = getNextMemberAfter(deadAddress, false, 1);
if (newMaster != null) node.setMasterAddress(newMaster.getAddress());
else node.setMasterAddress(null);
} else {
node.setMasterAddress(null);
}
logger.log(Level.FINEST, "Now Master " + node.getMasterAddress());
}
if (isMaster()) {
setJoins.remove(new MemberInfo(deadAddress));
resetMemberMasterConfirmations();
}
final Connection conn = node.connectionManager.getConnection(deadAddress);
if (destroyConnection && conn != null) {
node.connectionManager.destroyConnection(conn);
}
MemberImpl deadMember = getMember(deadAddress);
if (deadMember != null) {
removeMember(deadMember);
node.getClusterImpl().setMembers(lsMembers);
node.concurrentMapManager.syncForDead(deadMember);
node.blockingQueueManager.syncForDead(deadAddress);
node.listenerManager.syncForDead(deadAddress);
node.topicManager.syncForDead(deadAddress);
// node.getClusterImpl().setMembers(lsMembers); // shifted up to get members in syncForDead
// methods
disconnectExistingCalls(deadAddress);
if (isMaster()) {
logger.log(Level.FINEST, deadAddress + " is dead. Sending remove to all other members.");
sendRemoveMemberToOthers(deadAddress);
}
logger.log(Level.INFO, this.toString());
}
}
@Test
public void testUdpPort() throws Exception {
byte[] md1 = m1.getData();
byte[] md2 = m2.getData();
MemberImpl a1 = MemberImpl.getMember(md1);
MemberImpl a2 = MemberImpl.getMember(md2);
assertTrue(a1.getUdpPort() == a2.getUdpPort());
assertTrue(a1.getUdpPort() == udpPort);
}
public TimedMemberState getTimedMemberState() {
if (running.get()) {
final MemberStateImpl memberState = new MemberStateImpl();
createMemberState(memberState);
GroupConfig groupConfig = factory.getConfig().getGroupConfig();
TimedMemberState timedMemberState = new TimedMemberState();
timedMemberState.setMaster(factory.node.isMaster());
if (timedMemberState.getMaster()) {
timedMemberState.setMemberList(new ArrayList<String>());
Set<Member> memberSet = factory.getCluster().getMembers();
for (Member member : memberSet) {
MemberImpl memberImpl = (MemberImpl) member;
Address address = memberImpl.getAddress();
timedMemberState.getMemberList().add(address.getHost() + ":" + address.getPort());
}
}
timedMemberState.setMemberState(memberState);
timedMemberState.setClusterName(groupConfig.getName());
timedMemberState.setInstanceNames(getLongInstanceNames());
return timedMemberState;
}
return null;
}
public Member addMember(boolean checkServiceThread, MemberImpl member) {
if (checkServiceThread) {
checkServiceThread();
}
logger.log(Level.FINEST, "ClusterManager adding " + member);
if (lsMembers.contains(member)) {
for (MemberImpl m : lsMembers) {
if (m.equals(member)) {
member = m;
}
}
mapMembers.put(member.getAddress(), member);
} else {
lsMembers.add(member);
mapMembers.put(member.getAddress(), member);
if (!member.isLiteMember()) {
dataMemberCount.increment();
}
}
memberMasterConfirmationTimes.put(member, Clock.currentTimeMillis());
return member;
}
public JoinRequest createJoinRequest(boolean withCredentials) {
final Credentials credentials =
(withCredentials && securityContext != null)
? securityContext.getCredentialsFactory().newCredentials()
: null;
return new JoinRequest(
Packet.VERSION,
buildInfo.getBuildNumber(),
address,
localMember.getUuid(),
createConfigCheck(),
credentials,
config.getMemberAttributeConfig().getAttributes());
}
/**
*
* <!-- begin-user-doc -->
* <!-- end-user-doc -->
*
* @generated
*/
@Override
public void eUnset(int featureID) {
switch (featureID) {
case Roco2Package.CONSTANT_DEFINITION__ABSTRACT:
setAbstract(ABSTRACT_EDEFAULT);
return;
case Roco2Package.CONSTANT_DEFINITION__TYPE:
setType((LocalType) null);
return;
case Roco2Package.CONSTANT_DEFINITION__NAME:
setName(NAME_EDEFAULT);
return;
}
super.eUnset(featureID);
}
/**
*
* <!-- begin-user-doc -->
* <!-- end-user-doc -->
*
* @generated
*/
@Override
public void eSet(int featureID, Object newValue) {
switch (featureID) {
case Roco2Package.CONSTANT_DEFINITION__ABSTRACT:
setAbstract((Boolean) newValue);
return;
case Roco2Package.CONSTANT_DEFINITION__TYPE:
setType((LocalType) newValue);
return;
case Roco2Package.CONSTANT_DEFINITION__NAME:
setName((String) newValue);
return;
}
super.eSet(featureID, newValue);
}
@Test
public void testCompare() throws Exception {
assertTrue(m1.equals(m2));
assertTrue(m2.equals(m1));
assertTrue(p1.equals(m2));
assertFalse(m1.equals(p2));
assertFalse(m1.equals(p2));
assertFalse(m2.equals(p2));
assertFalse(p1.equals(p2));
}
public JoinRequest createJoinRequest(boolean withCredentials) {
final Credentials credentials =
(withCredentials && securityContext != null)
? securityContext.getCredentialsFactory().newCredentials()
: null;
return new JoinRequest(
Packet.VERSION,
buildNumber,
address,
localMember.getUuid(),
createConfigCheck(),
credentials,
clusterService.getSize(),
0);
}
@Before
public void setUp() throws Exception {
payload = new byte[333];
Arrays.fill(payload, (byte) 1);
m1 = new MemberImpl("localhost", 3333, 1, payload);
m2 = new MemberImpl("localhost", 3333, 1);
payload = new byte[333];
Arrays.fill(payload, (byte) 2);
p1 = new MemberImpl("127.0.0.1", 3333, 1, payload);
p2 = new MemberImpl("localhost", 3331, 1, payload);
m1.setDomain(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9});
m2.setDomain(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9});
m1.setCommand(new byte[] {1, 2, 4, 5, 6, 7, 8, 9});
m2.setCommand(new byte[] {1, 2, 4, 5, 6, 7, 8, 9});
m1.setUdpPort(udpPort);
m2.setUdpPort(m1.getUdpPort());
}
/** Sets the local member properties for broadcasting */
@Override
public void setLocalMemberProperties(
String listenHost, int listenPort, int securePort, int udpPort) {
properties.setProperty("tcpListenHost", listenHost);
properties.setProperty("tcpListenPort", String.valueOf(listenPort));
properties.setProperty("udpListenPort", String.valueOf(udpPort));
properties.setProperty("tcpSecurePort", String.valueOf(securePort));
try {
if (localMember != null) {
localMember.setHostname(listenHost);
localMember.setPort(listenPort);
} else {
localMember = new MemberImpl(listenHost, listenPort, 0);
localMember.setUniqueId(UUIDGenerator.randomUUID(true));
localMember.setPayload(getPayload());
localMember.setDomain(getDomain());
}
localMember.setSecurePort(securePort);
localMember.setUdpPort(udpPort);
localMember.getData(true, true);
} catch (IOException x) {
throw new IllegalArgumentException(x);
}
}
void updateMembers(Collection<MemberInfo> lsMemberInfos) {
checkServiceThread();
final Map<Address, MemberImpl> mapOldMembers = new HashMap<Address, MemberImpl>();
for (MemberImpl member : lsMembers) {
mapOldMembers.put(member.getAddress(), member);
}
if (mapOldMembers.size() == lsMemberInfos.size()) {
boolean same = true;
for (MemberInfo memberInfo : lsMemberInfos) {
MemberImpl member = mapOldMembers.get(memberInfo.getAddress());
if (member == null || !member.getUuid().equals(memberInfo.uuid)) {
same = false;
break;
}
}
if (same) {
logger.log(Level.FINEST, "No need to process member update...");
return;
}
}
logger.log(Level.FINEST, "Updating Members");
lsMembers.clear();
dataMemberCount.reset();
mapMembers.clear();
memberMasterConfirmationTimes.clear();
for (MemberInfo memberInfo : lsMemberInfos) {
MemberImpl member = mapOldMembers.get(memberInfo.address);
if (member == null) {
member = addMember(memberInfo.address, memberInfo.nodeType, memberInfo.uuid);
} else {
addMember(member);
}
member.didRead();
}
if (!lsMembers.contains(thisMember)) {
throw new RuntimeException("Member list doesn't contain local member!");
}
heartBeater();
node.getClusterImpl().setMembers(lsMembers);
node.setJoined();
logger.log(Level.INFO, this.toString());
}
public final void heartBeater() {
if (!node.joined() || !node.isActive()) return;
long now = Clock.currentTimeMillis();
if (isMaster()) {
List<Address> lsDeadAddresses = null;
for (MemberImpl memberImpl : lsMembers) {
final Address address = memberImpl.getAddress();
if (!thisAddress.equals(address)) {
try {
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null && conn.live()) {
if ((now - memberImpl.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
conn = null;
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because of timeout since last read");
lsDeadAddresses.add(address);
} else if ((now - memberImpl.getLastRead()) >= 5000
&& (now - memberImpl.getLastPing()) >= 5000) {
ping(memberImpl);
}
if ((now - memberImpl.getLastWrite()) > 500) {
sendHeartbeat(conn);
}
Long lastConfirmation = memberMasterConfirmationTimes.get(memberImpl);
if (lastConfirmation == null
|| (now - lastConfirmation > MAX_NO_MASTER_CONFIRMATION_MILLIS)) {
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because it has not sent a master confirmation recently");
lsDeadAddresses.add(address);
}
} else if (conn == null && (now - memberImpl.getLastRead()) > 5000) {
logMissingConnection(address);
memberImpl.didRead();
}
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);
}
}
}
if (lsDeadAddresses != null) {
for (Address address : lsDeadAddresses) {
logger.log(Level.FINEST, "No heartbeat should remove " + address);
doRemoveAddress(address);
}
}
} else {
// send heartbeat to master
Address masterAddress = getMasterAddress();
if (masterAddress != null) {
final Connection connMaster = node.connectionManager.getOrConnect(masterAddress);
MemberImpl masterMember = getMember(masterAddress);
boolean removed = false;
if (masterMember != null) {
if ((now - masterMember.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
logger.log(
Level.WARNING, "Master node has timed out its heartbeat and will be removed");
doRemoveAddress(masterAddress);
removed = true;
} else if ((now - masterMember.getLastRead()) >= 5000
&& (now - masterMember.getLastPing()) >= 5000) {
ping(masterMember);
}
}
if (!removed) {
sendHeartbeat(connMaster);
}
}
for (MemberImpl member : lsMembers) {
if (!member.localMember()) {
Address address = member.getAddress();
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null) {
sendHeartbeat(conn);
} else {
logger.log(Level.FINEST, "Could not connect to " + address + " to send heartbeat");
}
}
}
}
}
@Test
public void testSerializationOne() throws Exception {
MemberImpl m = m1;
byte[] md1 = m.getData(false, true);
byte[] mda1 = m.getData(false, false);
assertTrue(Arrays.equals(md1, mda1));
assertTrue(md1 == mda1);
mda1 = m.getData(true, true);
MemberImpl ma1 = MemberImpl.getMember(mda1);
assertTrue(compareMembers(m, ma1));
mda1 = p1.getData(false);
assertFalse(Arrays.equals(md1, mda1));
ma1 = MemberImpl.getMember(mda1);
assertTrue(compareMembers(p1, ma1));
md1 = m.getData(true, true);
Thread.sleep(50);
mda1 = m.getData(true, true);
MemberImpl a1 = MemberImpl.getMember(md1);
MemberImpl a2 = MemberImpl.getMember(mda1);
assertTrue(a1.equals(a2));
assertFalse(Arrays.equals(md1, mda1));
}
/*package*/ FieldImpl(
String simpleName, ClassImpl containingClass, String qualifiedTypeClassName) {
super(containingClass);
super.setSimpleName(simpleName);
mTypeClassRef = QualifiedJClassRef.create(qualifiedTypeClassName, containingClass);
}
/** Return the local member name */
public String getLocalMemberName() {
return localMember.toString();
}
void handleJoinRequest(JoinRequest joinRequest) {
final long now = Clock.currentTimeMillis();
String msg =
"Handling join from "
+ joinRequest.address
+ ", inProgress: "
+ joinInProgress
+ (timeToStartJoin > 0 ? ", timeToStart: " + (timeToStartJoin - now) : "");
logger.log(Level.FINEST, msg);
boolean validJoinRequest;
try {
validJoinRequest = node.validateJoinRequest(joinRequest);
} catch (Exception e) {
validJoinRequest = false;
}
final Connection conn = joinRequest.getConnection();
if (validJoinRequest) {
final MemberImpl member = getMember(joinRequest.address);
if (member != null) {
if (joinRequest.getUuid().equals(member.getUuid())) {
String message = "Ignoring join request, member already exists.. => " + joinRequest;
logger.log(Level.FINEST, message);
// send members update back to node trying to join again...
final long clusterTime = node.getClusterImpl().getClusterTime();
sendProcessableTo(new MembersUpdateCall(lsMembers, clusterTime), conn);
sendProcessableTo(new SyncProcess(), conn);
return;
}
// If this node is master then remove old member and process join request.
// If requesting address is equal to master node's address, that means master node
// somehow disconnected and wants to join back.
// So drop old member and process join request if this node becomes master.
if (isMaster() || member.getAddress().equals(getMasterAddress())) {
logger.log(
Level.WARNING,
"New join request has been received from an existing endpoint! => "
+ member
+ " Removing old member and processing join request...");
// If existing connection of endpoint is different from current connection
// destroy it, otherwise keep it.
// final Connection existingConnection =
// node.connectionManager.getConnection(joinRequest.address);
// final boolean destroyExistingConnection = existingConnection !=
// conn;
doRemoveAddress(member.getAddress(), false);
}
}
if (!node.getConfig().getNetworkConfig().getJoin().getMulticastConfig().isEnabled()) {
if (node.isActive() && node.joined() && node.getMasterAddress() != null && !isMaster()) {
sendProcessableTo(new Master(node.getMasterAddress()), conn);
}
}
if (isMaster() && node.joined() && node.isActive()) {
final MemberInfo newMemberInfo =
new MemberInfo(joinRequest.address, joinRequest.nodeType, joinRequest.getUuid());
if (node.securityContext != null && !setJoins.contains(newMemberInfo)) {
final ILogger securityLogger = node.loggingService.getLogger("com.hazelcast.security");
final Credentials cr = joinRequest.getCredentials();
if (cr == null) {
securityLogger.log(
Level.SEVERE,
"Expecting security credentials "
+ "but credentials could not be found in JoinRequest!");
sendAuthFail(conn);
return;
} else {
try {
LoginContext lc = node.securityContext.createMemberLoginContext(cr);
lc.login();
} catch (LoginException e) {
securityLogger.log(
Level.SEVERE,
"Authentication has failed for "
+ cr.getPrincipal()
+ '@'
+ cr.getEndpoint()
+ " => ("
+ e.getMessage()
+ ")");
securityLogger.log(Level.FINEST, e.getMessage(), e);
sendAuthFail(conn);
return;
}
}
}
if (joinRequest.to != null && !joinRequest.to.equals(thisAddress)) {
sendProcessableTo(new Master(node.getMasterAddress()), conn);
return;
}
if (!joinInProgress) {
if (firstJoinRequest != 0
&& now - firstJoinRequest >= MAX_WAIT_SECONDS_BEFORE_JOIN * 1000) {
startJoin();
} else {
if (setJoins.add(newMemberInfo)) {
sendProcessableTo(new Master(node.getMasterAddress()), conn);
if (firstJoinRequest == 0) {
firstJoinRequest = now;
}
if (now - firstJoinRequest < MAX_WAIT_SECONDS_BEFORE_JOIN * 1000) {
timeToStartJoin = now + WAIT_MILLIS_BEFORE_JOIN;
}
}
if (now > timeToStartJoin) {
startJoin();
}
}
}
}
} else {
conn.close();
}
}
/** Return the local member */
@Override
public Member getLocalMember(boolean alive) {
if (alive && localMember != null && impl != null)
localMember.setMemberAliveTime(System.currentTimeMillis() - impl.getServiceStartTime());
return localMember;
}
@Override
public void start(int level) throws java.lang.Exception {
hasProperty(properties, "mcastPort");
hasProperty(properties, "mcastAddress");
hasProperty(properties, "memberDropTime");
hasProperty(properties, "mcastFrequency");
hasProperty(properties, "tcpListenPort");
hasProperty(properties, "tcpListenHost");
hasProperty(properties, "tcpSecurePort");
hasProperty(properties, "udpListenPort");
if (impl != null) {
impl.start(level);
return;
}
String host = getProperties().getProperty("tcpListenHost");
int port = Integer.parseInt(getProperties().getProperty("tcpListenPort"));
int securePort = Integer.parseInt(getProperties().getProperty("tcpSecurePort"));
int udpPort = Integer.parseInt(getProperties().getProperty("udpListenPort"));
if (localMember == null) {
localMember = new MemberImpl(host, port, 100);
localMember.setUniqueId(UUIDGenerator.randomUUID(true));
} else {
localMember.setHostname(host);
localMember.setPort(port);
localMember.setMemberAliveTime(100);
}
localMember.setSecurePort(securePort);
localMember.setUdpPort(udpPort);
if (this.payload != null) localMember.setPayload(payload);
if (this.domain != null) localMember.setDomain(domain);
localMember.setServiceStartTime(System.currentTimeMillis());
java.net.InetAddress bind = null;
if (properties.getProperty("mcastBindAddress") != null) {
bind = java.net.InetAddress.getByName(properties.getProperty("mcastBindAddress"));
}
int ttl = -1;
int soTimeout = -1;
if (properties.getProperty("mcastTTL") != null) {
try {
ttl = Integer.parseInt(properties.getProperty("mcastTTL"));
} catch (Exception x) {
log.error(sm.getString("McastService.parseTTL", properties.getProperty("mcastTTL")), x);
}
}
if (properties.getProperty("mcastSoTimeout") != null) {
try {
soTimeout = Integer.parseInt(properties.getProperty("mcastSoTimeout"));
} catch (Exception x) {
log.error(
sm.getString("McastService.parseSoTimeout", properties.getProperty("mcastSoTimeout")),
x);
}
}
impl =
new McastServiceImpl(
localMember,
Long.parseLong(properties.getProperty("mcastFrequency")),
Long.parseLong(properties.getProperty("memberDropTime")),
Integer.parseInt(properties.getProperty("mcastPort")),
bind,
java.net.InetAddress.getByName(properties.getProperty("mcastAddress")),
ttl,
soTimeout,
this,
this,
Boolean.valueOf(properties.getProperty("localLoopbackDisabled", "false"))
.booleanValue());
String value = properties.getProperty("recoveryEnabled", "true");
boolean recEnabled = Boolean.valueOf(value).booleanValue();
impl.setRecoveryEnabled(recEnabled);
int recCnt = Integer.parseInt(properties.getProperty("recoveryCounter", "10"));
impl.setRecoveryCounter(recCnt);
long recSlpTime = Long.parseLong(properties.getProperty("recoverySleepTime", "5000"));
impl.setRecoverySleepTime(recSlpTime);
impl.start(level);
}