protected void updateMembersRef() {
final Map<Address, MemberImpl> map = new LinkedHashMap<Address, MemberImpl>(members.size());
for (MemberImpl member : members) {
map.put(member.getAddress(), member);
}
clusterService.setMembersRef(Collections.unmodifiableMap(map));
}
@Override
public boolean isMemberSafe(Member member) {
if (member == null) {
throw new NullPointerException("Parameter member should not be null");
}
final MemberImpl localMember = getNode().getLocalMember();
if (localMember.equals(member)) {
return isLocalMemberSafe();
}
final Address target = member.getAddress();
final Operation operation = new SafeStateCheckOperation();
final InternalCompletableFuture future =
getNode()
.getNodeEngine()
.getOperationService()
.invokeOnTarget(InternalPartitionService.SERVICE_NAME, operation, target);
boolean safe;
try {
final Object result = future.get(10, TimeUnit.SECONDS);
safe = (Boolean) result;
} catch (Throwable t) {
safe = false;
logger.warning("Error while querying member's safe state [" + member + "]", t);
}
return safe;
}
public void connectionRemoved(Connection connection) {
if (connection.isClient() && connection instanceof TcpIpConnection && nodeEngine.isActive()) {
final ClientEndpoint endpoint = endpoints.get(connection);
if (endpoint != null
&& node.getLocalMember().getUuid().equals(endpoint.getPrincipal().getOwnerUuid())) {
removeEndpoint(connection, true);
if (!endpoint.isFirstConnection()) {
return;
}
NodeEngine nodeEngine = node.nodeEngine;
final Collection<MemberImpl> memberList = nodeEngine.getClusterService().getMemberList();
for (MemberImpl member : memberList) {
final ClientDisconnectionOperation op =
new ClientDisconnectionOperation(endpoint.getUuid());
op.setNodeEngine(nodeEngine)
.setServiceName(SERVICE_NAME)
.setService(this)
.setResponseHandler(ResponseHandlerFactory.createEmptyResponseHandler());
if (member.localMember()) {
nodeEngine.getOperationService().runOperation(op);
} else {
nodeEngine.getOperationService().send(op, member.getAddress());
}
}
}
}
}
private void enableStatisticManagementOnNodes(
String cacheName, boolean statOrMan, boolean enabled) {
if (isClosed()) {
throw new IllegalStateException();
}
if (cacheName == null) {
throw new NullPointerException();
}
final ClientInvocationService invocationService = clientContext.getInvocationService();
final Collection<MemberImpl> members = clientContext.getClusterService().getMemberList();
final Collection<Future> futures = new ArrayList<Future>();
for (MemberImpl member : members) {
try {
CacheManagementConfigRequest request =
new CacheManagementConfigRequest(
getCacheNameWithPrefix(cacheName), statOrMan, enabled, member.getAddress());
final Future future = invocationService.invokeOnTarget(request, member.getAddress());
futures.add(future);
} catch (Exception e) {
ExceptionUtil.sneakyThrow(e);
}
}
// make sure all configs are created
try {
FutureUtil.waitWithDeadline(
futures, CacheProxyUtil.AWAIT_COMPLETION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} catch (TimeoutException e) {
logger.warning(e);
}
}
public void invalidateAllNearCaches(String mapName, Set<Data> keys) {
if (!isNearCacheEnabled(mapName)) {
return;
}
if (keys == null || keys.isEmpty()) {
return;
}
// send operation.
Operation operation =
new NearCacheKeySetInvalidationOperation(mapName, keys).setServiceName(SERVICE_NAME);
Collection<MemberImpl> members = nodeEngine.getClusterService().getMemberList();
for (MemberImpl member : members) {
try {
if (member.localMember()) continue;
nodeEngine.getOperationService().send(operation, member.getAddress());
} catch (Throwable throwable) {
logger.warning(throwable);
}
}
// below local invalidation is for the case the data is cached before partition is
// owned/migrated
for (final Data key : keys) {
invalidateNearCache(mapName, key);
}
}
private void updateMembersRef() {
final Map<Address, MemberImpl> map = new LinkedHashMap<Address, MemberImpl>(members.size());
for (MemberImpl member : members) {
map.put(member.getAddress(), member);
}
membersRef.set(Collections.unmodifiableMap(map));
}
private Collection<InetSocketAddress> getClusterAddresses() {
final List<InetSocketAddress> socketAddresses = new LinkedList<InetSocketAddress>();
for (MemberImpl member : members) {
socketAddresses.add(member.getInetSocketAddress());
}
Collections.shuffle(socketAddresses);
return socketAddresses;
}
public MemberImpl getMember(String uuid) {
final Collection<MemberImpl> memberList = getMemberList();
for (MemberImpl member : memberList) {
if (uuid.equals(member.getUuid())) {
return member;
}
}
return null;
}
private boolean hasMemberLeft(Invocation invocation) {
MemberImpl targetMember = invocation.targetMember;
if (targetMember == null) {
Address invTarget = invocation.invTarget;
return leftMember.getAddress().equals(invTarget);
} else {
return leftMember.getUuid().equals(targetMember.getUuid());
}
}
@Override
public void handleMemberPacket(final Packet packet) {
final Address endPoint = packet.getConn().getEndPoint();
if (endPoint != null) {
final MemberImpl member = node.clusterService.getMember(endPoint);
if (member != null) {
member.didRead();
}
}
nodeEngine.handlePacket(packet);
}
public boolean write(SocketWritable socketWritable) {
final Packet packet = (Packet) socketWritable;
if (nodeEngine.getNode().isActive()) {
packet.setConn(thisConnection);
MemberImpl member = nodeEngine.getClusterService().getMember(thisAddress);
if (member != null) {
member.didRead();
}
nodeEngine.handlePacket(packet);
return true;
}
return false;
}
public boolean write(SocketWritable socketWritable) {
final Packet packet = (Packet) socketWritable;
if (nodeEngine.getNode().isActive()) {
Packet newPacket = readFromPacket(packet);
MemberImpl member = nodeEngine.getClusterService().getMember(localEndpoint);
if (member != null) {
member.didRead();
}
nodeEngine.getPacketTransceiver().receive(newPacket);
return true;
}
return false;
}
/*
* Sets up a situation where the member list is out of order on node2. Both
* node2 and node1 think they are masters and both think each other are in
* their clusters.
*/
@Test
public void testOutOfSyncMemberListTwoMasters() throws Exception {
Config c1 = buildConfig(false);
Config c2 = buildConfig(false);
Config c3 = buildConfig(false);
c1.getNetworkConfig().setPort(45701);
c2.getNetworkConfig().setPort(45702);
c3.getNetworkConfig().setPort(45703);
List<String> allMembers = Arrays.asList("127.0.0.1:45701, 127.0.0.1:45702, 127.0.0.1:45703");
c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1);
final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2);
final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3);
final MemberImpl m1 = (MemberImpl) h1.getCluster().getLocalMember();
final MemberImpl m2 = (MemberImpl) h2.getCluster().getLocalMember();
final MemberImpl m3 = (MemberImpl) h3.getCluster().getLocalMember();
// All three nodes join into one cluster
assertEquals(3, h1.getCluster().getMembers().size());
assertEquals(3, h2.getCluster().getMembers().size());
assertEquals(3, h3.getCluster().getMembers().size());
final Node n2 = TestUtil.getNode(h2);
// Simulates node2 getting an out of order member list. That causes node2 to think it's the
// master.
List<MemberInfo> members = new ArrayList<MemberInfo>();
members.add(new MemberInfo(m2.getAddress(), m2.getUuid()));
members.add(new MemberInfo(m3.getAddress(), m3.getUuid()));
members.add(new MemberInfo(m1.getAddress(), m1.getUuid()));
n2.clusterService.updateMembers(members);
n2.setMasterAddress(m2.getAddress());
// Give the cluster some time to figure things out. The merge and heartbeat code should have
// kicked in by this point
Thread.sleep(30 * 1000);
assertEquals(m1, h1.getCluster().getMembers().iterator().next());
assertEquals(m1, h2.getCluster().getMembers().iterator().next());
assertEquals(m1, h3.getCluster().getMembers().iterator().next());
assertEquals(3, h1.getCluster().getMembers().size());
assertEquals(3, h2.getCluster().getMembers().size());
assertEquals(3, h3.getCluster().getMembers().size());
}
private void listenMembershipEvents() throws IOException {
final SerializationService serializationService = clusterService.getSerializationService();
while (!Thread.currentThread().isInterrupted()) {
final Data clientResponseData = conn.read();
final ClientResponse clientResponse = serializationService.toObject(clientResponseData);
final Object eventObject = serializationService.toObject(clientResponse.getResponse());
final ClientMembershipEvent event = (ClientMembershipEvent) eventObject;
final MemberImpl member = (MemberImpl) event.getMember();
boolean membersUpdated = false;
if (event.getEventType() == MembershipEvent.MEMBER_ADDED) {
members.add(member);
membersUpdated = true;
} else if (event.getEventType() == ClientMembershipEvent.MEMBER_REMOVED) {
members.remove(member);
membersUpdated = true;
// getConnectionManager().removeConnectionPool(member.getAddress());
// //TODO
} else if (event.getEventType() == ClientMembershipEvent.MEMBER_ATTRIBUTE_CHANGED) {
MemberAttributeChange memberAttributeChange = event.getMemberAttributeChange();
Map<Address, MemberImpl> memberMap = clusterService.getMembersRef();
if (memberMap != null) {
for (MemberImpl target : memberMap.values()) {
if (target.getUuid().equals(memberAttributeChange.getUuid())) {
final MemberAttributeOperationType operationType =
memberAttributeChange.getOperationType();
final String key = memberAttributeChange.getKey();
final Object value = memberAttributeChange.getValue();
target.updateAttribute(operationType, key, value);
MemberAttributeEvent memberAttributeEvent =
new MemberAttributeEvent(client.getCluster(), target, operationType, key, value);
clusterService.fireMemberAttributeEvent(memberAttributeEvent);
break;
}
}
}
}
if (membersUpdated) {
((ClientPartitionServiceImpl) client.getClientPartitionService()).refreshPartitions();
updateMembersRef();
LOGGER.info(clusterService.membersString());
clusterService.fireMembershipEvent(
new MembershipEvent(
client.getCluster(),
member,
event.getEventType(),
Collections.unmodifiableSet(new LinkedHashSet<Member>(members))));
}
}
}
@Override
public MemberImpl getMember(String uuid) {
if (uuid == null) {
return null;
}
Map<Address, MemberImpl> memberMap = membersMapRef.get();
for (MemberImpl member : memberMap.values()) {
if (uuid.equals(member.getUuid())) {
return member;
}
}
return null;
}
private Collection<InetSocketAddress> getSocketAddresses() throws Exception {
final List<InetSocketAddress> socketAddresses = new LinkedList<InetSocketAddress>();
if (!members.isEmpty()) {
for (MemberImpl member : members) {
socketAddresses.add(member.getInetSocketAddress());
}
Collections.shuffle(socketAddresses);
}
for (AddressProvider addressProvider : addressProviders) {
socketAddresses.addAll(addressProvider.loadAddresses());
}
return socketAddresses;
}
public void updateMemberAttribute(
String uuid, MemberAttributeOperationType operationType, String key, Object value) {
lock.lock();
try {
Map<Address, MemberImpl> memberMap = membersMapRef.get();
for (MemberImpl member : memberMap.values()) {
if (member.getUuid().equals(uuid)) {
if (!member.equals(getLocalMember())) {
member.updateAttribute(operationType, key, value);
}
sendMemberAttributeEvent(member, operationType, key, value);
break;
}
}
} finally {
lock.unlock();
}
}
private void setMembers(MemberImpl... members) {
if (members == null || members.length == 0) {
return;
}
if (logger.isFinestEnabled()) {
logger.finest("Updating members " + Arrays.toString(members));
}
lock.lock();
try {
// !!! ORDERED !!!
Map<Address, MemberImpl> memberMap = new LinkedHashMap<Address, MemberImpl>();
for (MemberImpl member : members) {
memberMap.put(member.getAddress(), member);
}
setMembersRef(memberMap);
} finally {
lock.unlock();
}
}
@Override
public void run0() {
lastHeartbeatPerMember.remove(leftMember.getAddress());
for (Invocation invocation : invocationRegistry) {
if (hasMemberLeft(invocation)) {
invocation.notifyError(new MemberLeftException(leftMember));
}
}
}
public void invalidateAllNearCaches(String mapName, Data key) {
if (!isNearCacheEnabled(mapName)) {
return;
}
Collection<MemberImpl> members = nodeEngine.getClusterService().getMemberList();
for (MemberImpl member : members) {
try {
if (member.localMember()) continue;
Operation operation =
new InvalidateNearCacheOperation(mapName, key).setServiceName(SERVICE_NAME);
nodeEngine.getOperationService().send(operation, member.getAddress());
} catch (Throwable throwable) {
throw new HazelcastException(throwable);
}
}
// below local invalidation is for the case the data is cached before partition is
// owned/migrated
invalidateNearCache(mapName, key);
}
public void updateMembers(Collection<MemberInfo> members) {
lock.lock();
try {
Map<Address, MemberImpl> currentMemberMap = membersMapRef.get();
if (!shouldProcessMemberUpdate(currentMemberMap, members)) {
return;
}
String scopeId = thisAddress.getScopeId();
Collection<MemberImpl> newMembers = new LinkedList<MemberImpl>();
MemberImpl[] updatedMembers = new MemberImpl[members.size()];
int memberIndex = 0;
for (MemberInfo memberInfo : members) {
MemberImpl member = currentMemberMap.get(memberInfo.getAddress());
if (member == null) {
member = createMember(memberInfo, scopeId);
newMembers.add(member);
long now = Clock.currentTimeMillis();
clusterHeartbeatManager.onHeartbeat(member, now);
clusterHeartbeatManager.acceptMasterConfirmation(member, now);
Map<Address, MemberImpl> membersRemovedInNotActiveState =
new LinkedHashMap<Address, MemberImpl>(membersRemovedInNotActiveStateRef.get());
membersRemovedInNotActiveState.remove(member.getAddress());
membersRemovedInNotActiveStateRef.set(
Collections.unmodifiableMap(membersRemovedInNotActiveState));
}
updatedMembers[memberIndex++] = member;
}
setMembers(updatedMembers);
sendMembershipEvents(currentMemberMap.values(), newMembers);
clusterJoinManager.reset();
clusterHeartbeatManager.heartBeat();
node.setJoined();
logger.info(membersString());
} finally {
lock.unlock();
}
}
private void testMemberLeftException(String uuid, String host, int port, Member member)
throws Exception {
MemberLeftException exception = new MemberLeftException(member);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
ObjectOutputStream out = new ObjectOutputStream(bout);
out.writeObject(exception);
ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
ObjectInputStream in = new ObjectInputStream(bin);
MemberLeftException exception2 = (MemberLeftException) in.readObject();
MemberImpl member2 = (MemberImpl) exception2.getMember();
assertEquals(uuid, member2.getUuid());
assertEquals(host, member2.getAddress().getHost());
assertEquals(port, member2.getAddress().getPort());
assertEquals(member.isLiteMember(), member2.isLiteMember());
assertEquals(member.getVersion(), member2.getVersion());
}
private void loadInitialMemberList() throws IOException {
final SerializationService serializationService = getSerializationService();
final Data request = serializationService.toData(new AddMembershipListenerRequest());
conn.write(request);
final Data response = conn.read();
SerializableCollection coll =
ErrorHandler.returnResultOrThrowException(serializationService.toObject(response));
Map<String, MemberImpl> prevMembers = Collections.emptyMap();
if (!members.isEmpty()) {
prevMembers = new HashMap<String, MemberImpl>(members.size());
for (MemberImpl member : members) {
prevMembers.put(member.getUuid(), member);
}
members.clear();
}
for (Data d : coll.getCollection()) {
members.add((MemberImpl) serializationService.toObject(d));
}
updateMembersRef();
logger.info(membersString());
final List<MembershipEvent> events = new LinkedList<MembershipEvent>();
final Set<Member> eventMembers =
Collections.unmodifiableSet(new LinkedHashSet<Member>(members));
for (MemberImpl member : members) {
final MemberImpl former = prevMembers.remove(member.getUuid());
if (former == null) {
events.add(
new MembershipEvent(
client.getCluster(), member, MembershipEvent.MEMBER_ADDED, eventMembers));
}
}
for (MemberImpl member : prevMembers.values()) {
events.add(
new MembershipEvent(
client.getCluster(), member, MembershipEvent.MEMBER_REMOVED, eventMembers));
}
for (MembershipEvent event : events) {
fireMembershipEvent(event);
}
}
private void loadInitialMemberList() throws Exception {
final SerializationService serializationService = clusterService.getSerializationService();
final AddMembershipListenerRequest request = new AddMembershipListenerRequest();
final SerializableCollection coll =
(SerializableCollection) connectionManager.sendAndReceive(request, conn);
Map<String, MemberImpl> prevMembers = Collections.emptyMap();
if (!members.isEmpty()) {
prevMembers = new HashMap<String, MemberImpl>(members.size());
for (MemberImpl member : members) {
prevMembers.put(member.getUuid(), member);
}
members.clear();
}
for (Data data : coll) {
members.add((MemberImpl) serializationService.toObject(data));
}
updateMembersRef();
LOGGER.info(clusterService.membersString());
final List<MembershipEvent> events = new LinkedList<MembershipEvent>();
final Set<Member> eventMembers =
Collections.unmodifiableSet(new LinkedHashSet<Member>(members));
for (MemberImpl member : members) {
final MemberImpl former = prevMembers.remove(member.getUuid());
if (former == null) {
events.add(
new MembershipEvent(
client.getCluster(), member, MembershipEvent.MEMBER_ADDED, eventMembers));
}
}
for (MemberImpl member : prevMembers.values()) {
events.add(
new MembershipEvent(
client.getCluster(), member, MembershipEvent.MEMBER_REMOVED, eventMembers));
}
for (MembershipEvent event : events) {
clusterService.fireMembershipEvent(event);
}
latch.countDown();
}
private void listenMembershipEvents() throws IOException {
final SerializationService serializationService = getSerializationService();
while (!Thread.currentThread().isInterrupted()) {
final Data eventData = conn.read();
final ClientMembershipEvent event =
(ClientMembershipEvent) serializationService.toObject(eventData);
final MemberImpl member = (MemberImpl) event.getMember();
if (event.getEventType() == MembershipEvent.MEMBER_ADDED) {
members.add(member);
} else {
members.remove(member);
getConnectionManager().removeConnectionPool(member.getAddress());
}
updateMembersRef();
logger.info(membersString());
fireMembershipEvent(
new MembershipEvent(
client.getCluster(),
member,
event.getEventType(),
Collections.unmodifiableSet(new LinkedHashSet<Member>(members))));
}
}
private void removeMember(MemberImpl deadMember) {
logger.info("Removing " + deadMember);
lock.lock();
try {
Map<Address, MemberImpl> members = membersMapRef.get();
final Address deadAddress = deadMember.getAddress();
if (members.containsKey(deadAddress)) {
// !!! ORDERED !!!
Map<Address, MemberImpl> newMembers = new LinkedHashMap<Address, MemberImpl>(members);
newMembers.remove(deadAddress);
clusterHeartbeatManager.removeMember(deadMember);
setMembersRef(newMembers);
if (node.isMaster()) {
if (logger.isFinestEnabled()) {
logger.finest(deadMember + " is dead, sending remove to all other members...");
}
invokeMemberRemoveOperation(deadAddress);
}
final ClusterState clusterState = clusterStateManager.getState();
if (clusterState == ClusterState.FROZEN || clusterState == ClusterState.PASSIVE) {
if (logger.isFinestEnabled()) {
logger.finest(
deadMember
+ " is dead, added to members left while cluster is "
+ clusterState
+ " state");
}
Map<Address, MemberImpl> membersRemovedInNotActiveState =
new LinkedHashMap<Address, MemberImpl>(membersRemovedInNotActiveStateRef.get());
membersRemovedInNotActiveState.put(deadAddress, deadMember);
membersRemovedInNotActiveStateRef.set(
Collections.unmodifiableMap(membersRemovedInNotActiveState));
final InternalPartitionServiceImpl partitionService = node.partitionService;
partitionService.cancelReplicaSyncRequestsTo(deadAddress);
} else {
onMemberRemove(deadMember, newMembers);
}
// async events
sendMembershipEventNotifications(
deadMember, unmodifiableSet(new LinkedHashSet<Member>(newMembers.values())), false);
}
} finally {
lock.unlock();
}
}
