private void ensureConnectionToAllMembers() {
boolean allConnected = false;
if (node.joined()) {
logger.finest("Waiting for all connections");
int connectAllWaitSeconds =
node.groupProperties.getSeconds(GroupProperty.CONNECT_ALL_WAIT_SECONDS);
int checkCount = 0;
while (checkCount++ < connectAllWaitSeconds && !allConnected) {
try {
//noinspection BusyWait
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException ignored) {
EmptyStatement.ignore(ignored);
}
allConnected = true;
Collection<Member> members = clusterService.getMembers();
for (Member member : members) {
if (!member.localMember()
&& node.connectionManager.getOrConnect(member.getAddress()) == null) {
allConnected = false;
if (logger.isFinestEnabled()) {
logger.finest("Not-connected to " + member.getAddress());
}
}
}
}
}
}
private void assignNewMaster() {
Address oldMasterAddress = node.getMasterAddress();
if (node.joined()) {
Collection<Member> members = getMembers();
Member newMaster = null;
int size = members.size();
if (size > 1) {
Iterator<Member> iterator = members.iterator();
Member member = iterator.next();
if (member.getAddress().equals(oldMasterAddress)) {
newMaster = iterator.next();
} else {
logger.severe(
format(
"Old master %s is dead, but the first of member list is a different member %s!",
oldMasterAddress, member));
newMaster = member;
}
} else {
logger.warning(
format(
"Old master %s is dead and this node is not master, "
+ "but member list contains only %d members: %s",
oldMasterAddress, size, members));
}
logger.info(
format(
"Old master %s left the cluster, assigning new master %s",
oldMasterAddress, newMaster));
if (newMaster != null) {
node.setMasterAddress(newMaster.getAddress());
} else {
node.setMasterAddress(null);
}
} else {
node.setMasterAddress(null);
}
if (logger.isFinestEnabled()) {
logger.finest(
format("Old master: %s, new master: %s ", oldMasterAddress, node.getMasterAddress()));
}
if (node.isMaster()) {
clusterHeartbeatManager.resetMemberMasterConfirmations();
clusterClock.reset();
} else {
clusterHeartbeatManager.sendMasterConfirmation();
}
}
private List<MembershipEvent> detectMembershipEvents(Map<String, Member> prevMembers) {
final List<MembershipEvent> events = new LinkedList<MembershipEvent>();
final Set<Member> eventMembers =
Collections.unmodifiableSet(new LinkedHashSet<Member>(members));
for (Member member : members) {
final Member former = prevMembers.remove(member.getUuid());
if (former == null) {
events.add(
new MembershipEvent(
client.getCluster(), member, MembershipEvent.MEMBER_ADDED, eventMembers));
}
}
for (Member member : prevMembers.values()) {
events.add(
new MembershipEvent(
client.getCluster(), member, MembershipEvent.MEMBER_REMOVED, eventMembers));
Address address = member.getAddress();
if (clusterService.getMember(address) == null) {
final Connection connection = connectionManager.getConnection(address);
if (connection != null) {
connectionManager.destroyConnection(connection);
}
}
}
return events;
}
private void shutdownNodes() {
final Operation op = new ShutdownNodeOperation();
logger.info("Sending shutting down operations to all members...");
Collection<Member> members = getMembers(NON_LOCAL_MEMBER_SELECTOR);
final long timeout =
node.groupProperties.getNanos(GroupProperty.CLUSTER_SHUTDOWN_TIMEOUT_SECONDS);
final long startTime = System.nanoTime();
while ((System.nanoTime() - startTime) < timeout && !members.isEmpty()) {
for (Member member : members) {
nodeEngine.getOperationService().send(op, member.getAddress());
}
try {
Thread.sleep(CLUSTER_SHUTDOWN_SLEEP_DURATION_IN_MILLIS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.warning("Shutdown sleep interrupted. ", e);
break;
}
members = getMembers(NON_LOCAL_MEMBER_SELECTOR);
}
logger.info(
"Number of other nodes remaining: "
+ getSize(NON_LOCAL_MEMBER_SELECTOR)
+ ". Shutting down itself.");
node.shutdown(false);
}
@Override
public Xid[] recover(int flag) throws XAException {
NodeEngine nodeEngine = getNodeEngine();
XAService xaService = getService();
OperationService operationService = nodeEngine.getOperationService();
ClusterService clusterService = nodeEngine.getClusterService();
Collection<Member> memberList = clusterService.getMembers();
List<InternalCompletableFuture<SerializableList>> futureList =
new ArrayList<InternalCompletableFuture<SerializableList>>();
for (Member member : memberList) {
if (member.localMember()) {
continue;
}
CollectRemoteTransactionsOperation op = new CollectRemoteTransactionsOperation();
Address address = member.getAddress();
InternalCompletableFuture<SerializableList> future =
operationService.invokeOnTarget(SERVICE_NAME, op, address);
futureList.add(future);
}
HashSet<SerializableXID> xids = new HashSet<SerializableXID>();
xids.addAll(xaService.getPreparedXids());
for (InternalCompletableFuture<SerializableList> future : futureList) {
SerializableList xidSet = future.getSafely();
for (Data xidData : xidSet) {
SerializableXID xid = nodeEngine.toObject(xidData);
xids.add(xid);
}
}
return xids.toArray(new SerializableXID[xids.size()]);
}
private void updateMembersRef() {
final Map<Address, Member> map = new LinkedHashMap<Address, Member>(members.size());
for (Member member : members) {
map.put(member.getAddress(), member);
}
clusterService.setMembersRef(Collections.unmodifiableMap(map));
}
public Address findNextAddressToSendCreateRequest() {
int clusterSize = client.getClientClusterService().getSize();
Member liteMember = null;
final LoadBalancer loadBalancer = client.getLoadBalancer();
for (int i = 0; i < clusterSize; i++) {
Member member = loadBalancer.next();
if (member != null && !member.isLiteMember()) {
return member.getAddress();
} else if (liteMember == null) {
liteMember = member;
}
}
return liteMember != null ? liteMember.getAddress() : null;
}
@Override
public boolean isMemberSafe(Member member) {
if (member == null) {
throw new NullPointerException("Parameter member should not be null");
}
final MemberImpl localMember = getNode().getLocalMember();
if (localMember.equals(member)) {
return isLocalMemberSafe();
}
final Address target = member.getAddress();
final Operation operation = new SafeStateCheckOperation();
final InternalCompletableFuture future =
getNode()
.getNodeEngine()
.getOperationService()
.invokeOnTarget(InternalPartitionService.SERVICE_NAME, operation, target);
boolean safe;
try {
final Object result = future.get(10, TimeUnit.SECONDS);
safe = (Boolean) result;
} catch (Throwable t) {
safe = false;
logger.warning("Error while querying member's safe state [" + member + "]", t);
}
return safe;
}
@Override
public boolean isClusterSafe() {
final Node node = getNode();
final Collection<Member> memberList = node.clusterService.getMembers();
if (memberList == null || memberList.isEmpty() || memberList.size() < 2) {
return true;
}
final Collection<Future> futures = new ArrayList<Future>(memberList.size());
for (Member member : memberList) {
final Address target = member.getAddress();
final Operation operation = new SafeStateCheckOperation();
final InternalCompletableFuture future =
node.getNodeEngine()
.getOperationService()
.invokeOnTarget(InternalPartitionService.SERVICE_NAME, operation, target);
futures.add(future);
}
// todo this max wait is appropriate?
final int maxWaitTime = getMaxWaitTime(node);
for (Future future : futures) {
try {
final Object result = future.get(maxWaitTime, TimeUnit.SECONDS);
final boolean safe = (Boolean) result;
if (!safe) {
return false;
}
} catch (Exception e) {
logger.warning("Error while querying cluster's safe state", e);
return false;
}
}
return true;
}
public void clearInternal() {
try {
Operation clearOperation = operationProvider.createClearOperation(name);
clearOperation.setServiceName(SERVICE_NAME);
BinaryOperationFactory factory = new BinaryOperationFactory(clearOperation, getNodeEngine());
Map<Integer, Object> resultMap =
operationService.invokeOnAllPartitions(SERVICE_NAME, factory);
int numberOfAffectedEntries = 0;
for (Object object : resultMap.values()) {
numberOfAffectedEntries += (Integer) object;
}
MemberSelector selector =
MemberSelectors.and(LITE_MEMBER_SELECTOR, NON_LOCAL_MEMBER_SELECTOR);
for (Member member : getNodeEngine().getClusterService().getMembers(selector)) {
operationService.invokeOnTarget(
SERVICE_NAME, new ClearOperation(name), member.getAddress());
}
if (numberOfAffectedEntries > 0) {
publishMapEvent(numberOfAffectedEntries, EntryEventType.CLEAR_ALL);
}
} catch (Throwable t) {
throw rethrow(t);
}
}
private Object handleAuthenticated() {
if (isOwnerConnection()) {
final String uuid = getUuid();
final String localMemberUUID = clientEngine.getLocalMember().getUuid();
principal = new ClientPrincipal(uuid, localMemberUUID);
reAuthLocal();
Collection<Member> members = nodeEngine.getClusterService().getMembers();
for (Member member : members) {
if (!member.localMember()) {
ClientReAuthOperation op = new ClientReAuthOperation(uuid);
op.setCallerUuid(localMemberUUID);
nodeEngine.getOperationService().send(op, member.getAddress());
}
}
}
boolean isNotMember =
clientEngine.getClusterService().getMember(principal.getOwnerUuid()) == null;
if (isNotMember) {
throw new AuthenticationException(
"Invalid owner-uuid: " + principal.getOwnerUuid() + ", it's not member of this cluster!");
}
endpoint.authenticated(principal, credentials, isOwnerConnection());
setConnectionType();
endpointManager.registerEndpoint(endpoint);
clientEngine.bind(endpoint);
final Address thisAddress = clientEngine.getThisAddress();
return encodeAuth(thisAddress, principal.getUuid(), principal.getOwnerUuid());
}
protected void updateCacheListenerConfigOnOtherNodes(
CacheEntryListenerConfiguration<K, V> cacheEntryListenerConfiguration, boolean isRegister) {
final Collection<Member> members = clientContext.getClusterService().getMemberList();
final HazelcastClientInstanceImpl client =
(HazelcastClientInstanceImpl) clientContext.getHazelcastInstance();
final Collection<Future> futures = new ArrayList<Future>();
for (Member member : members) {
try {
final Address address = member.getAddress();
Data configData = toData(cacheEntryListenerConfiguration);
final ClientMessage request =
CacheListenerRegistrationCodec.encodeRequest(
nameWithPrefix, configData, isRegister, address.getHost(), address.getPort());
final ClientInvocation invocation = new ClientInvocation(client, request, address);
final Future future = invocation.invoke();
futures.add(future);
} catch (Exception e) {
ExceptionUtil.sneakyThrow(e);
}
}
// make sure all configs are created
// TODO do we need this ???s
// try {
// FutureUtil.waitWithDeadline(futures,
// CacheProxyUtil.AWAIT_COMPLETION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
// } catch (TimeoutException e) {
// logger.warning(e);
// }
}
public Member startMember(String clusterId) throws ServerException {
LOG.info("Starting a Member on cluster : " + clusterId);
HzCluster hzCluster = clusterMap.get(clusterId);
if (hzCluster != null) {
Config config = hzCluster.getConfig();
LOG.info(config.getNetworkConfig().getJoin().getTcpIpConfig());
HazelcastInstance hzInstance = Hazelcast.newHazelcastInstance(config);
com.hazelcast.core.Member member = hzInstance.getCluster().getLocalMember();
if (hzCluster.addInstance(member.getUuid(), hzInstance)) {
return new Member(
member.getUuid(), member.getAddress().getHost(), member.getAddress().getPort());
}
}
throw new ServerException("Cannot find Cluster with id:" + clusterId);
}
private void invokeMemberRemoveOperation(Address deadAddress) {
for (Member member : getMembers()) {
Address address = member.getAddress();
if (!thisAddress.equals(address) && !address.equals(deadAddress)) {
nodeEngine.getOperationService().send(new MemberRemoveOperation(deadAddress), address);
}
}
}
@Override
public Collection<Address> getTargets() {
Collection<Member> memberList = nodeEngine.getClusterService().getMembers();
Collection<Address> addresses = new HashSet<Address>();
for (Member member : memberList) {
addresses.add(member.getAddress());
}
return addresses;
}
protected void destroyCacheOnAllMembers(String name, String callerUuid) {
final OperationService operationService = nodeEngine.getOperationService();
final Collection<Member> members = nodeEngine.getClusterService().getMembers();
for (Member member : members) {
if (!member.localMember() && !member.getUuid().equals(callerUuid)) {
final CacheDestroyOperation op = new CacheDestroyOperation(name, true);
operationService.invokeOnTarget(AbstractCacheService.SERVICE_NAME, op, member.getAddress());
}
}
}
@Override
public Map<ClientType, Integer> getConnectedClientStats() {
int numberOfCppClients = 0;
int numberOfDotNetClients = 0;
int numberOfJavaClients = 0;
int numberOfOtherClients = 0;
Operation clientInfoOperation = new GetConnectedClientsOperation();
OperationService operationService = node.nodeEngine.getOperationService();
Map<ClientType, Integer> resultMap = new HashMap<ClientType, Integer>();
Map<String, ClientType> clientsMap = new HashMap<String, ClientType>();
for (Member member : node.getClusterService().getMembers()) {
Address target = member.getAddress();
Future<Map<String, ClientType>> future =
operationService.invokeOnTarget(SERVICE_NAME, clientInfoOperation, target);
try {
Map<String, ClientType> endpoints = future.get();
if (endpoints == null) {
continue;
}
// Merge connected clients according to their uuid.
for (Map.Entry<String, ClientType> entry : endpoints.entrySet()) {
clientsMap.put(entry.getKey(), entry.getValue());
}
} catch (Exception e) {
logger.warning("Cannot get client information from: " + target.toString(), e);
}
}
// Now we are regrouping according to the client type
for (ClientType clientType : clientsMap.values()) {
switch (clientType) {
case JAVA:
numberOfJavaClients++;
break;
case CSHARP:
numberOfDotNetClients++;
break;
case CPP:
numberOfCppClients++;
break;
default:
numberOfOtherClients++;
}
}
resultMap.put(ClientType.CPP, numberOfCppClients);
resultMap.put(ClientType.CSHARP, numberOfDotNetClients);
resultMap.put(ClientType.JAVA, numberOfJavaClients);
resultMap.put(ClientType.OTHER, numberOfOtherClients);
return resultMap;
}
@Override
public void memberRemoved(MembershipEvent membershipEvent) {
synchronized (listenerRegLock) {
Member member = membershipEvent.getMember();
members.remove(member);
for (Map<Address, ClientEventRegistration> registrationMap : registrations.values()) {
ClientEventRegistration registration = registrationMap.remove(member.getAddress());
removeEventHandler(registration.getCallId());
}
}
}
public static void encode(Member member, ClientMessage clientMessage) {
AddressCodec.encode(member.getAddress(), clientMessage);
clientMessage.set(member.getUuid());
Map<String, Object> attributes = new HashMap<String, Object>(member.getAttributes());
clientMessage.set(attributes.size());
for (Map.Entry<String, Object> entry : attributes.entrySet()) {
clientMessage.set(entry.getKey());
Object value = entry.getValue();
clientMessage.set(value.toString());
}
}
private void callDisconnectionOperation(ClientEndpointImpl endpoint) {
Collection<Member> memberList = nodeEngine.getClusterService().getMembers();
OperationService operationService = nodeEngine.getOperationService();
ClientDisconnectionOperation op = createClientDisconnectionOperation(endpoint.getUuid());
operationService.runOperationOnCallingThread(op);
for (Member member : memberList) {
if (!member.localMember()) {
op = createClientDisconnectionOperation(endpoint.getUuid());
operationService.send(op, member.getAddress());
}
}
}
public static int calculateDataSize(Member member) {
int dataSize = AddressCodec.calculateDataSize(member.getAddress());
dataSize += ParameterUtil.calculateDataSize(member.getUuid());
dataSize += Bits.INT_SIZE_IN_BYTES;
Map<String, Object> attributes = member.getAttributes();
for (Map.Entry<String, Object> entry : attributes.entrySet()) {
dataSize += ParameterUtil.calculateDataSize(entry.getKey());
Object value = entry.getValue();
// TODO: this is costly to use toString
dataSize += ParameterUtil.calculateDataSize(value.toString());
}
return dataSize;
}
protected Collection<Address> getMemberAddresses() {
Address thisAddress = getNodeEngine().getThisAddress();
Collection<Member> members =
getNodeEngine().getClusterService().getMembers(MemberSelectors.DATA_MEMBER_SELECTOR);
Collection<Address> addresses = new ArrayList<Address>();
for (Member member : members) {
Address address = member.getAddress();
if (address.equals(thisAddress)) {
continue;
}
addresses.add(address);
}
return addresses;
}
private void memberRemoved(Member member) {
members.remove(member);
final Connection connection = connectionManager.getConnection(member.getAddress());
if (connection != null) {
connectionManager.destroyConnection(connection);
}
applyMemberListChanges();
MembershipEvent event =
new MembershipEvent(
client.getCluster(),
member,
ClientInitialMembershipEvent.MEMBER_REMOVED,
Collections.unmodifiableSet(new LinkedHashSet<Member>(members)));
clusterService.fireMembershipEvent(event);
}
@Override
public void init(InitialMembershipEvent event) {
synchronized (listenerRegLock) {
members.addAll(event.getMembers());
for (Member member : members) {
for (ClientRegistrationKey registrationKey : registrations.keySet()) {
try {
invoke(registrationKey, member.getAddress());
} catch (Exception e) {
logger.warning(
"Listener " + registrationKey + " can not added to new member " + member);
}
}
}
}
}
private LiveOperations populate() {
liveOperations.clear();
ClusterService clusterService = nodeEngine.getClusterService();
liveOperations.initMember(thisAddress);
for (Member member : clusterService.getMembers()) {
liveOperations.initMember(member.getAddress());
}
for (LiveOperationsTracker tracker :
serviceManager.getServices(LiveOperationsTracker.class)) {
tracker.populate(liveOperations);
}
return liveOperations;
}
public void removeMapInterceptorInternal(String id) {
NodeEngine nodeEngine = getNodeEngine();
mapServiceContext.removeInterceptor(name, id);
Collection<Member> members = nodeEngine.getClusterService().getMembers();
for (Member member : members) {
try {
if (member.localMember()) {
continue;
}
RemoveInterceptorOperation op = new RemoveInterceptorOperation(name, id);
Future future = operationService.invokeOnTarget(SERVICE_NAME, op, member.getAddress());
future.get();
} catch (Throwable t) {
throw rethrow(t);
}
}
}
@Override
public String registerListener(ListenerMessageCodec codec, EventHandler handler) {
String userRegistrationId = UuidUtil.newUnsecureUuidString();
synchronized (listenerRegLock) {
ClientRegistrationKey registrationKey =
new ClientRegistrationKey(userRegistrationId, handler, codec);
registrations.put(registrationKey, new ConcurrentHashMap<Address, ClientEventRegistration>());
try {
for (Member member : this.members) {
invoke(registrationKey, member.getAddress());
}
} catch (Exception e) {
deregisterListener(userRegistrationId);
throw new HazelcastException("Listener can not be added", e);
}
return userRegistrationId;
}
}
public String addMapInterceptorInternal(MapInterceptor interceptor) {
NodeEngine nodeEngine = getNodeEngine();
if (interceptor instanceof HazelcastInstanceAware) {
((HazelcastInstanceAware) interceptor)
.setHazelcastInstance(nodeEngine.getHazelcastInstance());
}
String id = mapServiceContext.generateInterceptorId(name, interceptor);
Collection<Member> members = nodeEngine.getClusterService().getMembers();
for (Member member : members) {
try {
AddInterceptorOperation op = new AddInterceptorOperation(id, interceptor, name);
Future future = operationService.invokeOnTarget(SERVICE_NAME, op, member.getAddress());
future.get();
} catch (Throwable t) {
throw rethrow(t);
}
}
return id;
}
// For Testing
public Collection<ClientEventRegistration> getActiveRegistrations(String uuid) {
synchronized (listenerRegLock) {
Map<Address, ClientEventRegistration> registrationMap =
registrations.get(new ClientRegistrationKey(uuid));
if (registrationMap == null) {
return Collections.EMPTY_LIST;
}
LinkedList<ClientEventRegistration> activeRegistrations =
new LinkedList<ClientEventRegistration>();
for (ClientEventRegistration registration : registrationMap.values()) {
for (Member member : members) {
if (member.getAddress().equals(registration.getSubscriber())) {
activeRegistrations.add(registration);
}
}
}
return activeRegistrations;
}
}
protected void startClusterMerge(final Address targetAddress) {
ClusterServiceImpl clusterService = node.clusterService;
if (!prepareClusterState(clusterService)) {
return;
}
OperationService operationService = node.nodeEngine.getOperationService();
Collection<Member> memberList = clusterService.getMembers();
for (Member member : memberList) {
if (!member.localMember()) {
Operation op = new MergeClustersOperation(targetAddress);
operationService.invokeOnTarget(ClusterServiceImpl.SERVICE_NAME, op, member.getAddress());
}
}
Operation mergeClustersOperation = new MergeClustersOperation(targetAddress);
mergeClustersOperation
.setNodeEngine(node.nodeEngine)
.setService(clusterService)
.setOperationResponseHandler(createEmptyResponseHandler());
operationService.runOperationOnCallingThread(mergeClustersOperation);
}