MockConnectionManager(Node node) {
this.node = node;
thisConnection = new MockConnection(node.getThisAddress(), node.nodeEngine);
synchronized (joinerLock) {
nodes.put(node.getThisAddress(), node.nodeEngine);
}
}
void doRemoveAddress(Address deadAddress, boolean destroyConnection) {
if (!ensureMemberIsRemovable(deadAddress)) {
return;
}
lock.lock();
try {
if (deadAddress.equals(node.getMasterAddress())) {
assignNewMaster();
}
if (node.isMaster()) {
clusterJoinManager.removeJoin(new MemberInfo(deadAddress));
}
Connection conn = node.connectionManager.getConnection(deadAddress);
if (destroyConnection && conn != null) {
node.connectionManager.destroyConnection(conn);
}
MemberImpl deadMember = getMember(deadAddress);
if (deadMember != null) {
removeMember(deadMember);
logger.info(membersString());
}
} finally {
lock.unlock();
}
}
@Override
public boolean isClusterSafe() {
final Node node = getNode();
final Collection<Member> memberList = node.clusterService.getMembers();
if (memberList == null || memberList.isEmpty() || memberList.size() < 2) {
return true;
}
final Collection<Future> futures = new ArrayList<Future>(memberList.size());
for (Member member : memberList) {
final Address target = member.getAddress();
final Operation operation = new SafeStateCheckOperation();
final InternalCompletableFuture future =
node.getNodeEngine()
.getOperationService()
.invokeOnTarget(InternalPartitionService.SERVICE_NAME, operation, target);
futures.add(future);
}
// todo this max wait is appropriate?
final int maxWaitTime = getMaxWaitTime(node);
for (Future future : futures) {
try {
final Object result = future.get(maxWaitTime, TimeUnit.SECONDS);
final boolean safe = (Boolean) result;
if (!safe) {
return false;
}
} catch (Exception e) {
logger.warning("Error while querying cluster's safe state", e);
return false;
}
}
return true;
}
public void check(final Node hazelcastNode, final String version, final boolean isEnterprise) {
ILogger logger = hazelcastNode.getLogger(PhoneHome.class);
if (!hazelcastNode.getGroupProperties().getBoolean(GroupProperty.VERSION_CHECK_ENABLED)) {
logger.warning(
GroupProperty.VERSION_CHECK_ENABLED.getName()
+ " property is deprecated. Please use "
+ GroupProperty.PHONE_HOME_ENABLED.getName()
+ " instead to disable phone home.");
return;
}
if (!hazelcastNode.getGroupProperties().getBoolean(GroupProperty.PHONE_HOME_ENABLED)) {
return;
}
try {
hazelcastNode
.nodeEngine
.getExecutionService()
.scheduleAtFixedRate(
new Runnable() {
public void run() {
phoneHome(hazelcastNode, version, isEnterprise);
}
},
0,
1,
TimeUnit.DAYS);
} catch (RejectedExecutionException e) {
logger.warning("Could not schedule phone home! Most probably Hazelcast is failed to start.");
}
}
public ClusterServiceImpl(Node node) {
this.node = node;
nodeEngine = node.nodeEngine;
logger = node.getLogger(ClusterService.class.getName());
clusterClock = new ClusterClockImpl(logger);
thisAddress = node.getThisAddress();
thisMember = node.getLocalMember();
clusterStateManager = new ClusterStateManager(node, lock);
clusterJoinManager = new ClusterJoinManager(node, this, lock);
clusterHeartbeatManager = new ClusterHeartbeatManager(node, this);
registerThisMember();
node.connectionManager.addConnectionListener(this);
// MEMBERSHIP_EVENT_EXECUTOR is a single threaded executor to ensure that events are executed in
// correct order.
nodeEngine
.getExecutionService()
.register(MEMBERSHIP_EVENT_EXECUTOR_NAME, 1, Integer.MAX_VALUE, ExecutorType.CACHED);
registerMetrics();
}
private void sendMembershipEvents(
Collection<MemberImpl> currentMembers, Collection<MemberImpl> newMembers) {
Set<Member> eventMembers = new LinkedHashSet<Member>(currentMembers);
if (!newMembers.isEmpty()) {
if (newMembers.size() == 1) {
MemberImpl newMember = newMembers.iterator().next();
// sync call
node.getPartitionService().memberAdded(newMember);
// async events
eventMembers.add(newMember);
sendMembershipEventNotifications(newMember, unmodifiableSet(eventMembers), true);
} else {
for (MemberImpl newMember : newMembers) {
// sync call
node.getPartitionService().memberAdded(newMember);
// async events
eventMembers.add(newMember);
sendMembershipEventNotifications(
newMember, unmodifiableSet(new LinkedHashSet<Member>(eventMembers)), true);
}
}
}
}
public ClientEngineImpl(Node node) {
this.node = node;
this.serializationService = node.getSerializationService();
nodeEngine = node.nodeEngine;
executor = nodeEngine.getExecutionService().getExecutor(ExecutionService.CLIENT_EXECUTOR);
logger = node.getLogger(ClientEngine.class);
}
@Before
public void setup() {
when(node.getPartitionService()).thenReturn(partitionService);
when(node.getClusterService()).thenReturn(clusterService);
when(node.getNodeExtension()).thenReturn(mock(NodeExtension.class));
when(node.getLogger(ClusterStateManager.class)).thenReturn(mock(ILogger.class));
clusterStateManager = new ClusterStateManager(node, lock);
}
private void registerServices(
Map<String, Properties> serviceProps, Map<String, Object> serviceConfigObjects) {
registerCoreServices();
Node node = nodeEngine.getNode();
ServicesConfig servicesConfig = node.getConfig().getServicesConfig();
if (servicesConfig != null) {
registerDefaultServices(servicesConfig);
registerUserServices(servicesConfig, serviceProps, serviceConfigObjects);
}
}
private HazelcastInstance[] findOwnerAndReplicas(HazelcastInstance[] instances, String name) {
Node node = getNode(instances[0]);
InternalPartitionService partitionService = node.getPartitionService();
int partitionId = partitionService.getPartitionId(name);
InternalPartition partition = partitionService.getPartition(partitionId);
HazelcastInstance[] ownerAndReplicas = new HazelcastInstance[instances.length];
for (int i = 0; i < instances.length; i++) {
ownerAndReplicas[i] = getInstanceForAddress(instances, partition.getReplicaAddress(i));
}
return ownerAndReplicas;
}
private void registerCoreServices() {
logger.finest("Registering core services...");
Node node = nodeEngine.getNode();
registerService(ClusterServiceImpl.SERVICE_NAME, node.getClusterService());
registerService(InternalPartitionService.SERVICE_NAME, node.getPartitionService());
registerService(ProxyServiceImpl.SERVICE_NAME, nodeEngine.getProxyService());
registerService(
TransactionManagerServiceImpl.SERVICE_NAME, nodeEngine.getTransactionManagerService());
registerService(ClientEngineImpl.SERVICE_NAME, node.clientEngine);
registerService(QuorumServiceImpl.SERVICE_NAME, nodeEngine.getQuorumService());
}
protected AbstractMessageTask(ClientMessage clientMessage, Node node, Connection connection) {
this.clientMessage = clientMessage;
this.logger = node.getLogger(getClass());
this.node = node;
this.nodeEngine = node.nodeEngine;
this.serializationService = node.getSerializationService();
this.connection = connection;
this.parameters = decodeClientMessage(clientMessage);
this.clientEngine = node.clientEngine;
this.endpointManager = clientEngine.getEndpointManager();
this.endpoint = getEndpoint();
}
@Override
public void connectionRemoved(Connection connection) {
if (logger.isFinestEnabled()) {
logger.finest("Removed connection " + connection.getEndPoint());
}
if (!node.joined()) {
Address masterAddress = node.getMasterAddress();
if (masterAddress != null && masterAddress.equals(connection.getEndPoint())) {
node.setMasterAddress(null);
}
}
}
public Map<String, String> phoneHome(Node hazelcastNode, String version, boolean isEnterprise) {
String downloadId = "source";
InputStream is = null;
try {
is = getClass().getClassLoader().getResourceAsStream("hazelcast-download.properties");
if (is != null) {
final Properties properties = new Properties();
properties.load(is);
downloadId = properties.getProperty("hazelcastDownloadId");
}
} catch (IOException ignored) {
EmptyStatement.ignore(ignored);
} finally {
IOUtil.closeResource(is);
}
// Calculate native memory usage from native memory config
NativeMemoryConfig memoryConfig = hazelcastNode.getConfig().getNativeMemoryConfig();
final ClusterServiceImpl clusterService = hazelcastNode.getClusterService();
long totalNativeMemorySize =
clusterService.getSize(DATA_MEMBER_SELECTOR) * memoryConfig.getSize().bytes();
String nativeMemoryParameter =
(isEnterprise) ? Long.toString(MemoryUnit.BYTES.toGigaBytes(totalNativeMemorySize)) : "0";
// Calculate connected clients to the cluster.
Map<ClientType, Integer> clusterClientStats =
hazelcastNode.clientEngine.getConnectedClientStats();
RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
Long clusterUpTime = clusterService.getClusterClock().getClusterUpTime();
PhoneHomeParameterCreator parameterCreator = new PhoneHomeParameterCreator();
parameterCreator.addParam("version", version);
parameterCreator.addParam("m", hazelcastNode.getLocalMember().getUuid());
parameterCreator.addParam("e", Boolean.toString(isEnterprise));
parameterCreator.addParam("l", MD5Util.toMD5String(hazelcastNode.getConfig().getLicenseKey()));
parameterCreator.addParam("p", downloadId);
parameterCreator.addParam("c", clusterService.getClusterId());
parameterCreator.addParam("crsz", convertToLetter(clusterService.getMembers().size()));
parameterCreator.addParam(
"cssz", convertToLetter(hazelcastNode.clientEngine.getClientEndpointCount()));
parameterCreator.addParam("hdgb", nativeMemoryParameter);
parameterCreator.addParam("ccpp", Integer.toString(clusterClientStats.get(ClientType.CPP)));
parameterCreator.addParam("cdn", Integer.toString(clusterClientStats.get(ClientType.CSHARP)));
parameterCreator.addParam("cjv", Integer.toString(clusterClientStats.get(ClientType.JAVA)));
parameterCreator.addParam("cuptm", Long.toString(clusterUpTime));
parameterCreator.addParam("nuptm", Long.toString(runtimeMxBean.getUptime()));
String urlStr = BASE_PHONE_HOME_URL + parameterCreator.build();
fetchWebService(urlStr);
return parameterCreator.getParameters();
}
public static OperationService getOperationService(HazelcastInstance hz) {
Node node = getNode(hz);
if (node == null) {
throw new NullPointerException("node is null in Hazelcast instance " + hz);
}
NodeEngineImpl nodeEngine = node.getNodeEngine();
try {
return nodeEngine.getOperationService();
} catch (NoSuchMethodError e) {
// fallback for a binary incompatible change (see commit http://git.io/vtfKU)
return getOperationServiceViaReflection(nodeEngine);
}
}
public Connection getConnection(Address address) {
MockConnection conn = mapConnections.get(address);
if (conn == null) {
NodeEngineImpl nodeEngine = nodes.get(address);
MockConnection thisConnection =
new MockConnection(address, node.getThisAddress(), node.nodeEngine);
conn = new MockConnection(node.getThisAddress(), address, nodeEngine);
conn.localConnection = thisConnection;
thisConnection.localConnection = conn;
mapConnections.put(address, conn);
}
return conn;
}
/*
* Sets up a situation where the member list is out of order on node2. Both
* node2 and node1 think they are masters and both think each other are in
* their clusters.
*/
@Test
public void testOutOfSyncMemberListTwoMasters() throws Exception {
Config c1 = buildConfig(false);
Config c2 = buildConfig(false);
Config c3 = buildConfig(false);
c1.getNetworkConfig().setPort(45701);
c2.getNetworkConfig().setPort(45702);
c3.getNetworkConfig().setPort(45703);
List<String> allMembers = Arrays.asList("127.0.0.1:45701, 127.0.0.1:45702, 127.0.0.1:45703");
c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1);
final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2);
final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3);
final MemberImpl m1 = (MemberImpl) h1.getCluster().getLocalMember();
final MemberImpl m2 = (MemberImpl) h2.getCluster().getLocalMember();
final MemberImpl m3 = (MemberImpl) h3.getCluster().getLocalMember();
// All three nodes join into one cluster
assertEquals(3, h1.getCluster().getMembers().size());
assertEquals(3, h2.getCluster().getMembers().size());
assertEquals(3, h3.getCluster().getMembers().size());
final Node n2 = TestUtil.getNode(h2);
// Simulates node2 getting an out of order member list. That causes node2 to think it's the
// master.
List<MemberInfo> members = new ArrayList<MemberInfo>();
members.add(new MemberInfo(m2.getAddress(), m2.getUuid()));
members.add(new MemberInfo(m3.getAddress(), m3.getUuid()));
members.add(new MemberInfo(m1.getAddress(), m1.getUuid()));
n2.clusterService.updateMembers(members);
n2.setMasterAddress(m2.getAddress());
// Give the cluster some time to figure things out. The merge and heartbeat code should have
// kicked in by this point
Thread.sleep(30 * 1000);
assertEquals(m1, h1.getCluster().getMembers().iterator().next());
assertEquals(m1, h2.getCluster().getMembers().iterator().next());
assertEquals(m1, h3.getCluster().getMembers().iterator().next());
assertEquals(3, h1.getCluster().getMembers().size());
assertEquals(3, h2.getCluster().getMembers().size());
assertEquals(3, h3.getCluster().getMembers().size());
}
public ClientEngineImpl(Node node) {
this.logger = node.getLogger(ClientEngine.class);
this.node = node;
this.serializationService = node.getSerializationService();
this.nodeEngine = node.nodeEngine;
this.endpointManager = new ClientEndpointManagerImpl(this, nodeEngine);
this.executor = newExecutor();
this.messageTaskFactory = new CompositeMessageTaskFactory(this.nodeEngine);
this.clientExceptionFactory = initClientExceptionFactory();
ClientHeartbeatMonitor heartBeatMonitor =
new ClientHeartbeatMonitor(
endpointManager, this, nodeEngine.getExecutionService(), node.getProperties());
heartBeatMonitor.start();
}
public void initialClusterState(ClusterState clusterState) {
if (node.joined()) {
throw new IllegalStateException(
"Cannot set initial state after node joined! -> " + clusterState);
}
clusterStateManager.initialClusterState(clusterState);
}
private void shutdownNodes() {
final Operation op = new ShutdownNodeOperation();
logger.info("Sending shutting down operations to all members...");
Collection<Member> members = getMembers(NON_LOCAL_MEMBER_SELECTOR);
final long timeout =
node.groupProperties.getNanos(GroupProperty.CLUSTER_SHUTDOWN_TIMEOUT_SECONDS);
final long startTime = System.nanoTime();
while ((System.nanoTime() - startTime) < timeout && !members.isEmpty()) {
for (Member member : members) {
nodeEngine.getOperationService().send(op, member.getAddress());
}
try {
Thread.sleep(CLUSTER_SHUTDOWN_SLEEP_DURATION_IN_MILLIS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.warning("Shutdown sleep interrupted. ", e);
break;
}
members = getMembers(NON_LOCAL_MEMBER_SELECTOR);
}
logger.info(
"Number of other nodes remaining: "
+ getSize(NON_LOCAL_MEMBER_SELECTOR)
+ ". Shutting down itself.");
node.shutdown(false);
}
@Override
public void shutdown() {
for (Address address : nodes.keySet()) {
if (address.equals(node.getThisAddress())) {
continue;
}
final NodeEngineImpl nodeEngine = nodes.get(address);
if (nodeEngine != null && nodeEngine.isRunning()) {
nodeEngine
.getExecutionService()
.execute(
ExecutionService.SYSTEM_EXECUTOR,
new Runnable() {
public void run() {
ClusterServiceImpl clusterService =
(ClusterServiceImpl) nodeEngine.getClusterService();
clusterService.removeAddress(node.getThisAddress());
}
});
}
}
for (MockConnection connection : mapConnections.values()) {
connection.close();
}
}
private void ensureConnectionToAllMembers() {
boolean allConnected = false;
if (node.joined()) {
logger.finest("Waiting for all connections");
int connectAllWaitSeconds =
node.groupProperties.getSeconds(GroupProperty.CONNECT_ALL_WAIT_SECONDS);
int checkCount = 0;
while (checkCount++ < connectAllWaitSeconds && !allConnected) {
try {
//noinspection BusyWait
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException ignored) {
EmptyStatement.ignore(ignored);
}
allConnected = true;
Collection<Member> members = clusterService.getMembers();
for (Member member : members) {
if (!member.localMember()
&& node.connectionManager.getOrConnect(member.getAddress()) == null) {
allConnected = false;
if (logger.isFinestEnabled()) {
logger.finest("Not-connected to " + member.getAddress());
}
}
}
}
}
}
protected final long getMaxJoinTimeToMasterNode() {
// max join time to found master node,
// this should be significantly greater than MAX_WAIT_SECONDS_BEFORE_JOIN property
// hence we add 10 seconds more
return TimeUnit.SECONDS.toMillis(MIN_WAIT_SECONDS_BEFORE_JOIN)
+ node.getGroupProperties().getMillis(GroupProperty.MAX_WAIT_SECONDS_BEFORE_JOIN);
}
@Override
public void onFailedConnection(final Address address) {
if (!node.joined()) {
node.getJoiner().blacklist(address, false);
} else {
if (node.clusterService.getMember(address) != null) {
nodeEngine
.getExecutionService()
.schedule(
ExecutionService.IO_EXECUTOR,
new ReconnectionTask(address),
getConnectionMonitorInterval(),
TimeUnit.MILLISECONDS);
}
}
}
private int makeBackups(
BackupAwareOperation backupAwareOp,
int partitionId,
long[] replicaVersions,
int syncBackups,
int asyncBackups) {
int sendSyncBackups = 0;
InternalPartitionService partitionService = node.getPartitionService();
InternalPartition partition = partitionService.getPartition(partitionId);
Data backupOpData = getBackupOperationData(backupAwareOp);
for (int replicaIndex = 1; replicaIndex <= syncBackups + asyncBackups; replicaIndex++) {
Address target = partition.getReplicaAddress(replicaIndex);
if (target == null) {
continue;
}
assertNoBackupOnPrimaryMember(partition, target);
boolean isSyncBackup = replicaIndex <= syncBackups;
Backup backup =
newBackup(backupAwareOp, backupOpData, replicaVersions, replicaIndex, isSyncBackup);
operationService.send(backup, target);
if (isSyncBackup) {
sendSyncBackups++;
}
}
return sendSyncBackups;
}
public int backup(BackupAwareOperation backupAwareOp) throws Exception {
int requestedSyncBackups = requestedSyncBackups(backupAwareOp);
int requestedAsyncBackups = requestedAsyncBackups(backupAwareOp);
int requestedTotalBackups = requestedTotalBackups(backupAwareOp);
if (requestedTotalBackups == 0) {
return 0;
}
Operation op = (Operation) backupAwareOp;
InternalPartitionService partitionService = node.getPartitionService();
long[] replicaVersions =
partitionService.incrementPartitionReplicaVersions(
op.getPartitionId(), requestedTotalBackups);
boolean syncForced = backpressureRegulator.isSyncForced(backupAwareOp);
int syncBackups = syncBackups(requestedSyncBackups, requestedAsyncBackups, syncForced);
int asyncBackups = asyncBackups(requestedSyncBackups, requestedAsyncBackups, syncForced);
// TODO: This could cause a problem with back pressure
if (!op.returnsResponse()) {
asyncBackups += syncBackups;
syncBackups = 0;
}
if (syncBackups + asyncBackups == 0) {
return 0;
}
return makeBackups(
backupAwareOp, op.getPartitionId(), replicaVersions, syncBackups, asyncBackups);
}
public void connectionRemoved(Connection connection) {
if (connection.isClient() && connection instanceof TcpIpConnection && nodeEngine.isActive()) {
final ClientEndpoint endpoint = endpoints.get(connection);
if (endpoint != null
&& node.getLocalMember().getUuid().equals(endpoint.getPrincipal().getOwnerUuid())) {
removeEndpoint(connection, true);
if (!endpoint.isFirstConnection()) {
return;
}
NodeEngine nodeEngine = node.nodeEngine;
final Collection<MemberImpl> memberList = nodeEngine.getClusterService().getMemberList();
for (MemberImpl member : memberList) {
final ClientDisconnectionOperation op =
new ClientDisconnectionOperation(endpoint.getUuid());
op.setNodeEngine(nodeEngine)
.setServiceName(SERVICE_NAME)
.setService(this)
.setResponseHandler(ResponseHandlerFactory.createEmptyResponseHandler());
if (member.localMember()) {
nodeEngine.getOperationService().runOperation(op);
} else {
nodeEngine.getOperationService().send(op, member.getAddress());
}
}
}
}
}
@Override
public void init(NodeEngine nodeEngine, Properties properties) {
ClassDefinitionBuilder builder =
new ClassDefinitionBuilder(ClientPortableHook.ID, ClientPortableHook.PRINCIPAL);
builder.addUTFField("uuid").addUTFField("ownerUuid");
serializationService.getSerializationContext().registerClassDefinition(builder.build());
node.getConnectionManager().addConnectionListener(this);
}
@Override
public Connection getOrConnect(Address address) {
MockConnection conn = mapConnections.get(address);
if (conn == null || !conn.isAlive()) {
NodeEngineImpl nodeEngine = nodes.get(address);
if (nodeEngine != null && nodeEngine.isRunning()) {
MockConnection thisConnection =
new MockConnection(address, node.getThisAddress(), node.nodeEngine);
conn = new MockConnection(node.getThisAddress(), address, nodeEngine);
conn.localConnection = thisConnection;
thisConnection.localConnection = conn;
mapConnections.put(address, conn);
logger.info("Created connection to endpoint: " + address + ", connection: " + conn);
}
}
return conn;
}
private int getPartitionOperationThreadCount() {
int threadCount = node.getGroupProperties().PARTITION_OPERATION_THREAD_COUNT.getInteger();
if (threadCount <= 0) {
int coreSize = Runtime.getRuntime().availableProcessors();
threadCount = coreSize * 2;
}
return threadCount;
}