@Override
public void shutdown() {
for (Address address : nodes.keySet()) {
if (address.equals(node.getThisAddress())) {
continue;
}
final NodeEngineImpl nodeEngine = nodes.get(address);
if (nodeEngine != null && nodeEngine.isRunning()) {
nodeEngine
.getExecutionService()
.execute(
ExecutionService.SYSTEM_EXECUTOR,
new Runnable() {
public void run() {
ClusterServiceImpl clusterService =
(ClusterServiceImpl) nodeEngine.getClusterService();
clusterService.removeAddress(node.getThisAddress());
}
});
}
}
for (MockConnection connection : mapConnections.values()) {
connection.close();
}
}
@SuppressWarnings("unchecked")
@Override
public TransactionalObject getTransactionalObject(String serviceName, String name) {
if (transaction.getState() != Transaction.State.ACTIVE) {
throw new TransactionNotActiveException(
"No transaction is found while accessing "
+ "transactional object -> "
+ serviceName
+ "["
+ name
+ "]!");
}
TransactionalObjectKey key = new TransactionalObjectKey(serviceName, name);
TransactionalObject obj = txnObjectMap.get(key);
if (obj != null) {
return obj;
}
final Object service = nodeEngine.getService(serviceName);
if (service instanceof TransactionalService) {
nodeEngine.getProxyService().initializeDistributedObject(serviceName, name);
obj = ((TransactionalService) service).createTransactionalObject(name, transaction);
txnObjectMap.put(key, obj);
} else {
throw new IllegalArgumentException("Service[" + serviceName + "] is not transactional!");
}
return obj;
}
private void sendMemberAttributeEvent(
MemberImpl member, MemberAttributeOperationType operationType, String key, Object value) {
final MemberAttributeServiceEvent event =
new MemberAttributeServiceEvent(this, member, operationType, key, value);
MemberAttributeEvent attributeEvent =
new MemberAttributeEvent(this, member, operationType, key, value);
Collection<MembershipAwareService> membershipAwareServices =
nodeEngine.getServices(MembershipAwareService.class);
if (membershipAwareServices != null && !membershipAwareServices.isEmpty()) {
for (final MembershipAwareService service : membershipAwareServices) {
// service events should not block each other
nodeEngine
.getExecutionService()
.execute(
ExecutionService.SYSTEM_EXECUTOR,
new Runnable() {
public void run() {
service.memberAttributeChanged(event);
}
});
}
}
EventService eventService = nodeEngine.getEventService();
Collection<EventRegistration> registrations =
eventService.getRegistrations(SERVICE_NAME, SERVICE_NAME);
for (EventRegistration reg : registrations) {
eventService.publishEvent(SERVICE_NAME, reg, attributeEvent, reg.getId().hashCode());
}
}
private void sendMembershipEventNotifications(
MemberImpl member, Set<Member> members, final boolean added) {
int eventType = added ? MembershipEvent.MEMBER_ADDED : MembershipEvent.MEMBER_REMOVED;
MembershipEvent membershipEvent = new MembershipEvent(this, member, eventType, members);
Collection<MembershipAwareService> membershipAwareServices =
nodeEngine.getServices(MembershipAwareService.class);
if (membershipAwareServices != null && !membershipAwareServices.isEmpty()) {
final MembershipServiceEvent event = new MembershipServiceEvent(membershipEvent);
for (final MembershipAwareService service : membershipAwareServices) {
nodeEngine
.getExecutionService()
.execute(
MEMBERSHIP_EVENT_EXECUTOR_NAME,
new Runnable() {
public void run() {
if (added) {
service.memberAdded(event);
} else {
service.memberRemoved(event);
}
}
});
}
}
EventService eventService = nodeEngine.getEventService();
Collection<EventRegistration> registrations =
eventService.getRegistrations(SERVICE_NAME, SERVICE_NAME);
for (EventRegistration reg : registrations) {
eventService.publishEvent(SERVICE_NAME, reg, membershipEvent, reg.getId().hashCode());
}
}
void shutdown() {
final Collection<NodeEngineImpl> values = new ArrayList<NodeEngineImpl>(nodes.values());
nodes.clear();
for (NodeEngineImpl value : values) {
value.getHazelcastInstance().getLifecycleService().shutdown();
}
}
@Override
public void run() {
final ClusterServiceImpl clusterService = getService();
final ILogger logger = getLogger();
final ClusterState clusterState = clusterService.getClusterState();
if (clusterState == ClusterState.PASSIVE) {
final NodeEngineImpl nodeEngine = (NodeEngineImpl) getNodeEngine();
if (nodeEngine.isRunning()) {
logger.info(
"Shutting down node in cluster passive state. Requested by: " + getCallerAddress());
new Thread(
new Runnable() {
@Override
public void run() {
final Node node = nodeEngine.getNode();
node.hazelcastInstance.getLifecycleService().shutdown();
}
},
nodeEngine.getHazelcastThreadGroup().getThreadNamePrefix(".clusterShutdown"))
.start();
} else {
logger.info("Node is already shutting down. NodeState: " + nodeEngine.getNode().getState());
}
} else {
logger.severe(
"Can not shut down node because cluster is in "
+ clusterState
+ " state. Requested by: "
+ getCallerAddress());
}
}
void terminate() {
final Collection<NodeEngineImpl> values = new ArrayList<NodeEngineImpl>(nodes.values());
nodes.clear();
for (NodeEngineImpl value : values) {
HazelcastInstance hz = value.getHazelcastInstance();
hz.getLifecycleService().terminate();
}
}
public InvocationPlugin(NodeEngineImpl nodeEngine) {
super(nodeEngine.getLogger(PendingInvocationsPlugin.class));
InternalOperationService operationService = nodeEngine.getOperationService();
this.invocationRegistry = ((OperationServiceImpl) operationService).getInvocationRegistry();
HazelcastProperties props = nodeEngine.getProperties();
this.samplePeriodMillis = props.getMillis(SAMPLE_PERIOD_SECONDS);
this.thresholdMillis = props.getMillis(SLOW_THRESHOLD_SECONDS);
}
Collection<HazelcastInstance> getAllHazelcastInstances() {
Collection<HazelcastInstance> all = new LinkedList<HazelcastInstance>();
for (NodeEngineImpl nodeEngine : nodes.values()) {
if (nodeEngine.isActive()) {
all.add(nodeEngine.getHazelcastInstance());
}
}
return all;
}
public boolean write(SocketWritable socketWritable) {
final Packet packet = (Packet) socketWritable;
if (nodeEngine.getNode().isActive()) {
packet.setConn(thisConnection);
MemberImpl member = nodeEngine.getClusterService().getMember(thisAddress);
if (member != null) {
member.didRead();
}
nodeEngine.handlePacket(packet);
return true;
}
return false;
}
public boolean write(SocketWritable socketWritable) {
final Packet packet = (Packet) socketWritable;
if (nodeEngine.getNode().isActive()) {
Packet newPacket = readFromPacket(packet);
MemberImpl member = nodeEngine.getClusterService().getMember(localEndpoint);
if (member != null) {
member.didRead();
}
nodeEngine.getPacketTransceiver().receive(newPacket);
return true;
}
return false;
}
public static OperationService getOperationService(HazelcastInstance hz) {
Node node = getNode(hz);
if (node == null) {
throw new NullPointerException("node is null in Hazelcast instance " + hz);
}
NodeEngineImpl nodeEngine = node.getNodeEngine();
try {
return nodeEngine.getOperationService();
} catch (NoSuchMethodError e) {
// fallback for a binary incompatible change (see commit http://git.io/vtfKU)
return getOperationServiceViaReflection(nodeEngine);
}
}
@Override
public boolean open(NodeEngine nodeEngine) {
NodeEngineImpl nei = (NodeEngineImpl) nodeEngine;
InternalPartitionService ps = nei.getPartitionService();
MapService mapService = nei.getService(MapService.SERVICE_NAME);
ss = nei.getSerializationService();
Address partitionOwner = ps.getPartitionOwner(partitionId);
if (partitionOwner == null) {
return false;
}
RecordStore recordStore =
mapService.getMapServiceContext().getRecordStore(partitionId, mapName);
iterator = recordStore.entrySetData().iterator();
return true;
}
@Test
public void test_CacheReplicationOperation_serialization() throws Exception {
TestHazelcastInstanceFactory factory = new TestHazelcastInstanceFactory(1);
HazelcastInstance hazelcastInstance = factory.newHazelcastInstance();
try {
CachingProvider provider =
HazelcastServerCachingProvider.createCachingProvider(hazelcastInstance);
CacheManager manager = provider.getCacheManager();
CompleteConfiguration configuration = new MutableConfiguration();
Cache cache1 = manager.createCache("cache1", configuration);
Cache cache2 = manager.createCache("cache2", configuration);
Cache cache3 = manager.createCache("cache3", configuration);
for (int i = 0; i < 1000; i++) {
cache1.put("key" + i, i);
cache2.put("key" + i, i);
cache3.put("key" + i, i);
}
HazelcastInstanceProxy proxy = (HazelcastInstanceProxy) hazelcastInstance;
Field original = HazelcastInstanceProxy.class.getDeclaredField("original");
original.setAccessible(true);
HazelcastInstanceImpl impl = (HazelcastInstanceImpl) original.get(proxy);
NodeEngineImpl nodeEngine = impl.node.nodeEngine;
CacheService cacheService = nodeEngine.getService(CacheService.SERVICE_NAME);
int partitionCount = nodeEngine.getPartitionService().getPartitionCount();
for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
CachePartitionSegment segment = cacheService.getSegment(partitionId);
CacheReplicationOperation operation = new CacheReplicationOperation(segment, 1);
Data serialized = service.toData(operation);
try {
service.toObject(serialized);
} catch (Exception e) {
throw new Exception("Partition: " + partitionId, e);
}
}
} finally {
factory.shutdownAll();
}
}
@Override
public boolean isCallTimedOut(Operation op) {
// Join operations should not be checked for timeout
// because caller is not member of this cluster
// and can have a different clock.
if (!op.returnsResponse() || isJoinOperation(op)) {
return false;
}
long callTimeout = op.getCallTimeout();
long invocationTime = op.getInvocationTime();
long expireTime = invocationTime + callTimeout;
if (expireTime <= 0 || expireTime == Long.MAX_VALUE) {
return false;
}
ClusterClock clusterClock = nodeEngine.getClusterService().getClusterClock();
long now = clusterClock.getClusterTime();
if (expireTime < now) {
return true;
}
return false;
}
void sendClientEvent(ClientEvent event) {
final EventService eventService = nodeEngine.getEventService();
final Collection<EventRegistration> regs =
eventService.getRegistrations(SERVICE_NAME, SERVICE_NAME);
String uuid = event.getUuid();
eventService.publishEvent(SERVICE_NAME, regs, event, uuid.hashCode());
}
private void doShutdown(boolean force) {
long start = Clock.currentTimeMillis();
logger.finest("** we are being asked to shutdown when active = " + String.valueOf(active));
if (!force && isActive()) {
final int maxWaitSeconds = groupProperties.GRACEFUL_SHUTDOWN_MAX_WAIT.getInteger();
if (!partitionService.prepareToSafeShutdown(maxWaitSeconds, TimeUnit.SECONDS)) {
logger.warning(
"Graceful shutdown could not be completed in " + maxWaitSeconds + " seconds!");
}
}
if (isActive()) {
if (!force) {
clusterService.sendShutdownMessage();
}
// set the joined=false first so that
// threads do not process unnecessary
// events, such as remove address
joined.set(false);
setActive(false);
setMasterAddress(null);
try {
Runtime.getRuntime().removeShutdownHook(shutdownHookThread);
} catch (Throwable ignored) {
}
if (managementCenterService != null) {
managementCenterService.shutdown();
}
logger.finest("Shutting down client command service");
clientEngine.shutdown();
logger.finest("Shutting down node engine");
nodeEngine.shutdown();
if (multicastService != null) {
logger.finest("Shutting down multicast service");
multicastService.stop();
}
logger.finest("Shutting down connection manager");
connectionManager.shutdown();
textCommandService.stop();
masterAddress = null;
if (securityContext != null) {
securityContext.destroy();
}
initializer.destroy();
serializationService.destroy();
int numThreads = threadGroup.activeCount();
Thread[] threads = new Thread[numThreads * 2];
numThreads = threadGroup.enumerate(threads, false);
for (int i = 0; i < numThreads; i++) {
Thread thread = threads[i];
if (thread.isAlive()) {
logger.finest("Shutting down thread " + thread.getName());
thread.interrupt();
}
}
failedConnections.clear();
systemLogService.shutdown();
logger.info(
"Hazelcast Shutdown is completed in " + (Clock.currentTimeMillis() - start) + " ms.");
}
}
public ClientEngineImpl(Node node) {
this.node = node;
this.serializationService = node.getSerializationService();
nodeEngine = node.nodeEngine;
executor = nodeEngine.getExecutionService().getExecutor(ExecutionService.CLIENT_EXECUTOR);
logger = node.getLogger(ClientEngine.class);
}
String addClientListener(ClientListener clientListener) {
final EventRegistration registration =
nodeEngine
.getEventService()
.registerLocalListener(SERVICE_NAME, SERVICE_NAME, clientListener);
return registration.getId();
}
public void memberRemoved(MembershipServiceEvent event) {
if (event.getMember().localMember()) {
return;
}
final String uuid = event.getMember().getUuid();
try {
nodeEngine
.getExecutionService()
.schedule(
new Runnable() {
public void run() {
final Iterator<ClientEndpoint> iter = endpoints.values().iterator();
while (iter.hasNext()) {
final ClientEndpoint endpoint = iter.next();
if (uuid.equals(endpoint.getPrincipal().getOwnerUuid())) {
iter.remove();
destroyEndpoint(endpoint, true);
}
}
}
},
10,
TimeUnit.SECONDS);
} catch (RejectedExecutionException e) {
// means node is shutting down...
}
}
public void connectionRemoved(Connection connection) {
if (connection.isClient() && connection instanceof TcpIpConnection && nodeEngine.isActive()) {
final ClientEndpoint endpoint = endpoints.get(connection);
if (endpoint != null
&& node.getLocalMember().getUuid().equals(endpoint.getPrincipal().getOwnerUuid())) {
removeEndpoint(connection, true);
if (!endpoint.isFirstConnection()) {
return;
}
NodeEngine nodeEngine = node.nodeEngine;
final Collection<MemberImpl> memberList = nodeEngine.getClusterService().getMemberList();
for (MemberImpl member : memberList) {
final ClientDisconnectionOperation op =
new ClientDisconnectionOperation(endpoint.getUuid());
op.setNodeEngine(nodeEngine)
.setServiceName(SERVICE_NAME)
.setService(this)
.setResponseHandler(ResponseHandlerFactory.createEmptyResponseHandler());
if (member.localMember()) {
nodeEngine.getOperationService().runOperation(op);
} else {
nodeEngine.getOperationService().send(op, member.getAddress());
}
}
}
}
}
private void initializeListeners(Config config) {
for (final ListenerConfig listenerCfg : config.getListenerConfigs()) {
Object listener = listenerCfg.getImplementation();
if (listener == null) {
try {
listener = ClassLoaderUtil.newInstance(configClassLoader, listenerCfg.getClassName());
} catch (Exception e) {
logger.severe(e);
}
}
if (listener instanceof HazelcastInstanceAware) {
((HazelcastInstanceAware) listener).setHazelcastInstance(hazelcastInstance);
}
if (listener instanceof DistributedObjectListener) {
final ProxyServiceImpl proxyService = (ProxyServiceImpl) nodeEngine.getProxyService();
proxyService.addProxyListener((DistributedObjectListener) listener);
} else if (listener instanceof MembershipListener) {
clusterService.addMembershipListener((MembershipListener) listener);
} else if (listener instanceof MigrationListener) {
partitionService.addMigrationListener((MigrationListener) listener);
} else if (listener instanceof LifecycleListener) {
hazelcastInstance.lifecycleService.addLifecycleListener((LifecycleListener) listener);
} else if (listener != null) {
final String error = "Unknown listener type: " + listener.getClass();
Throwable t = new IllegalArgumentException(error);
logger.warning(error, t);
}
}
}
public ClusterServiceImpl(Node node) {
this.node = node;
nodeEngine = node.nodeEngine;
logger = node.getLogger(ClusterService.class.getName());
clusterClock = new ClusterClockImpl(logger);
thisAddress = node.getThisAddress();
thisMember = node.getLocalMember();
clusterStateManager = new ClusterStateManager(node, lock);
clusterJoinManager = new ClusterJoinManager(node, this, lock);
clusterHeartbeatManager = new ClusterHeartbeatManager(node, this);
registerThisMember();
node.connectionManager.addConnectionListener(this);
// MEMBERSHIP_EVENT_EXECUTOR is a single threaded executor to ensure that events are executed in
// correct order.
nodeEngine
.getExecutionService()
.register(MEMBERSHIP_EVENT_EXECUTOR_NAME, 1, Integer.MAX_VALUE, ExecutorType.CACHED);
registerMetrics();
}
Map<Integer, Object> invokeOnPartitions(
String serviceName, OperationFactory operationFactory, Collection<Integer> partitions)
throws Exception {
return nodeEngine
.getOperationService()
.invokeOnPartitions(serviceName, operationFactory, partitions);
}
@Test
public void testQuorumIgnoresMemberAttributeEvents() throws Exception {
Config config = new Config();
QuorumConfig quorumConfig = new QuorumConfig().setName(randomString()).setEnabled(true);
RecordingQuorumFunction function = new RecordingQuorumFunction();
quorumConfig.setQuorumFunctionImplementation(function);
config.addQuorumConfig(quorumConfig);
HazelcastInstance hazelcastInstance = createHazelcastInstance(config);
NodeEngineImpl nodeEngine = getNodeEngineImpl(hazelcastInstance);
MembershipAwareService service = nodeEngine.getService(QuorumServiceImpl.SERVICE_NAME);
MemberAttributeServiceEvent event = mock(MemberAttributeServiceEvent.class);
service.memberAttributeChanged(event);
assertFalse(function.wasCalled);
}
private void shutdownNodes() {
final Operation op = new ShutdownNodeOperation();
logger.info("Sending shutting down operations to all members...");
Collection<Member> members = getMembers(NON_LOCAL_MEMBER_SELECTOR);
final long timeout =
node.groupProperties.getNanos(GroupProperty.CLUSTER_SHUTDOWN_TIMEOUT_SECONDS);
final long startTime = System.nanoTime();
while ((System.nanoTime() - startTime) < timeout && !members.isEmpty()) {
for (Member member : members) {
nodeEngine.getOperationService().send(op, member.getAddress());
}
try {
Thread.sleep(CLUSTER_SHUTDOWN_SLEEP_DURATION_IN_MILLIS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.warning("Shutdown sleep interrupted. ", e);
break;
}
members = getMembers(NON_LOCAL_MEMBER_SELECTOR);
}
logger.info(
"Number of other nodes remaining: "
+ getSize(NON_LOCAL_MEMBER_SELECTOR)
+ ". Shutting down itself.");
node.shutdown(false);
}
public static int writeBehindQueueSize(HazelcastInstance node, String mapName) {
int size = 0;
final NodeEngineImpl nodeEngine = getNode(node).getNodeEngine();
MapService mapService = nodeEngine.getService(MapService.SERVICE_NAME);
final MapServiceContext mapServiceContext = mapService.getMapServiceContext();
final int partitionCount = nodeEngine.getPartitionService().getPartitionCount();
for (int i = 0; i < partitionCount; i++) {
final RecordStore recordStore = mapServiceContext.getExistingRecordStore(i, mapName);
if (recordStore == null) {
continue;
}
final MapDataStore<Data, Object> mapDataStore = recordStore.getMapDataStore();
size += ((WriteBehindStore) mapDataStore).getWriteBehindQueue().size();
}
return size;
}
@Override
public Connection getOrConnect(Address address) {
MockConnection conn = mapConnections.get(address);
if (conn == null || !conn.isAlive()) {
NodeEngineImpl nodeEngine = nodes.get(address);
if (nodeEngine != null && nodeEngine.isRunning()) {
MockConnection thisConnection =
new MockConnection(address, node.getThisAddress(), node.nodeEngine);
conn = new MockConnection(node.getThisAddress(), address, nodeEngine);
conn.localConnection = thisConnection;
thisConnection.localConnection = conn;
mapConnections.put(address, conn);
logger.info("Created connection to endpoint: " + address + ", connection: " + conn);
}
}
return conn;
}
private void sendClientEvent(ClientEndpoint endpoint) {
if (endpoint.isFirstConnection()) {
final EventService eventService = nodeEngine.getEventService();
final Collection<EventRegistration> regs =
eventService.getRegistrations(SERVICE_NAME, SERVICE_NAME);
eventService.publishEvent(SERVICE_NAME, regs, endpoint, endpoint.getUuid().hashCode());
}
}
private void invokeMemberRemoveOperation(Address deadAddress) {
for (Member member : getMembers()) {
Address address = member.getAddress();
if (!thisAddress.equals(address) && !address.equals(deadAddress)) {
nodeEngine.getOperationService().send(new MemberRemoveOperation(deadAddress), address);
}
}
}