@Override
public void destroyConnection(final Connection connection) {
if (connection == null) {
return;
}
if (logger.isFinestEnabled()) {
logger.finest("Destroying " + connection);
}
if (activeConnections.remove(connection)) {
// this should not be needed; but some tests are using DroppingConnection which is not a
// TcpIpConnection.
if (connection instanceof TcpIpConnection) {
ioThreadingModel.onConnectionRemoved((TcpIpConnection) connection);
}
}
final Address endPoint = connection.getEndPoint();
if (endPoint != null) {
connectionsInProgress.remove(endPoint);
connectionsMap.remove(endPoint, connection);
fireConnectionRemovedEvent(connection, endPoint);
}
if (connection.isAlive()) {
connection.close();
closedCount.inc();
}
}
protected void sendClientMessage(ClientMessage resultClientMessage) {
resultClientMessage.setCorrelationId(clientMessage.getCorrelationId());
resultClientMessage.addFlag(ClientMessage.BEGIN_AND_END_FLAGS);
resultClientMessage.setVersion(ClientMessage.VERSION);
final Connection connection = endpoint.getConnection();
// TODO framing not implemented yet, should be split into frames before writing to connection
connection.write(resultClientMessage);
}
protected final boolean isValid() {
final ClusterServiceImpl clusterService = getService();
final Connection conn = getConnection();
final Address masterAddress = conn != null ? conn.getEndPoint() : null;
return conn == null
|| // which means this is a local call.
(masterAddress != null && masterAddress.equals(clusterService.getMasterAddress()));
}
public void sendResponse(ClientEndpoint endpoint, Object response) {
if (response instanceof Throwable) {
response =
ClientExceptionConverters.get(endpoint.getClientType()).convert((Throwable) response);
}
final Data resultData = response != null ? serializationService.toData(response) : NULL;
Connection conn = endpoint.getConnection();
conn.write(new DataAdapter(resultData, serializationService.getSerializationContext()));
}
void bind(final ClientEndpoint endpoint) {
final Connection conn = endpoint.getConnection();
if (conn instanceof TcpIpConnection) {
Address address = new Address(conn.getRemoteSocketAddress());
TcpIpConnectionManager connectionManager =
(TcpIpConnectionManager) node.getConnectionManager();
connectionManager.bind((TcpIpConnection) conn, address, null, false);
}
sendClientEvent(endpoint);
}
public void connectionRemoved(Connection connection) {
logger.log(Level.FINEST, "Connection is removed " + connection.getEndPoint());
if (!node.joined()) {
if (getMasterAddress() != null) {
if (getMasterAddress().equals(connection.getEndPoint())) {
node.setMasterAddress(null);
}
}
}
}
@Override
public void connectionRemoved(Connection connection) {
if (logger.isFinestEnabled()) {
logger.finest("Removed connection " + connection.getEndPoint());
}
if (!node.joined()) {
Address masterAddress = node.getMasterAddress();
if (masterAddress != null && masterAddress.equals(connection.getEndPoint())) {
node.setMasterAddress(null);
}
}
}
private boolean authenticate(SecurityContext securityContext) {
Connection connection = endpoint.getConnection();
credentials.setEndpoint(connection.getInetAddress().getHostAddress());
try {
LoginContext lc = securityContext.createClientLoginContext(credentials);
lc.login();
endpoint.setLoginContext(lc);
return true;
} catch (LoginException e) {
logger.warning(e);
return false;
}
}
ClientEndpoint createEndpoint(Connection conn) {
if (!conn.live()) {
LOGGER.severe("Can't create and endpoint for a dead connection");
return null;
}
String clientUuid = UuidUtil.createClientUuid(conn.getEndPoint());
ClientEndpoint endpoint = new ClientEndpoint(clientEngine, conn, clientUuid);
if (endpoints.putIfAbsent(conn, endpoint) != null) {
LOGGER.severe("An endpoint already exists for connection:" + conn);
}
return endpoint;
}
public void bind(final ClientEndpoint endpoint) {
final Connection conn = endpoint.getConnection();
if (conn instanceof TcpIpConnection) {
Address address = new Address(conn.getRemoteSocketAddress());
((TcpIpConnection) conn).setEndPoint(address);
}
ClientEvent event =
new ClientEvent(
endpoint.getUuid(),
ClientEventType.CONNECTED,
endpoint.getSocketAddress(),
endpoint.getClientType());
sendClientEvent(event);
}
@Test
public void cleanup_whenConnectionIsNotAlive() {
BasicBackPressureService service = newEnabledBackPressureService();
GenericOperation op = new GenericOperation();
Connection connection = mock(Connection.class);
when(connection.isAlive()).thenReturn(false);
OperationAccessor.setConnection(op, connection);
service.isBackPressureNeeded(op);
service.cleanup();
assertNull(service.getSyncDelays(connection));
}
public void connectionRemoved(Connection connection) {
if (connection.isClient() && connection instanceof TcpIpConnection && nodeEngine.isActive()) {
final ClientEndpoint endpoint = endpoints.get(connection);
if (endpoint != null
&& node.getLocalMember().getUuid().equals(endpoint.getPrincipal().getOwnerUuid())) {
removeEndpoint(connection, true);
if (!endpoint.isFirstConnection()) {
return;
}
NodeEngine nodeEngine = node.nodeEngine;
final Collection<MemberImpl> memberList = nodeEngine.getClusterService().getMemberList();
for (MemberImpl member : memberList) {
final ClientDisconnectionOperation op =
new ClientDisconnectionOperation(endpoint.getUuid());
op.setNodeEngine(nodeEngine)
.setServiceName(SERVICE_NAME)
.setService(this)
.setResponseHandler(ResponseHandlerFactory.createEmptyResponseHandler());
if (member.localMember()) {
nodeEngine.getOperationService().runOperation(op);
} else {
nodeEngine.getOperationService().send(op, member.getAddress());
}
}
}
}
}
public void destroyConnection(final Connection connection) {
final Address endPoint = connection.getEndPoint();
final boolean removed = mapConnections.remove(endPoint, connection);
if (!removed) {
return;
}
logger.info("Removed connection to endpoint: " + endPoint + ", connection: " + connection);
ioService
.getEventService()
.executeEventCallback(
new StripedRunnable() {
@Override
public void run() {
for (ConnectionListener listener : connectionListeners) {
listener.connectionRemoved(connection);
}
}
@Override
public int getKey() {
return endPoint.hashCode();
}
});
}
@Override
public void removeEndpoint(final ClientEndpoint ce, boolean closeImmediately) {
checkNotNull(ce, "endpoint can't be null");
ClientEndpointImpl endpoint = (ClientEndpointImpl) ce;
endpoints.remove(endpoint.getConnection());
logger.info("Destroying " + endpoint);
try {
endpoint.destroy();
} catch (LoginException e) {
logger.warning(e);
}
final Connection connection = endpoint.getConnection();
if (closeImmediately) {
try {
connection.close();
} catch (Throwable e) {
logger.warning("While closing client connection: " + connection, e);
}
} else {
nodeEngine
.getExecutionService()
.schedule(
new Runnable() {
public void run() {
if (connection.isAlive()) {
try {
connection.close();
} catch (Throwable e) {
logger.warning("While closing client connection: " + e.toString());
}
}
}
},
DESTROY_ENDPOINT_DELAY_MS,
TimeUnit.MILLISECONDS);
}
ClientEvent event =
new ClientEvent(
endpoint.getUuid(),
ClientEventType.DISCONNECTED,
endpoint.getSocketAddress(),
endpoint.getClientType());
clientEngine.sendClientEvent(event);
}
public void shutdown() {
for (ClientEndpoint endpoint : endpoints.values()) {
try {
endpoint.destroy();
} catch (LoginException e) {
logger.finest(e.getMessage());
}
try {
final Connection conn = endpoint.getConnection();
if (conn.live()) {
conn.close();
}
} catch (Exception e) {
logger.finest(e);
}
}
endpoints.clear();
}
private void handleMissingEndpoint() {
if (connection.isAlive()) {
logger.severe("Dropping: " + parameters + " -> no endpoint found for live connection.");
} else {
if (logger.isFinestEnabled()) {
logger.finest("Dropping: " + parameters + " -> no endpoint found for dead connection.");
}
}
}
private boolean checkAlreadyConnected(TcpIpConnection connection, Address remoteEndPoint) {
final Connection existingConnection = connectionsMap.get(remoteEndPoint);
if (existingConnection != null && existingConnection.isAlive()) {
if (existingConnection != connection) {
if (logger.isFinestEnabled()) {
logger.finest(
existingConnection
+ " is already bound to "
+ remoteEndPoint
+ ", new one is "
+ connection);
}
activeConnections.add(connection);
}
return true;
}
return false;
}
@Override
public boolean transmit(Packet packet, Connection connection) {
checkNotNull(packet, "Packet can't be null");
if (connection == null) {
return false;
}
return connection.write(packet);
}
private boolean send(Packet packet, Address target, SendTask sendTask) {
Connection connection = getConnection(target);
if (connection != null) {
return connection.write(packet);
}
if (sendTask == null) {
sendTask = new SendTask(packet, target);
}
int retries = sendTask.retries;
if (retries < RETRY_NUMBER && ioService.isActive()) {
getOrConnect(target, true);
// TODO: Caution: may break the order guarantee of the packets sent from the same thread!
scheduler.schedule(sendTask, (retries + 1) * DELAY_FACTOR, TimeUnit.MILLISECONDS);
return true;
}
return false;
}
@Override
public void shutdown(boolean terminate) {
for (ClientEndpoint ce : endpointManager.getEndpoints()) {
ClientEndpointImpl endpoint = (ClientEndpointImpl) ce;
try {
endpoint.destroy();
} catch (LoginException e) {
logger.finest(e.getMessage());
}
try {
final Connection conn = endpoint.getConnection();
if (conn.isAlive()) {
conn.close("Shutdown of ClientEngine", null);
}
} catch (Exception e) {
logger.finest(e);
}
}
endpointManager.clear();
ownershipMappings.clear();
}
private void destroyEndpoint(ClientEndpoint endpoint, boolean closeImmediately) {
if (endpoint != null) {
logger.info("Destroying " + endpoint);
try {
endpoint.destroy();
} catch (LoginException e) {
logger.warning(e);
}
final Connection connection = endpoint.getConnection();
if (closeImmediately) {
try {
connection.close();
} catch (Throwable e) {
logger.warning("While closing client connection: " + connection, e);
}
} else {
nodeEngine
.getExecutionService()
.schedule(
new Runnable() {
public void run() {
if (connection.live()) {
try {
connection.close();
} catch (Throwable e) {
logger.warning("While closing client connection: " + e.toString());
}
}
}
},
1111,
TimeUnit.MILLISECONDS);
}
sendClientEvent(endpoint);
}
}
public void handleMaster(Master master) {
if (!node.joined() && !thisAddress.equals(master.address)) {
logger.log(Level.FINEST, "Handling master response: " + master);
final Address currentMaster = node.getMasterAddress();
if (currentMaster != null && !currentMaster.equals(master.address)) {
final Connection conn = node.connectionManager.getConnection(currentMaster);
if (conn != null && conn.live()) {
logger.log(
Level.FINEST,
"Ignoring master response "
+ master
+ " since node has an active master: "
+ currentMaster);
return;
}
}
node.setMasterAddress(master.address);
node.connectionManager.getOrConnect(master.address);
if (!sendJoinRequest(master.address, true)) {
logger.log(
Level.WARNING, "Could not create connection to possible master " + master.address);
}
}
}
void removeEndpoint(final ClientEndpoint endpoint, boolean closeImmediately) {
endpoints.remove(endpoint.getConnection());
LOGGER.info("Destroying " + endpoint);
try {
endpoint.destroy();
} catch (LoginException e) {
LOGGER.warning(e);
}
final Connection connection = endpoint.getConnection();
if (closeImmediately) {
try {
connection.close();
} catch (Throwable e) {
LOGGER.warning("While closing client connection: " + connection, e);
}
} else {
nodeEngine
.getExecutionService()
.schedule(
new Runnable() {
public void run() {
if (connection.live()) {
try {
connection.close();
} catch (Throwable e) {
LOGGER.warning("While closing client connection: " + e.toString());
}
}
}
},
DESTROY_ENDPOINT_DELAY_MS,
TimeUnit.MILLISECONDS);
}
clientEngine.sendClientEvent(endpoint);
}
@Override
public void connectionRemoved(Connection connection) {
if (connection.isClient() && nodeEngine.isRunning()) {
ClientEndpointImpl endpoint = (ClientEndpointImpl) endpointManager.getEndpoint(connection);
if (endpoint == null) {
return;
}
if (!endpoint.isFirstConnection()) {
return;
}
String localMemberUuid = node.getLocalMember().getUuid();
String ownerUuid = endpoint.getPrincipal().getOwnerUuid();
if (localMemberUuid.equals(ownerUuid)) {
callDisconnectionOperation(endpoint);
}
}
}
@Override
public void heartBeatStopped(Connection connection) {
ClientMessage request = ClientRemoveAllListenersCodec.encodeRequest();
ClientInvocation removeListenerInvocation = new ClientInvocation(client, request, connection);
removeListenerInvocation.setBypassHeartbeatCheck(true);
removeListenerInvocation.invoke();
final Address remoteEndpoint = connection.getEndPoint();
final Iterator<ClientListenerInvocation> iterator = eventHandlerMap.values().iterator();
final TargetDisconnectedException response = new TargetDisconnectedException(remoteEndpoint);
while (iterator.hasNext()) {
final ClientInvocation clientInvocation = iterator.next();
if (clientInvocation.getSendConnection().equals(connection)) {
iterator.remove();
clientInvocation.notifyException(response);
}
}
}
@Override
public boolean registerConnection(final Address remoteEndPoint, final Connection connection) {
if (remoteEndPoint.equals(ioService.getThisAddress())) {
return false;
}
if (connection instanceof TcpIpConnection) {
TcpIpConnection tcpConnection = (TcpIpConnection) connection;
Address currentEndPoint = tcpConnection.getEndPoint();
if (currentEndPoint != null && !currentEndPoint.equals(remoteEndPoint)) {
throw new IllegalArgumentException(
connection + " has already a different endpoint than: " + remoteEndPoint);
}
tcpConnection.setEndPoint(remoteEndPoint);
if (!connection.isClient()) {
TcpIpConnectionMonitor connectionMonitor = getConnectionMonitor(remoteEndPoint, true);
tcpConnection.setMonitor(connectionMonitor);
}
}
connectionsMap.put(remoteEndPoint, connection);
connectionsInProgress.remove(remoteEndPoint);
ioService
.getEventService()
.executeEventCallback(
new StripedRunnable() {
@Override
public void run() {
for (ConnectionListener listener : connectionListeners) {
listener.connectionAdded(connection);
}
}
@Override
public int getKey() {
return remoteEndPoint.hashCode();
}
});
return true;
}
public ClientEndpoint createNew(Connection conn) {
return new ClientEndpoint(
ClientEngineImpl.this, conn, UuidUtil.createClientUuid(conn.getEndPoint()));
}
@Override
public boolean transmit(Packet packet, Connection connection) {
return (connection != null && connection.write(packet));
}
public final void heartBeater() {
if (!node.joined() || !node.isActive()) return;
long now = Clock.currentTimeMillis();
if (isMaster()) {
List<Address> lsDeadAddresses = null;
for (MemberImpl memberImpl : lsMembers) {
final Address address = memberImpl.getAddress();
if (!thisAddress.equals(address)) {
try {
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null && conn.live()) {
if ((now - memberImpl.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
conn = null;
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because of timeout since last read");
lsDeadAddresses.add(address);
} else if ((now - memberImpl.getLastRead()) >= 5000
&& (now - memberImpl.getLastPing()) >= 5000) {
ping(memberImpl);
}
if ((now - memberImpl.getLastWrite()) > 500) {
sendHeartbeat(conn);
}
Long lastConfirmation = memberMasterConfirmationTimes.get(memberImpl);
if (lastConfirmation == null
|| (now - lastConfirmation > MAX_NO_MASTER_CONFIRMATION_MILLIS)) {
if (lsDeadAddresses == null) {
lsDeadAddresses = new ArrayList<Address>();
}
logger.log(
Level.WARNING,
"Added "
+ address
+ " to list of dead addresses because it has not sent a master confirmation recently");
lsDeadAddresses.add(address);
}
} else if (conn == null && (now - memberImpl.getLastRead()) > 5000) {
logMissingConnection(address);
memberImpl.didRead();
}
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);
}
}
}
if (lsDeadAddresses != null) {
for (Address address : lsDeadAddresses) {
logger.log(Level.FINEST, "No heartbeat should remove " + address);
doRemoveAddress(address);
}
}
} else {
// send heartbeat to master
Address masterAddress = getMasterAddress();
if (masterAddress != null) {
final Connection connMaster = node.connectionManager.getOrConnect(masterAddress);
MemberImpl masterMember = getMember(masterAddress);
boolean removed = false;
if (masterMember != null) {
if ((now - masterMember.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
logger.log(
Level.WARNING, "Master node has timed out its heartbeat and will be removed");
doRemoveAddress(masterAddress);
removed = true;
} else if ((now - masterMember.getLastRead()) >= 5000
&& (now - masterMember.getLastPing()) >= 5000) {
ping(masterMember);
}
}
if (!removed) {
sendHeartbeat(connMaster);
}
}
for (MemberImpl member : lsMembers) {
if (!member.localMember()) {
Address address = member.getAddress();
Connection conn = node.connectionManager.getOrConnect(address);
if (conn != null) {
sendHeartbeat(conn);
} else {
logger.log(Level.FINEST, "Could not connect to " + address + " to send heartbeat");
}
}
}
}
}
void handleJoinRequest(JoinRequest joinRequest) {
final long now = Clock.currentTimeMillis();
String msg =
"Handling join from "
+ joinRequest.address
+ ", inProgress: "
+ joinInProgress
+ (timeToStartJoin > 0 ? ", timeToStart: " + (timeToStartJoin - now) : "");
logger.log(Level.FINEST, msg);
boolean validJoinRequest;
try {
validJoinRequest = node.validateJoinRequest(joinRequest);
} catch (Exception e) {
validJoinRequest = false;
}
final Connection conn = joinRequest.getConnection();
if (validJoinRequest) {
final MemberImpl member = getMember(joinRequest.address);
if (member != null) {
if (joinRequest.getUuid().equals(member.getUuid())) {
String message = "Ignoring join request, member already exists.. => " + joinRequest;
logger.log(Level.FINEST, message);
// send members update back to node trying to join again...
final long clusterTime = node.getClusterImpl().getClusterTime();
sendProcessableTo(new MembersUpdateCall(lsMembers, clusterTime), conn);
sendProcessableTo(new SyncProcess(), conn);
return;
}
// If this node is master then remove old member and process join request.
// If requesting address is equal to master node's address, that means master node
// somehow disconnected and wants to join back.
// So drop old member and process join request if this node becomes master.
if (isMaster() || member.getAddress().equals(getMasterAddress())) {
logger.log(
Level.WARNING,
"New join request has been received from an existing endpoint! => "
+ member
+ " Removing old member and processing join request...");
// If existing connection of endpoint is different from current connection
// destroy it, otherwise keep it.
// final Connection existingConnection =
// node.connectionManager.getConnection(joinRequest.address);
// final boolean destroyExistingConnection = existingConnection !=
// conn;
doRemoveAddress(member.getAddress(), false);
}
}
if (!node.getConfig().getNetworkConfig().getJoin().getMulticastConfig().isEnabled()) {
if (node.isActive() && node.joined() && node.getMasterAddress() != null && !isMaster()) {
sendProcessableTo(new Master(node.getMasterAddress()), conn);
}
}
if (isMaster() && node.joined() && node.isActive()) {
final MemberInfo newMemberInfo =
new MemberInfo(joinRequest.address, joinRequest.nodeType, joinRequest.getUuid());
if (node.securityContext != null && !setJoins.contains(newMemberInfo)) {
final ILogger securityLogger = node.loggingService.getLogger("com.hazelcast.security");
final Credentials cr = joinRequest.getCredentials();
if (cr == null) {
securityLogger.log(
Level.SEVERE,
"Expecting security credentials "
+ "but credentials could not be found in JoinRequest!");
sendAuthFail(conn);
return;
} else {
try {
LoginContext lc = node.securityContext.createMemberLoginContext(cr);
lc.login();
} catch (LoginException e) {
securityLogger.log(
Level.SEVERE,
"Authentication has failed for "
+ cr.getPrincipal()
+ '@'
+ cr.getEndpoint()
+ " => ("
+ e.getMessage()
+ ")");
securityLogger.log(Level.FINEST, e.getMessage(), e);
sendAuthFail(conn);
return;
}
}
}
if (joinRequest.to != null && !joinRequest.to.equals(thisAddress)) {
sendProcessableTo(new Master(node.getMasterAddress()), conn);
return;
}
if (!joinInProgress) {
if (firstJoinRequest != 0
&& now - firstJoinRequest >= MAX_WAIT_SECONDS_BEFORE_JOIN * 1000) {
startJoin();
} else {
if (setJoins.add(newMemberInfo)) {
sendProcessableTo(new Master(node.getMasterAddress()), conn);
if (firstJoinRequest == 0) {
firstJoinRequest = now;
}
if (now - firstJoinRequest < MAX_WAIT_SECONDS_BEFORE_JOIN * 1000) {
timeToStartJoin = now + WAIT_MILLIS_BEFORE_JOIN;
}
}
if (now > timeToStartJoin) {
startJoin();
}
}
}
}
} else {
conn.close();
}
}