@Override
public void run() {
final Address endpoint = getCallerAddress();
if (endpoint == null) {
return;
}
final ClusterServiceImpl clusterService = getService();
final ILogger logger = getLogger();
final MemberImpl member = clusterService.getMember(endpoint);
if (member == null) {
logger.warning(
"MasterConfirmation has been received from "
+ endpoint
+ ", but it is not a member of this cluster!");
OperationService operationService = getNodeEngine().getOperationService();
operationService.send(new MemberRemoveOperation(clusterService.getThisAddress()), endpoint);
} else {
if (clusterService.isMaster()) {
clusterService.getClusterHeartbeatManager().acceptMasterConfirmation(member, timestamp);
} else {
logger.warning(endpoint + " has sent MasterConfirmation, but this node is not master!");
}
}
}
@Override
public void run() {
final ClusterServiceImpl clusterService = getService();
final ILogger logger = getLogger();
final ClusterState clusterState = clusterService.getClusterState();
if (clusterState == ClusterState.PASSIVE) {
final NodeEngineImpl nodeEngine = (NodeEngineImpl) getNodeEngine();
if (nodeEngine.isRunning()) {
logger.info(
"Shutting down node in cluster passive state. Requested by: " + getCallerAddress());
new Thread(
new Runnable() {
@Override
public void run() {
final Node node = nodeEngine.getNode();
node.hazelcastInstance.getLifecycleService().shutdown();
}
},
nodeEngine.getHazelcastThreadGroup().getThreadNamePrefix(".clusterShutdown"))
.start();
} else {
logger.info("Node is already shutting down. NodeState: " + nodeEngine.getNode().getState());
}
} else {
logger.severe(
"Can not shut down node because cluster is in "
+ clusterState
+ " state. Requested by: "
+ getCallerAddress());
}
}
@Override
public void run() {
ClusterServiceImpl clusterService = node.clusterService;
if (clusterService.getMember(endpoint) != null) {
node.connectionManager.getOrConnect(endpoint);
}
}
private boolean prepareClusterState(ClusterServiceImpl clusterService) {
if (!preCheckClusterState(clusterService)) {
return false;
}
long until = Clock.currentTimeMillis() + mergeNextRunDelayMs;
while (clusterService.getClusterState() == ClusterState.ACTIVE) {
try {
clusterService.changeClusterState(ClusterState.FROZEN);
return true;
} catch (Exception e) {
String error = e.getClass().getName() + ": " + e.getMessage();
logger.warning("While freezing cluster state! " + error);
}
if (Clock.currentTimeMillis() >= until) {
logger.warning(
"Could not change cluster state to FROZEN in time. "
+ "Postponing merge process until next attempt.");
return false;
}
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
logger.warning("Interrupted while preparing cluster for merge!");
// restore interrupt flag
Thread.currentThread().interrupt();
return false;
}
}
return false;
}
@Override
public void run() throws Exception {
ClusterServiceImpl service = getService();
ClusterStateManager clusterStateManager = service.getClusterStateManager();
getLogger().info("Changing cluster state state to " + newState + ", Initiator: " + initiator);
clusterStateManager.commitClusterState(newState, initiator, txnId);
}
private void ensureConnectionToAllMembers() {
boolean allConnected = false;
if (node.joined()) {
logger.finest("Waiting for all connections");
int connectAllWaitSeconds =
node.groupProperties.getSeconds(GroupProperty.CONNECT_ALL_WAIT_SECONDS);
int checkCount = 0;
while (checkCount++ < connectAllWaitSeconds && !allConnected) {
try {
//noinspection BusyWait
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException ignored) {
EmptyStatement.ignore(ignored);
}
allConnected = true;
Collection<Member> members = clusterService.getMembers();
for (Member member : members) {
if (!member.localMember()
&& node.connectionManager.getOrConnect(member.getAddress()) == null) {
allConnected = false;
if (logger.isFinestEnabled()) {
logger.finest("Not-connected to " + member.getAddress());
}
}
}
}
}
}
private void postJoin() {
blacklistedAddresses.clear();
if (logger.isFinestEnabled()) {
logger.finest(
"PostJoin master: " + node.getMasterAddress() + ", isMaster: " + node.isMaster());
}
if (node.getState() != NodeState.ACTIVE) {
return;
}
if (tryCount.incrementAndGet() == JOIN_TRY_COUNT) {
logger.warning("Join try count exceed limit, setting this node as master!");
node.setAsMaster();
}
if (node.joined()) {
if (!node.isMaster()) {
ensureConnectionToAllMembers();
}
if (clusterService.getSize() == 1) {
logger.info('\n' + node.clusterService.membersString());
}
}
}
public AbstractJoiner(Node node) {
this.node = node;
this.logger = node.loggingService.getLogger(getClass());
this.config = node.config;
this.clusterService = node.getClusterService();
this.clusterJoinManager = clusterService.getClusterJoinManager();
mergeNextRunDelayMs =
node.groupProperties.getMillis(GroupProperty.MERGE_NEXT_RUN_DELAY_SECONDS);
}
protected void startClusterMerge(final Address targetAddress) {
ClusterServiceImpl clusterService = node.clusterService;
if (!prepareClusterState(clusterService)) {
return;
}
OperationService operationService = node.nodeEngine.getOperationService();
Collection<Member> memberList = clusterService.getMembers();
for (Member member : memberList) {
if (!member.localMember()) {
Operation op = new MergeClustersOperation(targetAddress);
operationService.invokeOnTarget(ClusterServiceImpl.SERVICE_NAME, op, member.getAddress());
}
}
Operation mergeClustersOperation = new MergeClustersOperation(targetAddress);
mergeClustersOperation
.setNodeEngine(node.nodeEngine)
.setService(clusterService)
.setOperationResponseHandler(createEmptyResponseHandler());
operationService.runOperationOnCallingThread(mergeClustersOperation);
}
@SuppressWarnings({
"checkstyle:methodlength",
"checkstyle:returncount",
"checkstyle:npathcomplexity",
"checkstyle:cyclomaticcomplexity"
})
protected boolean shouldMerge(JoinMessage joinMessage) {
if (joinMessage == null) {
return false;
}
try {
boolean validJoinRequest = clusterJoinManager.validateJoinMessage(joinMessage);
if (!validJoinRequest) {
logger.finest(
"Cannot process split brain merge message from "
+ joinMessage.getAddress()
+ ", since join-message could not be validated.");
return false;
}
} catch (Exception e) {
logger.finest(e.getMessage());
return false;
}
try {
if (clusterService.getMember(joinMessage.getAddress()) != null) {
if (logger.isFinestEnabled()) {
logger.finest(
"Should not merge to "
+ joinMessage.getAddress()
+ ", because it is already member of this cluster.");
}
return false;
}
final ClusterState clusterState = clusterService.getClusterState();
if (clusterState != ClusterState.ACTIVE) {
if (logger.isFinestEnabled()) {
logger.finest(
"Should not merge to "
+ joinMessage.getAddress()
+ ", because this cluster is in "
+ clusterState
+ " state.");
}
return false;
}
Collection<Address> targetMemberAddresses = joinMessage.getMemberAddresses();
if (targetMemberAddresses.contains(node.getThisAddress())) {
node.nodeEngine
.getOperationService()
.send(new MemberRemoveOperation(node.getThisAddress()), joinMessage.getAddress());
logger.info(
node.getThisAddress()
+ " CANNOT merge to "
+ joinMessage.getAddress()
+ ", because it thinks this-node as its member.");
return false;
}
Collection<Address> thisMemberAddresses = clusterService.getMemberAddresses();
for (Address address : thisMemberAddresses) {
if (targetMemberAddresses.contains(address)) {
logger.info(
node.getThisAddress()
+ " CANNOT merge to "
+ joinMessage.getAddress()
+ ", because it thinks "
+ address
+ " as its member. "
+ "But "
+ address
+ " is member of this cluster.");
return false;
}
}
int targetDataMemberCount = joinMessage.getDataMemberCount();
int currentDataMemberCount = clusterService.getSize(DATA_MEMBER_SELECTOR);
if (targetDataMemberCount > currentDataMemberCount) {
// I should join the other cluster
logger.info(
node.getThisAddress()
+ " is merging to "
+ joinMessage.getAddress()
+ ", because : joinMessage.getDataMemberCount() > currentDataMemberCount ["
+ (targetDataMemberCount + " > " + currentDataMemberCount)
+ ']');
if (logger.isFinestEnabled()) {
logger.finest(joinMessage.toString());
}
return true;
} else if (targetDataMemberCount == currentDataMemberCount) {
// compare the hashes
if (node.getThisAddress().hashCode() > joinMessage.getAddress().hashCode()) {
logger.info(
node.getThisAddress()
+ " is merging to "
+ joinMessage.getAddress()
+ ", because : node.getThisAddress().hashCode() > joinMessage.address.hashCode() "
+ ", this node member count: "
+ currentDataMemberCount);
if (logger.isFinestEnabled()) {
logger.finest(joinMessage.toString());
}
return true;
} else {
logger.info(
joinMessage.getAddress()
+ " should merge to this node "
+ ", because : node.getThisAddress().hashCode() < joinMessage.address.hashCode() "
+ ", this node data member count: "
+ currentDataMemberCount);
}
} else {
logger.info(
joinMessage.getAddress()
+ " should merge to this node "
+ ", because : currentDataMemberCount > joinMessage.getDataMemberCount() ["
+ (currentDataMemberCount + " > " + targetDataMemberCount)
+ ']');
}
} catch (Throwable e) {
logger.severe(e);
}
return false;
}
