// A node has left the cluster
// synchronize this in case the cluster manager is naughty and calls it concurrently
private synchronized void nodeLeft(String leftNodeID) {
checkQuorum();
if (attainedQuorum) {
// Check for failover
String sclusterInfo = clusterMap.get(leftNodeID);
if (sclusterInfo == null) {
// Clean close - do nothing
} else {
JsonObject clusterInfo = new JsonObject(sclusterInfo);
checkRemoveSubs(leftNodeID, clusterInfo);
checkFailover(leftNodeID, clusterInfo);
}
// We also check for and potentially resume any previous failovers that might have failed
// We can determine this if there any ids in the cluster map which aren't in the node list
List<String> nodes = clusterManager.getNodes();
for (Map.Entry<String, String> entry : clusterMap.entrySet()) {
if (!leftNodeID.equals(entry.getKey()) && !nodes.contains(entry.getKey())) {
JsonObject haInfo = new JsonObject(entry.getValue());
checkRemoveSubs(entry.getKey(), haInfo);
checkFailover(entry.getKey(), haInfo);
}
}
}
}
// Check if there is a quorum for our group
private void checkQuorum() {
if (quorumSize == 0) {
this.attainedQuorum = true;
} else {
List<String> nodes = clusterManager.getNodes();
int count = 0;
for (String node : nodes) {
String json = clusterMap.get(node);
if (json != null) {
JsonObject clusterInfo = new JsonObject(json);
String group = clusterInfo.getString("group");
if (group.equals(this.group)) {
count++;
}
}
}
boolean attained = count >= quorumSize;
if (!attainedQuorum && attained) {
// A quorum has been attained so we can deploy any currently undeployed HA deploymentIDs
log.info(
"A quorum has been obtained. Any deploymentIDs waiting on a quorum will now be deployed");
this.attainedQuorum = true;
} else if (attainedQuorum && !attained) {
// We had a quorum but we lost it - we must undeploy any HA deploymentIDs
log.info(
"There is no longer a quorum. Any HA deploymentIDs will be undeployed until a quorum is re-attained");
this.attainedQuorum = false;
}
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
VertxOptionsImpl that = (VertxOptionsImpl) o;
if (blockedThreadCheckPeriod != that.blockedThreadCheckPeriod) return false;
if (clusterPort != that.clusterPort) return false;
if (clustered != that.clustered) return false;
if (eventLoopPoolSize != that.eventLoopPoolSize) return false;
if (haEnabled != that.haEnabled) return false;
if (internalBlockingPoolSize != that.internalBlockingPoolSize) return false;
if (maxEventLoopExecuteTime != that.maxEventLoopExecuteTime) return false;
if (maxWorkerExecuteTime != that.maxWorkerExecuteTime) return false;
if (proxyOperationTimeout != that.proxyOperationTimeout) return false;
if (quorumSize != that.quorumSize) return false;
if (workerPoolSize != that.workerPoolSize) return false;
if (clusterHost != null ? !clusterHost.equals(that.clusterHost) : that.clusterHost != null)
return false;
if (clusterManager != null
? !clusterManager.equals(that.clusterManager)
: that.clusterManager != null) return false;
if (haGroup != null ? !haGroup.equals(that.haGroup) : that.haGroup != null) return false;
return true;
}
public void stop() {
if (!stopped) {
if (clusterManager.isActive()) {
clusterMap.remove(nodeID);
}
vertx.cancelTimer(quorumTimerID);
stopped = true;
}
}
public void simulateKill() {
if (!stopped) {
killed = true;
clusterManager.leave(
ar -> {
if (ar.failed()) {
log.error("Failed to leave cluster", ar.cause());
}
});
vertx.cancelTimer(quorumTimerID);
stopped = true;
}
}
public HAManager(
VertxInternal vertx,
DeploymentManager deploymentManager,
ClusterManager clusterManager,
int quorumSize,
String group,
boolean enabled) {
this.vertx = vertx;
this.deploymentManager = deploymentManager;
this.clusterManager = clusterManager;
this.quorumSize = enabled ? quorumSize : 0;
this.group = enabled ? group : "__DISABLED__";
this.enabled = enabled;
this.haInfo = new JsonObject();
haInfo.put("verticles", new JsonArray());
haInfo.put("group", this.group);
this.clusterMap = clusterManager.getSyncMap(CLUSTER_MAP_NAME);
this.nodeID = clusterManager.getNodeID();
clusterManager.nodeListener(
new NodeListener() {
@Override
public void nodeAdded(String nodeID) {
HAManager.this.nodeAdded(nodeID);
}
@Override
public void nodeLeft(String leftNodeID) {
HAManager.this.nodeLeft(leftNodeID);
}
});
clusterMap.put(nodeID, haInfo.encode());
quorumTimerID = vertx.setPeriodic(QUORUM_CHECK_PERIOD, tid -> checkHADeployments());
// Call check quorum to compute whether we have an initial quorum
synchronized (this) {
checkQuorum();
}
}
@Override
public int hashCode() {
int result = eventLoopPoolSize;
result = 31 * result + workerPoolSize;
result = 31 * result + internalBlockingPoolSize;
result = 31 * result + (clustered ? 1 : 0);
result = 31 * result + (clusterHost != null ? clusterHost.hashCode() : 0);
result = 31 * result + clusterPort;
result = 31 * result + (int) (blockedThreadCheckPeriod ^ (blockedThreadCheckPeriod >>> 32));
result = 31 * result + (int) (maxEventLoopExecuteTime ^ (maxEventLoopExecuteTime >>> 32));
result = 31 * result + (int) (maxWorkerExecuteTime ^ (maxWorkerExecuteTime >>> 32));
result = 31 * result + (clusterManager != null ? clusterManager.hashCode() : 0);
result = 31 * result + (int) (proxyOperationTimeout ^ (proxyOperationTimeout >>> 32));
result = 31 * result + (haEnabled ? 1 : 0);
result = 31 * result + quorumSize;
result = 31 * result + (haGroup != null ? haGroup.hashCode() : 0);
return result;
}
// Compute the failover node
private String chooseHashedNode(String group, int hashCode) {
List<String> nodes = clusterManager.getNodes();
ArrayList<String> matchingMembers = new ArrayList<>();
for (String node : nodes) {
String sclusterInfo = clusterMap.get(node);
if (sclusterInfo != null) {
JsonObject clusterInfo = new JsonObject(sclusterInfo);
String memberGroup = clusterInfo.getString("group");
if (group == null || group.equals(memberGroup)) {
matchingMembers.add(node);
}
}
}
if (!matchingMembers.isEmpty()) {
// Hashcodes can be -ve so make it positive
long absHash = (long) hashCode + Integer.MAX_VALUE;
long lpos = absHash % matchingMembers.size();
return matchingMembers.get((int) lpos);
} else {
return null;
}
}