private synchronized void checkQuorumWhenAdded(final String nodeID, final long start) {
if (clusterMap.containsKey(nodeID)) {
checkQuorum();
} else {
vertx.setTimer(
200,
tid -> {
// This can block on a monitor so it needs to run as a worker
vertx.executeBlockingInternal(
() -> {
if (System.currentTimeMillis() - start > 10000) {
log.warn("Timed out waiting for group information to appear");
} else if (!stopped) {
ContextImpl context = vertx.getContext();
try {
// Remove any context we have here (from the timer) otherwise will screw
// things up when verticles are deployed
ContextImpl.setContext(null);
checkQuorumWhenAdded(nodeID, start);
} finally {
ContextImpl.setContext(context);
}
}
return null;
},
null);
});
}
}
public void stop() {
if (!stopped) {
if (clusterManager.isActive()) {
clusterMap.remove(nodeID);
}
vertx.cancelTimer(quorumTimerID);
stopped = true;
}
}
public void simulateKill() {
if (!stopped) {
killed = true;
clusterManager.leave(
ar -> {
if (ar.failed()) {
log.error("Failed to leave cluster", ar.cause());
}
});
vertx.cancelTimer(quorumTimerID);
stopped = true;
}
}
// Add the deployment to an internal list of deploymentIDs - these will be executed when a quorum
// is attained
private void addToHADeployList(
final String verticleName,
final DeploymentOptions deploymentOptions,
final Handler<AsyncResult<String>> doneHandler) {
toDeployOnQuorum.add(
() -> {
ContextImpl ctx = vertx.getContext();
try {
ContextImpl.setContext(null);
deployVerticle(verticleName, deploymentOptions, doneHandler);
} finally {
ContextImpl.setContext(ctx);
}
});
}
private void callFailoverCompleteHandler(
FailoverCompleteHandler handler, String nodeID, JsonObject haInfo, boolean result) {
if (handler != null) {
CountDownLatch latch = new CountDownLatch(1);
// The testsuite requires that this is called on a Vert.x thread
vertx.runOnContext(
v -> {
handler.handle(nodeID, haInfo, result);
latch.countDown();
});
try {
latch.await(30, TimeUnit.SECONDS);
} catch (InterruptedException ignore) {
}
}
}
// Process the failover of a deployment
private void processFailover(JsonObject failedVerticle) {
if (failDuringFailover) {
throw new VertxException("Oops!");
}
// This method must block until the failover is complete - i.e. the verticle is successfully
// redeployed
final String verticleName = failedVerticle.getString("verticle_name");
final CountDownLatch latch = new CountDownLatch(1);
final AtomicReference<Throwable> err = new AtomicReference<>();
// Now deploy this verticle on this node
ContextImpl ctx = vertx.getContext();
if (ctx != null) {
// We could be on main thread in which case we don't want to overwrite tccl
ContextImpl.setContext(null);
}
JsonObject options = failedVerticle.getJsonObject("options");
try {
doDeployVerticle(
verticleName,
new DeploymentOptions(options),
result -> {
if (result.succeeded()) {
log.info("Successfully redeployed verticle " + verticleName + " after failover");
} else {
log.error("Failed to redeploy verticle after failover", result.cause());
err.set(result.cause());
}
latch.countDown();
Throwable t = err.get();
if (t != null) {
throw new VertxException(t);
}
});
} finally {
if (ctx != null) {
ContextImpl.setContext(ctx);
}
}
try {
if (!latch.await(120, TimeUnit.SECONDS)) {
throw new VertxException("Timed out waiting for redeploy on failover");
}
} catch (InterruptedException e) {
throw new IllegalStateException(e);
}
}
// Undeploy any HA deploymentIDs now there is no quorum
private void undeployHADeployments() {
for (String deploymentID : deploymentManager.deployments()) {
Deployment dep = deploymentManager.getDeployment(deploymentID);
if (dep != null) {
if (dep.deploymentOptions().isHa()) {
ContextImpl ctx = vertx.getContext();
try {
ContextImpl.setContext(null);
deploymentManager.undeployVerticle(
deploymentID,
result -> {
if (result.succeeded()) {
log.info(
"Successfully undeployed HA deployment "
+ deploymentID
+ "-"
+ dep.verticleIdentifier()
+ " as there is no quorum");
addToHADeployList(
dep.verticleIdentifier(),
dep.deploymentOptions(),
result1 -> {
if (result1.succeeded()) {
log.info(
"Successfully redeployed verticle "
+ dep.verticleIdentifier()
+ " after quorum was re-attained");
} else {
log.error(
"Failed to redeploy verticle "
+ dep.verticleIdentifier()
+ " after quorum was re-attained",
result1.cause());
}
});
} else {
log.error("Failed to undeploy deployment on lost quorum", result.cause());
}
});
} finally {
ContextImpl.setContext(ctx);
}
}
}
}
}
public HAManager(
VertxInternal vertx,
DeploymentManager deploymentManager,
ClusterManager clusterManager,
int quorumSize,
String group,
boolean enabled) {
this.vertx = vertx;
this.deploymentManager = deploymentManager;
this.clusterManager = clusterManager;
this.quorumSize = enabled ? quorumSize : 0;
this.group = enabled ? group : "__DISABLED__";
this.enabled = enabled;
this.haInfo = new JsonObject();
haInfo.put("verticles", new JsonArray());
haInfo.put("group", this.group);
this.clusterMap = clusterManager.getSyncMap(CLUSTER_MAP_NAME);
this.nodeID = clusterManager.getNodeID();
clusterManager.nodeListener(
new NodeListener() {
@Override
public void nodeAdded(String nodeID) {
HAManager.this.nodeAdded(nodeID);
}
@Override
public void nodeLeft(String leftNodeID) {
HAManager.this.nodeLeft(leftNodeID);
}
});
clusterMap.put(nodeID, haInfo.encode());
quorumTimerID = vertx.setPeriodic(QUORUM_CHECK_PERIOD, tid -> checkHADeployments());
// Call check quorum to compute whether we have an initial quorum
synchronized (this) {
checkQuorum();
}
}
