/**
* Check whether all links are established
*
* @return
*/
public boolean isFullyConnected() {
for (ClusterNode n : syncManager.getClusterConfig().getNodes()) {
if (n.getNodeId() != syncManager.getLocalNodeId() && !isConnected(n.getNodeId())) {
if (logger.isTraceEnabled()) {
logger.trace("[{}->{}] missing connection", syncManager.getLocalNodeId(), n.getNodeId());
}
return false;
}
}
return true;
}
@Test
public void testLeaderElection() throws Exception {
// Check we end up with master nodes
HashMap<Short, Short> leaderMap = checkElectionState(syncManagers, 5000);
// Kill the masters and verify failover
ArrayList<SyncManager> live = new ArrayList<>();
for (SyncManager m : syncManagers) {
ClusterNode n = m.getClusterConfig().getNode();
if (leaderMap.get(n.getDomainId()).equals(n.getNodeId())) m.shutdown();
else live.add(m);
}
checkElectionState(live.toArray(new SyncManager[0]), 5000);
}
private HashMap<Short, Short> checkElectionState(SyncManager[] syncManagers, int maxTime)
throws Exception {
long then = System.currentTimeMillis();
HashSet<Short> domainSet = new HashSet<>();
HashSet<Short> nodeSet = new HashSet<>();
HashMap<Short, Short> leaderMap = new HashMap<>();
while (true) {
leaderMap.clear();
domainSet.clear();
nodeSet.clear();
boolean converged = true;
for (SyncManager s : syncManagers) {
ClusterNode n = s.getClusterConfig().getNode();
Short domainId = n.getDomainId();
domainSet.add(domainId);
nodeSet.add(n.getNodeId());
Short l = s.clusterMgr.getDomainLeader();
if (l != null) {
if (leaderMap.containsKey(domainId)) {
assertEquals(l, leaderMap.get(domainId));
}
leaderMap.put(domainId, l);
} else {
converged = false;
}
}
for (Short l : leaderMap.values()) {
if (!nodeSet.contains(l)) {
converged = false;
break;
}
}
if (converged) break;
Thread.sleep(100);
assertTrue(then + maxTime > System.currentTimeMillis());
}
for (Short did : domainSet) {
assertTrue(leaderMap.containsKey(did));
assertTrue(nodeSet.contains(leaderMap.get(did)));
}
return leaderMap;
}
@Override
public void operationComplete(ChannelFuture cf) throws Exception {
if (!cf.isSuccess()) {
synchronized (connections) {
NodeConnection c = connections.remove(node.getNodeId());
if (c != null) c.nuke();
cf.getChannel().close();
}
String message = "[unknown error]";
if (cf.isCancelled()) message = "Timed out on connect";
if (cf.getCause() != null) message = cf.getCause().getMessage();
logger.debug(
"[{}->{}] Could not connect to RPC " + "node: {}",
new Object[] {syncManager.getLocalNodeId(), node.getNodeId(), message});
} else {
logger.trace(
"[{}->{}] Channel future successful", syncManager.getLocalNodeId(), node.getNodeId());
}
}
/**
* Connect to a remote node if appropriate
*
* @param bootstrap the client bootstrap object
* @param n the node to connect to
*/
protected void doNodeConnect(ClusterNode n) {
if (!shutDown && n.getNodeId() < syncManager.getLocalNodeId()) {
Short nodeId = n.getNodeId();
synchronized (connections) {
NodeConnection c = connections.get(n.getNodeId());
if (c == null) {
connections.put(nodeId, c = new NodeConnection());
}
if (logger.isTraceEnabled()) {
logger.trace(
"[{}->{}] Connection state: {}",
new Object[] {syncManager.getLocalNodeId(), nodeId, c.state});
}
if (c.state.equals(NodeConnectionState.NONE)) {
if (logger.isDebugEnabled()) {
logger.debug(
"[{}->{}] Attempting connection {} {}",
new Object[] {syncManager.getLocalNodeId(), nodeId, n.getHostname(), n.getPort()});
}
SocketAddress sa = new InetSocketAddress(n.getHostname(), n.getPort());
c.pendingFuture = clientBootstrap.connect(sa);
c.pendingFuture.addListener(new ConnectCFListener(n));
c.state = NodeConnectionState.PENDING;
}
}
}
}
protected void setupSyncManager(
FloodlightModuleContext fmc, SyncManager syncManager, ClusterNode thisNode) throws Exception {
fmc.addService(IThreadPoolService.class, tp);
fmc.addService(IDebugCounterService.class, new NullDebugCounter());
fmc.addConfigParam(syncManager, "configProviders", PropertyCCProvider.class.getName());
fmc.addConfigParam(syncManager, "nodes", nodeString);
fmc.addConfigParam(syncManager, "thisNode", "" + thisNode.getNodeId());
fmc.addConfigParam(syncManager, "persistenceEnabled", "false");
fmc.addConfigParam(syncManager, "authScheme", "CHALLENGE_RESPONSE");
fmc.addConfigParam(syncManager, "keyStorePath", keyStoreFile.getAbsolutePath());
fmc.addConfigParam(syncManager, "keyStorePassword", keyStorePassword);
tp.init(fmc);
syncManager.init(fmc);
tp.startUp(fmc);
syncManager.startUp(fmc);
syncManager.registerStore("global", Scope.GLOBAL);
syncManager.registerStore("local", Scope.LOCAL);
}