@Test
public void testRemoveNode() throws Exception {
waitForFullMesh(2000);
IStoreClient<String, String> client0 =
syncManagers[0].getStoreClient("global", String.class, String.class);
IStoreClient<String, String> client1 =
syncManagers[1].getStoreClient("global", String.class, String.class);
IStoreClient<String, String> client2 =
syncManagers[2].getStoreClient("global", String.class, String.class);
client0.put("key", "value");
waitForValue(client1, "key", "value", 2000, "client1");
nodes.remove(0);
nodeString = mapper.writeValueAsString(nodes);
SyncManager oldNode = syncManagers[0];
syncManagers = Arrays.copyOfRange(syncManagers, 1, 4);
moduleContexts = Arrays.copyOfRange(moduleContexts, 1, 4);
try {
for (int i = 0; i < syncManagers.length; i++) {
moduleContexts[i].addConfigParam(syncManagers[i], "nodes", nodeString);
syncManagers[i].doUpdateConfiguration();
waitForConnection(syncManagers[i], (short) 1, false, 2000);
}
} finally {
oldNode.shutdown();
}
waitForFullMesh(2000);
client1.put("newkey", "newvalue");
waitForValue(client2, "key", "value", 2000, "client4");
waitForValue(client2, "newkey", "newvalue", 2000, "client0");
}
@Test
public void testLeaderElection() throws Exception {
// Check we end up with master nodes
HashMap<Short, Short> leaderMap = checkElectionState(syncManagers, 5000);
// Kill the masters and verify failover
ArrayList<SyncManager> live = new ArrayList<>();
for (SyncManager m : syncManagers) {
ClusterNode n = m.getClusterConfig().getNode();
if (leaderMap.get(n.getDomainId()).equals(n.getNodeId())) m.shutdown();
else live.add(m);
}
checkElectionState(live.toArray(new SyncManager[0]), 5000);
}
private HashMap<Short, Short> checkElectionState(SyncManager[] syncManagers, int maxTime)
throws Exception {
long then = System.currentTimeMillis();
HashSet<Short> domainSet = new HashSet<>();
HashSet<Short> nodeSet = new HashSet<>();
HashMap<Short, Short> leaderMap = new HashMap<>();
while (true) {
leaderMap.clear();
domainSet.clear();
nodeSet.clear();
boolean converged = true;
for (SyncManager s : syncManagers) {
ClusterNode n = s.getClusterConfig().getNode();
Short domainId = n.getDomainId();
domainSet.add(domainId);
nodeSet.add(n.getNodeId());
Short l = s.clusterMgr.getDomainLeader();
if (l != null) {
if (leaderMap.containsKey(domainId)) {
assertEquals(l, leaderMap.get(domainId));
}
leaderMap.put(domainId, l);
} else {
converged = false;
}
}
for (Short l : leaderMap.values()) {
if (!nodeSet.contains(l)) {
converged = false;
break;
}
}
if (converged) break;
Thread.sleep(100);
assertTrue(then + maxTime > System.currentTimeMillis());
}
for (Short did : domainSet) {
assertTrue(leaderMap.containsKey(did));
assertTrue(nodeSet.contains(leaderMap.get(did)));
}
return leaderMap;
}
protected void setupSyncManager(
FloodlightModuleContext fmc, SyncManager syncManager, ClusterNode thisNode) throws Exception {
fmc.addService(IThreadPoolService.class, tp);
fmc.addService(IDebugCounterService.class, new NullDebugCounter());
fmc.addConfigParam(syncManager, "configProviders", PropertyCCProvider.class.getName());
fmc.addConfigParam(syncManager, "nodes", nodeString);
fmc.addConfigParam(syncManager, "thisNode", "" + thisNode.getNodeId());
fmc.addConfigParam(syncManager, "persistenceEnabled", "false");
fmc.addConfigParam(syncManager, "authScheme", "CHALLENGE_RESPONSE");
fmc.addConfigParam(syncManager, "keyStorePath", keyStoreFile.getAbsolutePath());
fmc.addConfigParam(syncManager, "keyStorePassword", keyStorePassword);
tp.init(fmc);
syncManager.init(fmc);
tp.startUp(fmc);
syncManager.startUp(fmc);
syncManager.registerStore("global", Scope.GLOBAL);
syncManager.registerStore("local", Scope.LOCAL);
}