protected void stopServer() throws Exception {
LOG.info("STOPPING server");
shutdownServerInstance(serverFactory, hostPort);
serverFactory = null;
// ensure no beans are leftover
JMXEnv.ensureOnly();
}
@After
public void tearDown() throws Exception {
LOG.info("tearDown starting");
tearDownAll();
stopServer();
if (tmpDir != null) {
Assert.assertTrue("delete " + tmpDir.toString(), recursiveDelete(tmpDir));
}
// This has to be set to null when the same instance of this class is reused between test cases
serverFactory = null;
JMXEnv.tearDown();
/* some useful information - log the number of fds used before
* and after a test is run. Helps to verify we are freeing resources
* correctly. Unfortunately this only works on unix systems (the
* only place sun has implemented as part of the mgmt bean api.
*/
OSMXBean osMbean = new OSMXBean();
if (osMbean.getUnix() == true) {
long fdCount = osMbean.getOpenFileDescriptorCount();
String message = "fdcount after test is: " + fdCount + " at start it was " + initialFdCount;
LOG.info(message);
if (fdCount > initialFdCount) {
LOG.info("sleeping for 20 secs");
// Thread.sleep(60000);
// assertTrue(message, fdCount <= initialFdCount);
}
}
}
@Before
public void setUp() throws Exception {
/* some useful information - log the number of fds used before
* and after a test is run. Helps to verify we are freeing resources
* correctly. Unfortunately this only works on unix systems (the
* only place sun has implemented as part of the mgmt bean api.
*/
OSMXBean osMbean = new OSMXBean();
if (osMbean.getUnix() == true) {
initialFdCount = osMbean.getOpenFileDescriptorCount();
LOG.info("Initial fdcount is: " + initialFdCount);
}
setupTestEnv();
JMXEnv.setUp();
setUpAll();
tmpDir = createTmpDir(BASETEST);
startServer();
LOG.info("Client test setup finished");
}
String getAddrPortFromBean(String beanName, String attribute) throws Exception {
String name = (String) JMXEnv.ensureBeanAttribute(beanName, attribute);
if (!name.contains(":")) {
return name;
}
return getNumericalAddrPort(name);
}
protected void startServer() throws Exception {
LOG.info("STARTING server");
serverFactory = createNewServerInstance(serverFactory, hostPort, maxCnxns);
startServerInstance(tmpDir, serverFactory, hostPort);
// ensure that server and data bean are registered
Set<ObjectName> children = JMXEnv.ensureParent("InMemoryDataTree", "StandaloneServer_port");
// Remove beans which are related to zk client sessions. Strong
// assertions cannot be done for these client sessions because
// registeration of these beans with server will happen only on their
// respective reconnection interval
verifyUnexpectedBeans(children);
}
private void assertLocalPeerMXBeanAttributes(
QuorumPeer qp, String beanName, Boolean isPartOfEnsemble) throws Exception {
Assert.assertEquals(
"Mismatches LearnerType!",
qp.getLearnerType().name(),
JMXEnv.ensureBeanAttribute(beanName, "LearnerType"));
Assert.assertEquals(
"Mismatches ClientAddress!",
qp.getClientAddress().getHostString() + ":" + qp.getClientAddress().getPort(),
JMXEnv.ensureBeanAttribute(beanName, "ClientAddress"));
Assert.assertEquals(
"Mismatches LearnerType!",
qp.getElectionAddress().getHostString() + ":" + qp.getElectionAddress().getPort(),
JMXEnv.ensureBeanAttribute(beanName, "ElectionAddress"));
Assert.assertEquals(
"Mismatches PartOfEnsemble!",
isPartOfEnsemble,
JMXEnv.ensureBeanAttribute(beanName, "PartOfEnsemble"));
Assert.assertEquals(
"Mismatches ConfigVersion!",
qp.getQuorumVerifier().getVersion(),
JMXEnv.ensureBeanAttribute(beanName, "ConfigVersion"));
Assert.assertEquals(
"Mismatches QuorumSystemInfo!",
qp.getQuorumVerifier().toString(),
JMXEnv.ensureBeanAttribute(beanName, "QuorumSystemInfo"));
}
@After
@Override
protected void tearDown() throws Exception {
LOG.info("TearDown started");
shutdownServers();
for (String hp : hostPort.split(",")) {
assertTrue(
"waiting for server down",
ClientBase.waitForServerDown(hp, ClientBase.CONNECTION_TIMEOUT));
LOG.info(hp + " is no longer accepting client connections");
}
JMXEnv.tearDown();
LOG.info("FINISHED " + getName());
}
private void assertRemotePeerMXBeanAttributes(QuorumServer qs, String beanName) throws Exception {
Assert.assertEquals(
"Mismatches LearnerType!",
qs.type.name(),
JMXEnv.ensureBeanAttribute(beanName, "LearnerType"));
Assert.assertEquals(
"Mismatches ClientAddress!",
getNumericalAddrPort(qs.clientAddr.getHostString() + ":" + qs.clientAddr.getPort()),
getAddrPortFromBean(beanName, "ClientAddress"));
Assert.assertEquals(
"Mismatches ElectionAddress!",
getNumericalAddrPort(qs.electionAddr.getHostString() + ":" + qs.electionAddr.getPort()),
getAddrPortFromBean(beanName, "ElectionAddress"));
Assert.assertEquals(
"Mismatches QuorumAddress!",
getNumericalAddrPort(qs.addr.getHostString() + ":" + qs.addr.getPort()),
getAddrPortFromBean(beanName, "QuorumAddress"));
}
protected TestableZooKeeper createClient(CountdownWatcher watcher, String hp, int timeout)
throws IOException, InterruptedException {
watcher.reset();
TestableZooKeeper zk = new TestableZooKeeper(hp, timeout, watcher);
if (!watcher.clientConnected.await(timeout, TimeUnit.MILLISECONDS)) {
Assert.fail("Unable to connect to server");
}
synchronized (this) {
if (!allClientsSetup) {
LOG.error("allClients never setup");
Assert.fail("allClients never setup");
}
if (allClients != null) {
allClients.add(zk);
JMXEnv.ensureAll(getHexSessionId(zk.getSessionId()));
} else {
// test done - close the zk, not needed
zk.close();
}
}
return zk;
}
void startServers() throws Exception {
int tickTime = 2000;
int initLimit = 3;
int syncLimit = 3;
JMXEnv.setUp();
HashMap<Long, QuorumServer> peers = new HashMap<Long, QuorumServer>();
peers.put(Long.valueOf(1), new QuorumServer(1, new InetSocketAddress("127.0.0.1", 3181)));
peers.put(Long.valueOf(2), new QuorumServer(2, new InetSocketAddress("127.0.0.1", 3182)));
peers.put(Long.valueOf(3), new QuorumServer(3, new InetSocketAddress("127.0.0.1", 3183)));
peers.put(Long.valueOf(4), new QuorumServer(4, new InetSocketAddress("127.0.0.1", 3184)));
peers.put(Long.valueOf(5), new QuorumServer(5, new InetSocketAddress("127.0.0.1", 3185)));
LOG.info("creating QuorumPeer 1");
s1 = new QuorumPeer(peers, s1dir, s1dir, 2181, 0, 1, tickTime, initLimit, syncLimit);
assertEquals(2181, s1.getClientPort());
LOG.info("creating QuorumPeer 2");
s2 = new QuorumPeer(peers, s2dir, s2dir, 2182, 0, 2, tickTime, initLimit, syncLimit);
assertEquals(2182, s2.getClientPort());
LOG.info("creating QuorumPeer 3");
s3 = new QuorumPeer(peers, s3dir, s3dir, 2183, 0, 3, tickTime, initLimit, syncLimit);
assertEquals(2183, s3.getClientPort());
LOG.info("creating QuorumPeer 4");
s4 = new QuorumPeer(peers, s4dir, s4dir, 2184, 0, 4, tickTime, initLimit, syncLimit);
assertEquals(2184, s4.getClientPort());
LOG.info("creating QuorumPeer 5");
s5 = new QuorumPeer(peers, s5dir, s5dir, 2185, 0, 5, tickTime, initLimit, syncLimit);
assertEquals(2185, s5.getClientPort());
LOG.info("start QuorumPeer 1");
s1.start();
LOG.info("start QuorumPeer 2");
s2.start();
LOG.info("start QuorumPeer 3");
s3.start();
LOG.info("start QuorumPeer 4");
s4.start();
LOG.info("start QuorumPeer 5");
s5.start();
LOG.info("started QuorumPeer 5");
for (String hp : hostPort.split(",")) {
assertTrue("waiting for server up", ClientBase.waitForServerUp(hp, CONNECTION_TIMEOUT));
LOG.info(hp + " is accepting client connections");
}
// interesting to see what's there...
JMXEnv.dump();
// make sure we have these 5 servers listed
Set<String> ensureNames = new LinkedHashSet<String>();
for (int i = 1; i <= 5; i++) {
ensureNames.add("InMemoryDataTree");
}
for (int i = 1; i <= 5; i++) {
ensureNames.add("name0=ReplicatedServer_id" + i + ",name1=replica." + i + ",name2=");
}
for (int i = 1; i <= 5; i++) {
for (int j = 1; j <= 5; j++) {
ensureNames.add("name0=ReplicatedServer_id" + i + ",name1=replica." + j);
}
}
for (int i = 1; i <= 5; i++) {
ensureNames.add("name0=ReplicatedServer_id" + i);
}
JMXEnv.ensureAll(ensureNames.toArray(new String[ensureNames.size()]));
}
public static MBeanServerConnection jmxConn() throws IOException {
return JMXEnv.conn();
}
/**
* Tests verifies the jmx attributes of local and remote peer bean - remove one quorum peer and
* again adding it back
*/
@Test
public void testJMXBeanAfterRemoveAddOne() throws Exception {
qu = new QuorumUtil(1); // create 3 servers
qu.disableJMXTest = true;
qu.startAll();
ZooKeeper[] zkArr = createHandles(qu);
List<String> leavingServers = new ArrayList<String>();
List<String> joiningServers = new ArrayList<String>();
// assert remotePeerBean.1 of ReplicatedServer_2
int leavingIndex = 1;
int replica2 = 2;
QuorumPeer peer2 = qu.getPeer(replica2).peer;
QuorumServer leavingQS2 = peer2.getView().get(new Long(leavingIndex));
String remotePeerBean2 =
CommonNames.DOMAIN
+ ":name0=ReplicatedServer_id"
+ replica2
+ ",name1=replica."
+ leavingIndex;
assertRemotePeerMXBeanAttributes(leavingQS2, remotePeerBean2);
// assert remotePeerBean.1 of ReplicatedServer_3
int replica3 = 3;
QuorumPeer peer3 = qu.getPeer(replica3).peer;
QuorumServer leavingQS3 = peer3.getView().get(new Long(leavingIndex));
String remotePeerBean3 =
CommonNames.DOMAIN
+ ":name0=ReplicatedServer_id"
+ replica3
+ ",name1=replica."
+ leavingIndex;
assertRemotePeerMXBeanAttributes(leavingQS3, remotePeerBean3);
ZooKeeper zk = zkArr[leavingIndex];
leavingServers.add(Integer.toString(leavingIndex));
// remember this server so we can add it back later
joiningServers.add(
"server."
+ leavingIndex
+ "=127.0.0.1:"
+ qu.getPeer(leavingIndex).peer.getQuorumAddress().getPort()
+ ":"
+ qu.getPeer(leavingIndex).peer.getElectionAddress().getPort()
+ ":participant;127.0.0.1:"
+ qu.getPeer(leavingIndex).peer.getClientPort());
// Remove ReplicatedServer_1 from the ensemble
reconfig(zk, null, leavingServers, null, -1);
// localPeerBean.1 of ReplicatedServer_1
QuorumPeer removedPeer = qu.getPeer(leavingIndex).peer;
String localPeerBean =
CommonNames.DOMAIN
+ ":name0=ReplicatedServer_id"
+ leavingIndex
+ ",name1=replica."
+ leavingIndex;
assertLocalPeerMXBeanAttributes(removedPeer, localPeerBean, false);
// remotePeerBean.1 shouldn't exists in ReplicatedServer_2
JMXEnv.ensureNone(remotePeerBean2);
// remotePeerBean.1 shouldn't exists in ReplicatedServer_3
JMXEnv.ensureNone(remotePeerBean3);
// Add ReplicatedServer_1 back to the ensemble
reconfig(zk, joiningServers, null, null, -1);
// localPeerBean.1 of ReplicatedServer_1
assertLocalPeerMXBeanAttributes(removedPeer, localPeerBean, true);
// assert remotePeerBean.1 of ReplicatedServer_2
leavingQS2 = peer2.getView().get(new Long(leavingIndex));
assertRemotePeerMXBeanAttributes(leavingQS2, remotePeerBean2);
// assert remotePeerBean.1 of ReplicatedServer_3
leavingQS3 = peer3.getView().get(new Long(leavingIndex));
assertRemotePeerMXBeanAttributes(leavingQS3, remotePeerBean3);
closeAllHandles(zkArr);
}
