@Test
public void testCnxManagerTimeout() throws Exception {
Random rand = new Random();
byte b = (byte) rand.nextInt();
int deadPort = PortAssignment.unique();
String deadAddress = new String("10.1.1." + b);
LOG.info("This is the dead address I'm trying: " + deadAddress);
peers.put(
Long.valueOf(2),
new QuorumServer(
2,
new InetSocketAddress(deadAddress, deadPort),
new InetSocketAddress(deadAddress, PortAssignment.unique())));
peerTmpdir[2] = ClientBase.createTmpDir();
QuorumPeer peer =
new QuorumPeer(peers, peerTmpdir[1], peerTmpdir[1], peerClientPort[1], 3, 1, 1000, 2, 2);
QuorumCnxManager cnxManager = new QuorumCnxManager(peer);
QuorumCnxManager.Listener listener = cnxManager.listener;
if (listener != null) {
listener.start();
} else {
LOG.error("Null listener when initializing cnx manager");
}
long begin = System.currentTimeMillis();
cnxManager.toSend(new Long(2), createMsg(ServerState.LOOKING.ordinal(), 1, -1, 1));
long end = System.currentTimeMillis();
if ((end - begin) > 6000) fail("Waited more than necessary");
}
public void testLE() throws Exception {
int count = 30;
HashMap<Long, QuorumServer> peers = new HashMap<Long, QuorumServer>(count);
ArrayList<LEThread> threads = new ArrayList<LEThread>(count);
File tmpdir[] = new File[count];
int port[] = new int[count];
votes = new Vote[count];
for (int i = 0; i < count; i++) {
peers.put(
Long.valueOf(i),
new QuorumServer(i, new InetSocketAddress("127.0.0.1", PortAssignment.unique())));
tmpdir[i] = ClientBase.createTmpDir();
port[i] = PortAssignment.unique();
}
LeaderElection le[] = new LeaderElection[count];
leaderDies = true;
boolean allowOneBadLeader = leaderDies;
for (int i = 0; i < le.length; i++) {
QuorumPeer peer = new QuorumPeer(peers, tmpdir[i], tmpdir[i], port[i], 0, i, 1000, 2, 2);
peer.startLeaderElection();
le[i] = new LeaderElection(peer);
LEThread thread = new LEThread(le[i], peer, i);
thread.start();
threads.add(thread);
}
for (int i = 0; i < threads.size(); i++) {
threads.get(i).join(15000);
if (threads.get(i).isAlive()) {
fail("Threads didn't join");
}
}
long id = votes[0].id;
for (int i = 1; i < votes.length; i++) {
if (votes[i] == null) {
fail("Thread " + i + " had a null vote");
}
if (votes[i].id != id) {
if (allowOneBadLeader && votes[i].id == i) {
allowOneBadLeader = false;
} else {
fail("Thread " + i + " got " + votes[i].id + " expected " + id);
}
}
}
}
@Test
public void testBulkReconfig() throws Exception {
qu = new QuorumUtil(3); // create 7 servers
qu.disableJMXTest = true;
qu.startAll();
ZooKeeper[] zkArr = createHandles(qu);
// new config will have three of the servers as followers
// two of the servers as observers, and all ports different
ArrayList<String> newServers = new ArrayList<String>();
for (int i = 1; i <= 5; i++) {
String server =
"server."
+ i
+ "=localhost:"
+ PortAssignment.unique()
+ ":"
+ PortAssignment.unique()
+ ":"
+ ((i == 4 || i == 5) ? "observer" : "participant")
+ ";localhost:"
+ qu.getPeer(i).peer.getClientPort();
newServers.add(server);
}
qu.shutdown(3);
qu.shutdown(6);
qu.shutdown(7);
reconfig(zkArr[1], null, null, newServers, -1);
testNormalOperation(zkArr[1], zkArr[2]);
testServerHasConfig(zkArr[1], newServers, null);
testServerHasConfig(zkArr[2], newServers, null);
testServerHasConfig(zkArr[4], newServers, null);
testServerHasConfig(zkArr[5], newServers, null);
qu.shutdown(5);
qu.shutdown(4);
testNormalOperation(zkArr[1], zkArr[2]);
closeAllHandles(zkArr);
}
public void setUp() throws Exception {
this.count = 3;
this.peers = new HashMap<Long, QuorumServer>(count);
peerTmpdir = new File[count];
peerQuorumPort = new int[count];
peerClientPort = new int[count];
for (int i = 0; i < count; i++) {
peerQuorumPort[i] = PortAssignment.unique();
peerClientPort[i] = PortAssignment.unique();
peers.put(
Long.valueOf(i),
new QuorumServer(
i,
new InetSocketAddress(peerQuorumPort[i]),
new InetSocketAddress(PortAssignment.unique())));
peerTmpdir[i] = ClientBase.createTmpDir();
}
}
@Test
public void testUnspecifiedClientAddress() throws Exception {
int[] ports = new int[3];
for (int port : ports) {
port = PortAssignment.unique();
}
String server = "server.0=localhost:" + ports[0] + ":" + ports[1] + ";" + ports[2];
QuorumServer qs = new QuorumServer(0, server);
Assert.assertEquals(qs.clientAddr.getHostString(), "0.0.0.0");
Assert.assertEquals(qs.clientAddr.getPort(), ports[2]);
}
/** Test case for https://issues.apache.org/jira/browse/ZOOKEEPER-2299 */
@Test
public void testClientAddress() throws Exception {
QuorumPeer quorumPeer = new QuorumPeer();
LocalPeerBean remotePeerBean = new LocalPeerBean(quorumPeer);
/** Case 1: When cnxnFactory is null */
String result = remotePeerBean.getClientAddress();
assertNotNull(result);
assertEquals(0, result.length());
/** Case 2: When only client port is configured */
ServerCnxnFactory cnxnFactory = ServerCnxnFactory.createFactory();
int clientPort = PortAssignment.unique();
InetSocketAddress address = new InetSocketAddress(clientPort);
cnxnFactory.configure(address, 5, false);
quorumPeer.setCnxnFactory(cnxnFactory);
result = remotePeerBean.getClientAddress();
String ipv4 = "0.0.0.0:" + clientPort;
String ipv6 = "0:0:0:0:0:0:0:0:" + clientPort;
assertTrue(result.equals(ipv4) || result.equals(ipv6));
// cleanup
cnxnFactory.shutdown();
/** Case 3: When both client port and client address is configured */
clientPort = PortAssignment.unique();
InetAddress clientIP = InetAddress.getLoopbackAddress();
address = new InetSocketAddress(clientIP, clientPort);
cnxnFactory = ServerCnxnFactory.createFactory();
cnxnFactory.configure(address, 5, false);
quorumPeer.setCnxnFactory(cnxnFactory);
result = remotePeerBean.getClientAddress();
String expectedResult = clientIP.getHostAddress() + ":" + clientPort;
assertEquals(expectedResult, result);
// cleanup
cnxnFactory.shutdown();
}
public class SessionTest extends ZKTestCase implements Watcher {
protected static final Logger LOG = Logger.getLogger(SessionTest.class);
private static final String HOSTPORT = "127.0.0.1:" + PortAssignment.unique();
private NIOServerCnxn.Factory serverFactory;
private CountDownLatch startSignal;
File tmpDir;
private final int TICK_TIME = 3000;
@Before
public void setUp() throws Exception {
if (tmpDir == null) {
tmpDir = ClientBase.createTmpDir();
}
ClientBase.setupTestEnv();
ZooKeeperServer zs = new ZooKeeperServer(tmpDir, tmpDir, TICK_TIME);
final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
serverFactory = new NIOServerCnxn.Factory(new InetSocketAddress(PORT));
serverFactory.startup(zs);
Assert.assertTrue(
"waiting for server up", ClientBase.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT));
}
@After
public void tearDown() throws Exception {
serverFactory.shutdown();
Assert.assertTrue(
"waiting for server down", ClientBase.waitForServerDown(HOSTPORT, CONNECTION_TIMEOUT));
}
private static class CountdownWatcher implements Watcher {
volatile CountDownLatch clientConnected = new CountDownLatch(1);
public void process(WatchedEvent event) {
if (event.getState() == KeeperState.SyncConnected) {
clientConnected.countDown();
}
}
}
private DisconnectableZooKeeper createClient() throws IOException, InterruptedException {
CountdownWatcher watcher = new CountdownWatcher();
return createClient(CONNECTION_TIMEOUT, watcher);
}
private DisconnectableZooKeeper createClient(int timeout)
throws IOException, InterruptedException {
CountdownWatcher watcher = new CountdownWatcher();
return createClient(timeout, watcher);
}
private DisconnectableZooKeeper createClient(int timeout, CountdownWatcher watcher)
throws IOException, InterruptedException {
DisconnectableZooKeeper zk = new DisconnectableZooKeeper(HOSTPORT, timeout, watcher);
if (!watcher.clientConnected.await(timeout, TimeUnit.MILLISECONDS)) {
Assert.fail("Unable to connect to server");
}
return zk;
}
// FIXME this test is Assert.failing due to client close race condition fixing in separate patch
// for ZOOKEEPER-63
// /**
// * this test checks to see if the sessionid that was created for the
// * first zookeeper client can be reused for the second one immidiately
// * after the first client closes and the new client resues them.
// * @throws IOException
// * @throws InterruptedException
// * @throws KeeperException
// */
// public void testSessionReuse() throws IOException, InterruptedException {
// ZooKeeper zk = createClient();
//
// long sessionId = zk.getSessionId();
// byte[] passwd = zk.getSessionPasswd();
// zk.close();
//
// zk.close();
//
// LOG.info("Closed first session");
//
// startSignal = new CountDownLatch(1);
// zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this,
// sessionId, passwd);
// startSignal.await();
//
// LOG.info("Opened reuse");
//
// Assert.assertEquals(sessionId, zk.getSessionId());
//
// zk.close();
// }
/**
* This test verifies that when the session id is reused, and the original client is disconnected,
* but not session closed, that the server will remove ephemeral nodes created by the original
* session.
*/
@Test
public void testSession() throws IOException, InterruptedException, KeeperException {
DisconnectableZooKeeper zk = createClient();
zk.create("/e", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
LOG.info("zk with session id 0x" + Long.toHexString(zk.getSessionId()) + " was destroyed!");
// disconnect the client by killing the socket, not sending the
// session disconnect to the server as usual. This allows the test
// to verify disconnect handling
zk.disconnect();
Stat stat = new Stat();
startSignal = new CountDownLatch(1);
zk =
new DisconnectableZooKeeper(
HOSTPORT, CONNECTION_TIMEOUT, this, zk.getSessionId(), zk.getSessionPasswd());
startSignal.await();
LOG.info("zk with session id 0x" + Long.toHexString(zk.getSessionId()) + " was created!");
zk.getData("/e", false, stat);
LOG.info("After get data /e");
zk.close();
zk = createClient();
Assert.assertEquals(null, zk.exists("/e", false));
LOG.info("before close zk with session id 0x" + Long.toHexString(zk.getSessionId()) + "!");
zk.close();
}
/** Make sure ephemerals get cleaned up when a session times out. */
@Test
public void testSessionTimeout() throws Exception {
final int TIMEOUT = 5000;
DisconnectableZooKeeper zk = createClient(TIMEOUT);
zk.create("/stest", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
zk.disconnect();
Thread.sleep(TIMEOUT * 2);
zk = createClient(TIMEOUT);
zk.create("/stest", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
tearDown();
zk.close();
zk.disconnect();
setUp();
zk = createClient(TIMEOUT);
Assert.assertTrue(zk.exists("/stest", false) != null);
Thread.sleep(TIMEOUT * 2);
Assert.assertTrue(zk.exists("/stest", false) == null);
zk.close();
}
/**
* Make sure that we cannot have two connections with the same session id.
*
* @throws IOException
* @throws InterruptedException
* @throws KeeperException
*/
@Test
public void testSessionMove() throws IOException, InterruptedException, KeeperException {
String hostPorts[] = HOSTPORT.split(",");
ZooKeeper zk = new DisconnectableZooKeeper(hostPorts[0], CONNECTION_TIMEOUT, this);
zk.create("/sessionMoveTest", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
// we want to loop through the list twice
for (int i = 0; i < hostPorts.length * 2; i++) {
// This should stomp the zk handle
ZooKeeper zknew =
new DisconnectableZooKeeper(
hostPorts[(i + 1) % hostPorts.length],
CONNECTION_TIMEOUT,
this,
zk.getSessionId(),
zk.getSessionPasswd());
zknew.setData("/", new byte[1], -1);
try {
zk.setData("/", new byte[1], -1);
Assert.fail("Should have lost the connection");
} catch (KeeperException.ConnectionLossException e) {
}
// zk.close();
zk = zknew;
}
zk.close();
}
/**
* This test makes sure that duplicate state changes are not communicated to the client watcher.
* For example we should not notify state as "disconnected" if the watch has already been
* disconnected. In general we don't consider a dup state notification if the event type is not
* "None" (ie non-None communicates an event).
*/
@Test
public void testSessionStateNoDupStateReporting()
throws IOException, InterruptedException, KeeperException {
final int TIMEOUT = 3000;
DupWatcher watcher = new DupWatcher();
ZooKeeper zk = createClient(TIMEOUT, watcher);
// shutdown the server
serverFactory.shutdown();
try {
Thread.sleep(10000);
} catch (InterruptedException e) {
// ignore
}
// verify that the size is just 2 - ie connect then disconnect
// if the client attempts reconnect and we are not handling current
// state correctly (ie eventing on duplicate disconnects) then we'll
// see a disconnect for each Assert.failed connection attempt
Assert.assertEquals(2, watcher.states.size());
zk.close();
}
/** Verify access to the negotiated session timeout. */
@Test
public void testSessionTimeoutAccess() throws Exception {
// validate typical case - requested == negotiated
DisconnectableZooKeeper zk = createClient(TICK_TIME * 4);
Assert.assertEquals(TICK_TIME * 4, zk.getSessionTimeout());
// make sure tostring works in both cases
LOG.info(zk.toString());
zk.close();
LOG.info(zk.toString());
// validate lower limit
zk = createClient(TICK_TIME);
Assert.assertEquals(TICK_TIME * 2, zk.getSessionTimeout());
LOG.info(zk.toString());
zk.close();
LOG.info(zk.toString());
// validate upper limit
zk = createClient(TICK_TIME * 30);
Assert.assertEquals(TICK_TIME * 20, zk.getSessionTimeout());
LOG.info(zk.toString());
zk.close();
LOG.info(zk.toString());
}
private class DupWatcher extends CountdownWatcher {
public LinkedList<WatchedEvent> states = new LinkedList<WatchedEvent>();
public void process(WatchedEvent event) {
super.process(event);
if (event.getType() == EventType.None) {
states.add(event);
}
}
}
public void process(WatchedEvent event) {
LOG.info("Event:" + event.getState() + " " + event.getType() + " " + event.getPath());
if (event.getState() == KeeperState.SyncConnected
&& startSignal != null
&& startSignal.getCount() > 0) {
startSignal.countDown();
}
}
@Test
public void testMinMaxSessionTimeout() throws Exception {
// override the defaults
final int MINSESS = 20000;
final int MAXSESS = 240000;
ZooKeeperServer zs = serverFactory.getZooKeeperServer();
zs.setMinSessionTimeout(MINSESS);
zs.setMaxSessionTimeout(MAXSESS);
// validate typical case - requested == negotiated
int timeout = 120000;
DisconnectableZooKeeper zk = createClient(timeout);
Assert.assertEquals(timeout, zk.getSessionTimeout());
// make sure tostring works in both cases
LOG.info(zk.toString());
zk.close();
LOG.info(zk.toString());
// validate lower limit
zk = createClient(MINSESS / 2);
Assert.assertEquals(MINSESS, zk.getSessionTimeout());
LOG.info(zk.toString());
zk.close();
LOG.info(zk.toString());
// validate upper limit
zk = createClient(MAXSESS * 2);
Assert.assertEquals(MAXSESS, zk.getSessionTimeout());
LOG.info(zk.toString());
zk.close();
LOG.info(zk.toString());
}
}
public abstract class ClientBase extends ZKTestCase {
protected static final Logger LOG = LoggerFactory.getLogger(ClientBase.class);
public static int CONNECTION_TIMEOUT = 30000;
static final File BASETEST = new File(System.getProperty("build.test.dir", "build"));
protected int port = PortAssignment.unique();
protected String hostPort = "127.0.0.1:" + port;
protected String ipv6HostPort = "[0:0:0:0:0:0:0:1]:" + port;
protected int maxCnxns = 0;
protected ServerCnxnFactory serverFactory = null;
protected File tmpDir = null;
long initialFdCount;
public ClientBase() {
super();
}
/**
* In general don't use this. Only use in the special case that you want to ignore results (for
* whatever reason) in your test. Don't use empty watchers in real code!
*/
protected class NullWatcher implements Watcher {
public void process(WatchedEvent event) {
/* nada */
}
}
public static class CountdownWatcher implements Watcher {
// XXX this doesn't need to be volatile! (Should probably be final)
volatile CountDownLatch clientConnected;
volatile boolean connected;
public CountdownWatcher() {
reset();
}
public synchronized void reset() {
clientConnected = new CountDownLatch(1);
connected = false;
}
public synchronized void process(WatchedEvent event) {
if (event.getState() == KeeperState.SyncConnected
|| event.getState() == KeeperState.ConnectedReadOnly) {
connected = true;
notifyAll();
clientConnected.countDown();
} else {
connected = false;
notifyAll();
}
}
public synchronized boolean isConnected() {
return connected;
}
public synchronized void waitForConnected(long timeout)
throws InterruptedException, TimeoutException {
long expire = System.currentTimeMillis() + timeout;
long left = timeout;
while (!connected && left > 0) {
wait(left);
left = expire - System.currentTimeMillis();
}
if (!connected) {
throw new TimeoutException("Did not connect");
}
}
public synchronized void waitForDisconnected(long timeout)
throws InterruptedException, TimeoutException {
long expire = System.currentTimeMillis() + timeout;
long left = timeout;
while (connected && left > 0) {
wait(left);
left = expire - System.currentTimeMillis();
}
if (connected) {
throw new TimeoutException("Did not disconnect");
}
}
}
protected TestableZooKeeper createClient() throws IOException, InterruptedException {
return createClient(hostPort);
}
protected TestableZooKeeper createClient(String hp) throws IOException, InterruptedException {
CountdownWatcher watcher = new CountdownWatcher();
return createClient(watcher, hp);
}
protected TestableZooKeeper createClient(CountdownWatcher watcher)
throws IOException, InterruptedException {
return createClient(watcher, hostPort);
}
private LinkedList<ZooKeeper> allClients;
private boolean allClientsSetup = false;
protected TestableZooKeeper createClient(CountdownWatcher watcher, String hp)
throws IOException, InterruptedException {
return createClient(watcher, hp, CONNECTION_TIMEOUT);
}
protected TestableZooKeeper createClient(CountdownWatcher watcher, String hp, int timeout)
throws IOException, InterruptedException {
watcher.reset();
TestableZooKeeper zk = new TestableZooKeeper(hp, timeout, watcher);
if (!watcher.clientConnected.await(timeout, TimeUnit.MILLISECONDS)) {
Assert.fail("Unable to connect to server");
}
synchronized (this) {
if (!allClientsSetup) {
LOG.error("allClients never setup");
Assert.fail("allClients never setup");
}
if (allClients != null) {
allClients.add(zk);
JMXEnv.ensureAll(getHexSessionId(zk.getSessionId()));
} else {
// test done - close the zk, not needed
zk.close();
}
}
return zk;
}
public static class HostPort {
String host;
int port;
public HostPort(String host, int port) {
this.host = host;
this.port = port;
}
}
public static List<HostPort> parseHostPortList(String hplist) {
ArrayList<HostPort> alist = new ArrayList<HostPort>();
for (String hp : hplist.split(",")) {
int idx = hp.lastIndexOf(':');
String host = hp.substring(0, idx);
int port;
try {
port = Integer.parseInt(hp.substring(idx + 1));
} catch (RuntimeException e) {
throw new RuntimeException("Problem parsing " + hp + e.toString());
}
alist.add(new HostPort(host, port));
}
return alist;
}
public static boolean waitForServerUp(String hp, long timeout) {
long start = System.currentTimeMillis();
while (true) {
try {
// if there are multiple hostports, just take the first one
HostPort hpobj = parseHostPortList(hp).get(0);
String result = send4LetterWord(hpobj.host, hpobj.port, "stat");
if (result.startsWith("Zookeeper version:") && !result.contains("READ-ONLY")) {
return true;
}
} catch (IOException e) {
// ignore as this is expected
LOG.info("server " + hp + " not up " + e);
}
if (System.currentTimeMillis() > start + timeout) {
break;
}
try {
Thread.sleep(250);
} catch (InterruptedException e) {
// ignore
}
}
return false;
}
public static boolean waitForServerDown(String hp, long timeout) {
long start = System.currentTimeMillis();
while (true) {
try {
HostPort hpobj = parseHostPortList(hp).get(0);
send4LetterWord(hpobj.host, hpobj.port, "stat");
} catch (IOException e) {
return true;
}
if (System.currentTimeMillis() > start + timeout) {
break;
}
try {
Thread.sleep(250);
} catch (InterruptedException e) {
// ignore
}
}
return false;
}
public static boolean waitForServerState(QuorumPeer qp, int timeout, String serverState) {
long start = System.currentTimeMillis();
while (true) {
try {
Thread.sleep(250);
} catch (InterruptedException e) {
// ignore
}
if (qp.getServerState().equals(serverState)) return true;
if (System.currentTimeMillis() > start + timeout) {
return false;
}
}
}
static void verifyThreadTerminated(Thread thread, long millis) throws InterruptedException {
thread.join(millis);
if (thread.isAlive()) {
LOG.error("Thread " + thread.getName() + " : " + Arrays.toString(thread.getStackTrace()));
Assert.assertFalse("thread " + thread.getName() + " still alive after join", true);
}
}
public static File createTmpDir() throws IOException {
return createTmpDir(BASETEST);
}
static File createTmpDir(File parentDir) throws IOException {
File tmpFile = File.createTempFile("test", ".junit", parentDir);
// don't delete tmpFile - this ensures we don't attempt to create
// a tmpDir with a duplicate name
File tmpDir = new File(tmpFile + ".dir");
Assert.assertFalse(tmpDir.exists()); // never true if tmpfile does it's job
Assert.assertTrue(tmpDir.mkdirs());
return tmpDir;
}
private static int getPort(String hostPort) {
String[] split = hostPort.split(":");
String portstr = split[split.length - 1];
String[] pc = portstr.split("/");
if (pc.length > 1) {
portstr = pc[0];
}
return Integer.parseInt(portstr);
}
/** Starting the given server instance */
public static void startServerInstance(File dataDir, ServerCnxnFactory factory, String hostPort)
throws IOException, InterruptedException {
final int port = getPort(hostPort);
LOG.info("STARTING server instance 127.0.0.1:{}", port);
ZooKeeperServer zks = new ZooKeeperServer(dataDir, dataDir, 3000);
factory.startup(zks);
Assert.assertTrue(
"waiting for server up",
ClientBase.waitForServerUp("127.0.0.1:" + port, CONNECTION_TIMEOUT));
}
/**
* This method instantiates a new server. Starting of the server instance has been moved to a
* separate method {@link ClientBase#startServerInstance(File, ServerCnxnFactory, String)}.
* Because any exception on starting the server would leave the server running and the caller
* would not be able to shutdown the instance. This may affect other test cases.
*
* @return newly created server instance
* @see <a href="https://issues.apache.org/jira/browse/ZOOKEEPER-1852">ZOOKEEPER-1852</a> for more
* information.
*/
public static ServerCnxnFactory createNewServerInstance(
ServerCnxnFactory factory, String hostPort, int maxCnxns)
throws IOException, InterruptedException {
final int port = getPort(hostPort);
LOG.info("CREATING server instance 127.0.0.1:{}", port);
if (factory == null) {
factory = ServerCnxnFactory.createFactory(port, maxCnxns);
}
return factory;
}
static void shutdownServerInstance(ServerCnxnFactory factory, String hostPort) {
if (factory != null) {
ZKDatabase zkDb = null;
{
ZooKeeperServer zs = getServer(factory);
if (zs != null) {
zkDb = zs.getZKDatabase();
}
}
factory.shutdown();
try {
if (zkDb != null) {
zkDb.close();
}
} catch (IOException ie) {
LOG.warn("Error closing logs ", ie);
}
final int PORT = getPort(hostPort);
Assert.assertTrue(
"waiting for server down",
ClientBase.waitForServerDown("127.0.0.1:" + PORT, CONNECTION_TIMEOUT));
}
}
/** Test specific setup */
public static void setupTestEnv() {
// during the tests we run with 100K prealloc in the logs.
// on windows systems prealloc of 64M was seen to take ~15seconds
// resulting in test Assert.failure (client timeout on first session).
// set env and directly in order to handle static init/gc issues
System.setProperty("zookeeper.preAllocSize", "100");
FileTxnLog.setPreallocSize(100 * 1024);
}
protected void setUpAll() throws Exception {
allClients = new LinkedList<ZooKeeper>();
allClientsSetup = true;
}
@Before
public void setUp() throws Exception {
/* some useful information - log the number of fds used before
* and after a test is run. Helps to verify we are freeing resources
* correctly. Unfortunately this only works on unix systems (the
* only place sun has implemented as part of the mgmt bean api.
*/
OSMXBean osMbean = new OSMXBean();
if (osMbean.getUnix() == true) {
initialFdCount = osMbean.getOpenFileDescriptorCount();
LOG.info("Initial fdcount is: " + initialFdCount);
}
setupTestEnv();
JMXEnv.setUp();
setUpAll();
tmpDir = createTmpDir(BASETEST);
startServer();
LOG.info("Client test setup finished");
}
protected void startServer() throws Exception {
LOG.info("STARTING server");
serverFactory = createNewServerInstance(serverFactory, hostPort, maxCnxns);
startServerInstance(tmpDir, serverFactory, hostPort);
// ensure that server and data bean are registered
Set<ObjectName> children = JMXEnv.ensureParent("InMemoryDataTree", "StandaloneServer_port");
// Remove beans which are related to zk client sessions. Strong
// assertions cannot be done for these client sessions because
// registeration of these beans with server will happen only on their
// respective reconnection interval
verifyUnexpectedBeans(children);
}
private void verifyUnexpectedBeans(Set<ObjectName> children) {
if (allClients != null) {
for (ZooKeeper zkc : allClients) {
Iterator<ObjectName> childItr = children.iterator();
while (childItr.hasNext()) {
ObjectName clientBean = childItr.next();
if (clientBean.toString().contains(getHexSessionId(zkc.getSessionId()))) {
LOG.info("found name:" + zkc.getSessionId() + " client bean:" + clientBean.toString());
childItr.remove();
}
}
}
}
for (ObjectName bean : children) {
LOG.info("unexpected:" + bean.toString());
}
TestCase.assertEquals("Unexpected bean exists!", 0, children.size());
}
/**
* Returns a string representation of the given long value session id
*
* @param sessionId long value of session id
* @return string representation of session id
*/
protected static String getHexSessionId(long sessionId) {
return "0x" + Long.toHexString(sessionId);
}
protected void stopServer() throws Exception {
LOG.info("STOPPING server");
shutdownServerInstance(serverFactory, hostPort);
serverFactory = null;
// ensure no beans are leftover
JMXEnv.ensureOnly();
}
protected static ZooKeeperServer getServer(ServerCnxnFactory fac) {
ZooKeeperServer zs = ServerCnxnFactoryAccessor.getZkServer(fac);
return zs;
}
protected void tearDownAll() throws Exception {
synchronized (this) {
if (allClients != null)
for (ZooKeeper zk : allClients) {
try {
if (zk != null) zk.close();
} catch (InterruptedException e) {
LOG.warn("ignoring interrupt", e);
}
}
allClients = null;
}
}
@After
public void tearDown() throws Exception {
LOG.info("tearDown starting");
tearDownAll();
stopServer();
if (tmpDir != null) {
Assert.assertTrue("delete " + tmpDir.toString(), recursiveDelete(tmpDir));
}
// This has to be set to null when the same instance of this class is reused between test cases
serverFactory = null;
JMXEnv.tearDown();
/* some useful information - log the number of fds used before
* and after a test is run. Helps to verify we are freeing resources
* correctly. Unfortunately this only works on unix systems (the
* only place sun has implemented as part of the mgmt bean api.
*/
OSMXBean osMbean = new OSMXBean();
if (osMbean.getUnix() == true) {
long fdCount = osMbean.getOpenFileDescriptorCount();
String message = "fdcount after test is: " + fdCount + " at start it was " + initialFdCount;
LOG.info(message);
if (fdCount > initialFdCount) {
LOG.info("sleeping for 20 secs");
// Thread.sleep(60000);
// assertTrue(message, fdCount <= initialFdCount);
}
}
}
public static MBeanServerConnection jmxConn() throws IOException {
return JMXEnv.conn();
}
public static boolean recursiveDelete(File d) {
if (d.isDirectory()) {
File children[] = d.listFiles();
for (File f : children) {
Assert.assertTrue("delete " + f.toString(), recursiveDelete(f));
}
}
return d.delete();
}
public static void logAllStackTraces() {
StringBuilder sb = new StringBuilder();
sb.append("Starting logAllStackTraces()\n");
Map<Thread, StackTraceElement[]> threads = Thread.getAllStackTraces();
for (Entry<Thread, StackTraceElement[]> e : threads.entrySet()) {
sb.append("Thread " + e.getKey().getName() + "\n");
for (StackTraceElement elem : e.getValue()) {
sb.append("\tat " + elem + "\n");
}
}
sb.append("Ending logAllStackTraces()\n");
LOG.error(sb.toString());
}
/*
* Verify that all of the servers see the same number of nodes
* at the root
*/
void verifyRootOfAllServersMatch(String hostPort)
throws InterruptedException, KeeperException, IOException {
String parts[] = hostPort.split(",");
// run through till the counts no longer change on each server
// max 15 tries, with 2 second sleeps, so approx 30 seconds
int[] counts = new int[parts.length];
int failed = 0;
for (int j = 0; j < 100; j++) {
int newcounts[] = new int[parts.length];
int i = 0;
for (String hp : parts) {
try {
ZooKeeper zk = createClient(hp);
try {
newcounts[i++] = zk.getChildren("/", false).size();
} finally {
zk.close();
}
} catch (Throwable t) {
failed++;
// if session creation Assert.fails dump the thread stack
// and try the next server
logAllStackTraces();
}
}
if (Arrays.equals(newcounts, counts)) {
LOG.info("Found match with array:" + Arrays.toString(newcounts));
counts = newcounts;
break;
} else {
counts = newcounts;
Thread.sleep(10000);
}
// don't keep this up too long, will Assert.assert false below
if (failed > 10) {
break;
}
}
// verify all the servers reporting same number of nodes
String logmsg = "node count not consistent{} {}";
for (int i = 1; i < parts.length; i++) {
if (counts[i - 1] != counts[i]) {
LOG.error(logmsg, Integer.valueOf(counts[i - 1]), Integer.valueOf(counts[i]));
} else {
LOG.info(logmsg, Integer.valueOf(counts[i - 1]), Integer.valueOf(counts[i]));
}
}
}
public static String readFile(File file) throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
BufferedInputStream is = new BufferedInputStream(new FileInputStream(file));
try {
IOUtils.copyBytes(is, os, 1024, true);
} finally {
is.close();
}
return os.toString();
}
public static String join(String separator, Object[] parts) {
StringBuilder sb = new StringBuilder();
boolean first = true;
for (Object part : parts) {
if (!first) {
sb.append(separator);
first = false;
}
sb.append(part);
}
return sb.toString();
}
}
@Test
public void testPortChange() throws Exception {
qu = new QuorumUtil(1); // create 3 servers
qu.disableJMXTest = true;
qu.startAll();
ZooKeeper[] zkArr = createHandles(qu);
List<String> joiningServers = new ArrayList<String>();
int leaderIndex = getLeaderId(qu);
int followerIndex = leaderIndex == 1 ? 2 : 1;
// modify follower's client port
int quorumPort = qu.getPeer(followerIndex).peer.getQuorumAddress().getPort();
int electionPort = qu.getPeer(followerIndex).peer.getElectionAddress().getPort();
int oldClientPort = qu.getPeer(followerIndex).peer.getClientPort();
int newClientPort = PortAssignment.unique();
joiningServers.add(
"server."
+ followerIndex
+ "=localhost:"
+ quorumPort
+ ":"
+ electionPort
+ ":participant;localhost:"
+ newClientPort);
// create a /test znode and check that read/write works before
// any reconfig is invoked
testNormalOperation(zkArr[followerIndex], zkArr[leaderIndex]);
reconfig(zkArr[followerIndex], joiningServers, null, null, -1);
try {
for (int i = 0; i < 20; i++) {
Thread.sleep(1000);
zkArr[followerIndex].setData("/test", "teststr".getBytes(), -1);
}
} catch (KeeperException.ConnectionLossException e) {
Assert.fail("Existing client disconnected when client port changed!");
}
zkArr[followerIndex].close();
zkArr[followerIndex] =
new ZooKeeper(
"127.0.0.1:" + oldClientPort,
ClientBase.CONNECTION_TIMEOUT,
new Watcher() {
public void process(WatchedEvent event) {}
});
for (int i = 0; i < 10; i++) {
try {
Thread.sleep(1000);
zkArr[followerIndex].setData("/test", "teststr".getBytes(), -1);
Assert.fail("New client connected to old client port!");
} catch (KeeperException.ConnectionLossException e) {
}
}
zkArr[followerIndex].close();
zkArr[followerIndex] =
new ZooKeeper(
"127.0.0.1:" + newClientPort,
ClientBase.CONNECTION_TIMEOUT,
new Watcher() {
public void process(WatchedEvent event) {}
});
testNormalOperation(zkArr[followerIndex], zkArr[leaderIndex]);
testServerHasConfig(zkArr[followerIndex], joiningServers, null);
Assert.assertEquals(newClientPort, qu.getPeer(followerIndex).peer.getClientPort());
joiningServers.clear();
// change leader's leading port - should renounce leadership
int newQuorumPort = PortAssignment.unique();
joiningServers.add(
"server."
+ leaderIndex
+ "=localhost:"
+ newQuorumPort
+ ":"
+ qu.getPeer(leaderIndex).peer.getElectionAddress().getPort()
+ ":participant;localhost:"
+ qu.getPeer(leaderIndex).peer.getClientPort());
reconfig(zkArr[leaderIndex], joiningServers, null, null, -1);
testNormalOperation(zkArr[followerIndex], zkArr[leaderIndex]);
Assert.assertTrue(qu.getPeer(leaderIndex).peer.getQuorumAddress().getPort() == newQuorumPort);
Assert.assertTrue(getLeaderId(qu) != leaderIndex); // the leader changed
joiningServers.clear();
// change everyone's leader election port
for (int i = 1; i <= 3; i++) {
joiningServers.add(
"server."
+ i
+ "=localhost:"
+ qu.getPeer(i).peer.getQuorumAddress().getPort()
+ ":"
+ PortAssignment.unique()
+ ":participant;localhost:"
+ qu.getPeer(i).peer.getClientPort());
}
reconfig(zkArr[1], joiningServers, null, null, -1);
leaderIndex = getLeaderId(qu);
int follower1 = leaderIndex == 1 ? 2 : 1;
int follower2 = 1;
while (follower2 == leaderIndex || follower2 == follower1) follower2++;
// lets kill the leader and see if a new one is elected
qu.shutdown(getLeaderId(qu));
testNormalOperation(zkArr[follower2], zkArr[follower1]);
testServerHasConfig(zkArr[follower1], joiningServers, null);
testServerHasConfig(zkArr[follower2], joiningServers, null);
closeAllHandles(zkArr);
}