@Test
  public void testCnxManagerTimeout() throws Exception {
    Random rand = new Random();
    byte b = (byte) rand.nextInt();
    int deadPort = PortAssignment.unique();
    String deadAddress = new String("10.1.1." + b);

    LOG.info("This is the dead address I'm trying: " + deadAddress);

    peers.put(
        Long.valueOf(2),
        new QuorumServer(
            2,
            new InetSocketAddress(deadAddress, deadPort),
            new InetSocketAddress(deadAddress, PortAssignment.unique())));
    peerTmpdir[2] = ClientBase.createTmpDir();

    QuorumPeer peer =
        new QuorumPeer(peers, peerTmpdir[1], peerTmpdir[1], peerClientPort[1], 3, 1, 1000, 2, 2);
    QuorumCnxManager cnxManager = new QuorumCnxManager(peer);
    QuorumCnxManager.Listener listener = cnxManager.listener;
    if (listener != null) {
      listener.start();
    } else {
      LOG.error("Null listener when initializing cnx manager");
    }

    long begin = System.currentTimeMillis();
    cnxManager.toSend(new Long(2), createMsg(ServerState.LOOKING.ordinal(), 1, -1, 1));
    long end = System.currentTimeMillis();

    if ((end - begin) > 6000) fail("Waited more than necessary");
  }
Esempio n. 2
0
 public void testLE() throws Exception {
   int count = 30;
   HashMap<Long, QuorumServer> peers = new HashMap<Long, QuorumServer>(count);
   ArrayList<LEThread> threads = new ArrayList<LEThread>(count);
   File tmpdir[] = new File[count];
   int port[] = new int[count];
   votes = new Vote[count];
   for (int i = 0; i < count; i++) {
     peers.put(
         Long.valueOf(i),
         new QuorumServer(i, new InetSocketAddress("127.0.0.1", PortAssignment.unique())));
     tmpdir[i] = ClientBase.createTmpDir();
     port[i] = PortAssignment.unique();
   }
   LeaderElection le[] = new LeaderElection[count];
   leaderDies = true;
   boolean allowOneBadLeader = leaderDies;
   for (int i = 0; i < le.length; i++) {
     QuorumPeer peer = new QuorumPeer(peers, tmpdir[i], tmpdir[i], port[i], 0, i, 1000, 2, 2);
     peer.startLeaderElection();
     le[i] = new LeaderElection(peer);
     LEThread thread = new LEThread(le[i], peer, i);
     thread.start();
     threads.add(thread);
   }
   for (int i = 0; i < threads.size(); i++) {
     threads.get(i).join(15000);
     if (threads.get(i).isAlive()) {
       fail("Threads didn't join");
     }
   }
   long id = votes[0].id;
   for (int i = 1; i < votes.length; i++) {
     if (votes[i] == null) {
       fail("Thread " + i + " had a null vote");
     }
     if (votes[i].id != id) {
       if (allowOneBadLeader && votes[i].id == i) {
         allowOneBadLeader = false;
       } else {
         fail("Thread " + i + " got " + votes[i].id + " expected " + id);
       }
     }
   }
 }
  @Test
  public void testBulkReconfig() throws Exception {
    qu = new QuorumUtil(3); // create 7 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    // new config will have three of the servers as followers
    // two of the servers as observers, and all ports different
    ArrayList<String> newServers = new ArrayList<String>();
    for (int i = 1; i <= 5; i++) {
      String server =
          "server."
              + i
              + "=localhost:"
              + PortAssignment.unique()
              + ":"
              + PortAssignment.unique()
              + ":"
              + ((i == 4 || i == 5) ? "observer" : "participant")
              + ";localhost:"
              + qu.getPeer(i).peer.getClientPort();
      newServers.add(server);
    }

    qu.shutdown(3);
    qu.shutdown(6);
    qu.shutdown(7);

    reconfig(zkArr[1], null, null, newServers, -1);
    testNormalOperation(zkArr[1], zkArr[2]);

    testServerHasConfig(zkArr[1], newServers, null);
    testServerHasConfig(zkArr[2], newServers, null);
    testServerHasConfig(zkArr[4], newServers, null);
    testServerHasConfig(zkArr[5], newServers, null);

    qu.shutdown(5);
    qu.shutdown(4);

    testNormalOperation(zkArr[1], zkArr[2]);

    closeAllHandles(zkArr);
  }
  public void setUp() throws Exception {

    this.count = 3;
    this.peers = new HashMap<Long, QuorumServer>(count);
    peerTmpdir = new File[count];
    peerQuorumPort = new int[count];
    peerClientPort = new int[count];

    for (int i = 0; i < count; i++) {
      peerQuorumPort[i] = PortAssignment.unique();
      peerClientPort[i] = PortAssignment.unique();
      peers.put(
          Long.valueOf(i),
          new QuorumServer(
              i,
              new InetSocketAddress(peerQuorumPort[i]),
              new InetSocketAddress(PortAssignment.unique())));
      peerTmpdir[i] = ClientBase.createTmpDir();
    }
  }
 @Test
 public void testUnspecifiedClientAddress() throws Exception {
   int[] ports = new int[3];
   for (int port : ports) {
     port = PortAssignment.unique();
   }
   String server = "server.0=localhost:" + ports[0] + ":" + ports[1] + ";" + ports[2];
   QuorumServer qs = new QuorumServer(0, server);
   Assert.assertEquals(qs.clientAddr.getHostString(), "0.0.0.0");
   Assert.assertEquals(qs.clientAddr.getPort(), ports[2]);
 }
  /** Test case for https://issues.apache.org/jira/browse/ZOOKEEPER-2299 */
  @Test
  public void testClientAddress() throws Exception {
    QuorumPeer quorumPeer = new QuorumPeer();
    LocalPeerBean remotePeerBean = new LocalPeerBean(quorumPeer);

    /** Case 1: When cnxnFactory is null */
    String result = remotePeerBean.getClientAddress();
    assertNotNull(result);
    assertEquals(0, result.length());

    /** Case 2: When only client port is configured */
    ServerCnxnFactory cnxnFactory = ServerCnxnFactory.createFactory();
    int clientPort = PortAssignment.unique();
    InetSocketAddress address = new InetSocketAddress(clientPort);
    cnxnFactory.configure(address, 5, false);
    quorumPeer.setCnxnFactory(cnxnFactory);

    result = remotePeerBean.getClientAddress();
    String ipv4 = "0.0.0.0:" + clientPort;
    String ipv6 = "0:0:0:0:0:0:0:0:" + clientPort;
    assertTrue(result.equals(ipv4) || result.equals(ipv6));
    // cleanup
    cnxnFactory.shutdown();

    /** Case 3: When both client port and client address is configured */
    clientPort = PortAssignment.unique();
    InetAddress clientIP = InetAddress.getLoopbackAddress();
    address = new InetSocketAddress(clientIP, clientPort);
    cnxnFactory = ServerCnxnFactory.createFactory();
    cnxnFactory.configure(address, 5, false);
    quorumPeer.setCnxnFactory(cnxnFactory);

    result = remotePeerBean.getClientAddress();
    String expectedResult = clientIP.getHostAddress() + ":" + clientPort;
    assertEquals(expectedResult, result);
    // cleanup
    cnxnFactory.shutdown();
  }
Esempio n. 7
0
public class SessionTest extends ZKTestCase implements Watcher {
  protected static final Logger LOG = Logger.getLogger(SessionTest.class);

  private static final String HOSTPORT = "127.0.0.1:" + PortAssignment.unique();
  private NIOServerCnxn.Factory serverFactory;

  private CountDownLatch startSignal;

  File tmpDir;

  private final int TICK_TIME = 3000;

  @Before
  public void setUp() throws Exception {
    if (tmpDir == null) {
      tmpDir = ClientBase.createTmpDir();
    }

    ClientBase.setupTestEnv();
    ZooKeeperServer zs = new ZooKeeperServer(tmpDir, tmpDir, TICK_TIME);

    final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
    serverFactory = new NIOServerCnxn.Factory(new InetSocketAddress(PORT));
    serverFactory.startup(zs);

    Assert.assertTrue(
        "waiting for server up", ClientBase.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT));
  }

  @After
  public void tearDown() throws Exception {
    serverFactory.shutdown();
    Assert.assertTrue(
        "waiting for server down", ClientBase.waitForServerDown(HOSTPORT, CONNECTION_TIMEOUT));
  }

  private static class CountdownWatcher implements Watcher {
    volatile CountDownLatch clientConnected = new CountDownLatch(1);

    public void process(WatchedEvent event) {
      if (event.getState() == KeeperState.SyncConnected) {
        clientConnected.countDown();
      }
    }
  }

  private DisconnectableZooKeeper createClient() throws IOException, InterruptedException {
    CountdownWatcher watcher = new CountdownWatcher();
    return createClient(CONNECTION_TIMEOUT, watcher);
  }

  private DisconnectableZooKeeper createClient(int timeout)
      throws IOException, InterruptedException {
    CountdownWatcher watcher = new CountdownWatcher();
    return createClient(timeout, watcher);
  }

  private DisconnectableZooKeeper createClient(int timeout, CountdownWatcher watcher)
      throws IOException, InterruptedException {
    DisconnectableZooKeeper zk = new DisconnectableZooKeeper(HOSTPORT, timeout, watcher);
    if (!watcher.clientConnected.await(timeout, TimeUnit.MILLISECONDS)) {
      Assert.fail("Unable to connect to server");
    }

    return zk;
  }

  // FIXME this test is Assert.failing due to client close race condition fixing in separate patch
  // for ZOOKEEPER-63
  //    /**
  //     * this test checks to see if the sessionid that was created for the
  //     * first zookeeper client can be reused for the second one immidiately
  //     * after the first client closes and the new client resues them.
  //     * @throws IOException
  //     * @throws InterruptedException
  //     * @throws KeeperException
  //     */
  //    public void testSessionReuse() throws IOException, InterruptedException {
  //        ZooKeeper zk = createClient();
  //
  //        long sessionId = zk.getSessionId();
  //        byte[] passwd = zk.getSessionPasswd();
  //        zk.close();
  //
  //        zk.close();
  //
  //        LOG.info("Closed first session");
  //
  //        startSignal = new CountDownLatch(1);
  //        zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this,
  //                sessionId, passwd);
  //        startSignal.await();
  //
  //        LOG.info("Opened reuse");
  //
  //        Assert.assertEquals(sessionId, zk.getSessionId());
  //
  //        zk.close();
  //    }

  /**
   * This test verifies that when the session id is reused, and the original client is disconnected,
   * but not session closed, that the server will remove ephemeral nodes created by the original
   * session.
   */
  @Test
  public void testSession() throws IOException, InterruptedException, KeeperException {
    DisconnectableZooKeeper zk = createClient();
    zk.create("/e", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
    LOG.info("zk with session id 0x" + Long.toHexString(zk.getSessionId()) + " was destroyed!");

    // disconnect the client by killing the socket, not sending the
    // session disconnect to the server as usual. This allows the test
    // to verify disconnect handling
    zk.disconnect();

    Stat stat = new Stat();
    startSignal = new CountDownLatch(1);
    zk =
        new DisconnectableZooKeeper(
            HOSTPORT, CONNECTION_TIMEOUT, this, zk.getSessionId(), zk.getSessionPasswd());
    startSignal.await();

    LOG.info("zk with session id 0x" + Long.toHexString(zk.getSessionId()) + " was created!");
    zk.getData("/e", false, stat);
    LOG.info("After get data /e");
    zk.close();

    zk = createClient();
    Assert.assertEquals(null, zk.exists("/e", false));
    LOG.info("before close zk with session id 0x" + Long.toHexString(zk.getSessionId()) + "!");
    zk.close();
  }

  /** Make sure ephemerals get cleaned up when a session times out. */
  @Test
  public void testSessionTimeout() throws Exception {
    final int TIMEOUT = 5000;
    DisconnectableZooKeeper zk = createClient(TIMEOUT);
    zk.create("/stest", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
    zk.disconnect();

    Thread.sleep(TIMEOUT * 2);

    zk = createClient(TIMEOUT);
    zk.create("/stest", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
    tearDown();
    zk.close();
    zk.disconnect();
    setUp();

    zk = createClient(TIMEOUT);
    Assert.assertTrue(zk.exists("/stest", false) != null);
    Thread.sleep(TIMEOUT * 2);
    Assert.assertTrue(zk.exists("/stest", false) == null);
    zk.close();
  }

  /**
   * Make sure that we cannot have two connections with the same session id.
   *
   * @throws IOException
   * @throws InterruptedException
   * @throws KeeperException
   */
  @Test
  public void testSessionMove() throws IOException, InterruptedException, KeeperException {
    String hostPorts[] = HOSTPORT.split(",");
    ZooKeeper zk = new DisconnectableZooKeeper(hostPorts[0], CONNECTION_TIMEOUT, this);
    zk.create("/sessionMoveTest", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
    // we want to loop through the list twice
    for (int i = 0; i < hostPorts.length * 2; i++) {
      // This should stomp the zk handle
      ZooKeeper zknew =
          new DisconnectableZooKeeper(
              hostPorts[(i + 1) % hostPorts.length],
              CONNECTION_TIMEOUT,
              this,
              zk.getSessionId(),
              zk.getSessionPasswd());
      zknew.setData("/", new byte[1], -1);
      try {
        zk.setData("/", new byte[1], -1);
        Assert.fail("Should have lost the connection");
      } catch (KeeperException.ConnectionLossException e) {
      }
      // zk.close();
      zk = zknew;
    }
    zk.close();
  }
  /**
   * This test makes sure that duplicate state changes are not communicated to the client watcher.
   * For example we should not notify state as "disconnected" if the watch has already been
   * disconnected. In general we don't consider a dup state notification if the event type is not
   * "None" (ie non-None communicates an event).
   */
  @Test
  public void testSessionStateNoDupStateReporting()
      throws IOException, InterruptedException, KeeperException {
    final int TIMEOUT = 3000;
    DupWatcher watcher = new DupWatcher();
    ZooKeeper zk = createClient(TIMEOUT, watcher);

    // shutdown the server
    serverFactory.shutdown();

    try {
      Thread.sleep(10000);
    } catch (InterruptedException e) {
      // ignore
    }

    // verify that the size is just 2 - ie connect then disconnect
    // if the client attempts reconnect and we are not handling current
    // state correctly (ie eventing on duplicate disconnects) then we'll
    // see a disconnect for each Assert.failed connection attempt
    Assert.assertEquals(2, watcher.states.size());

    zk.close();
  }

  /** Verify access to the negotiated session timeout. */
  @Test
  public void testSessionTimeoutAccess() throws Exception {
    // validate typical case - requested == negotiated
    DisconnectableZooKeeper zk = createClient(TICK_TIME * 4);
    Assert.assertEquals(TICK_TIME * 4, zk.getSessionTimeout());
    // make sure tostring works in both cases
    LOG.info(zk.toString());
    zk.close();
    LOG.info(zk.toString());

    // validate lower limit
    zk = createClient(TICK_TIME);
    Assert.assertEquals(TICK_TIME * 2, zk.getSessionTimeout());
    LOG.info(zk.toString());
    zk.close();
    LOG.info(zk.toString());

    // validate upper limit
    zk = createClient(TICK_TIME * 30);
    Assert.assertEquals(TICK_TIME * 20, zk.getSessionTimeout());
    LOG.info(zk.toString());
    zk.close();
    LOG.info(zk.toString());
  }

  private class DupWatcher extends CountdownWatcher {
    public LinkedList<WatchedEvent> states = new LinkedList<WatchedEvent>();

    public void process(WatchedEvent event) {
      super.process(event);
      if (event.getType() == EventType.None) {
        states.add(event);
      }
    }
  }

  public void process(WatchedEvent event) {
    LOG.info("Event:" + event.getState() + " " + event.getType() + " " + event.getPath());
    if (event.getState() == KeeperState.SyncConnected
        && startSignal != null
        && startSignal.getCount() > 0) {
      startSignal.countDown();
    }
  }

  @Test
  public void testMinMaxSessionTimeout() throws Exception {
    // override the defaults
    final int MINSESS = 20000;
    final int MAXSESS = 240000;
    ZooKeeperServer zs = serverFactory.getZooKeeperServer();
    zs.setMinSessionTimeout(MINSESS);
    zs.setMaxSessionTimeout(MAXSESS);

    // validate typical case - requested == negotiated
    int timeout = 120000;
    DisconnectableZooKeeper zk = createClient(timeout);
    Assert.assertEquals(timeout, zk.getSessionTimeout());
    // make sure tostring works in both cases
    LOG.info(zk.toString());
    zk.close();
    LOG.info(zk.toString());

    // validate lower limit
    zk = createClient(MINSESS / 2);
    Assert.assertEquals(MINSESS, zk.getSessionTimeout());
    LOG.info(zk.toString());
    zk.close();
    LOG.info(zk.toString());

    // validate upper limit
    zk = createClient(MAXSESS * 2);
    Assert.assertEquals(MAXSESS, zk.getSessionTimeout());
    LOG.info(zk.toString());
    zk.close();
    LOG.info(zk.toString());
  }
}
Esempio n. 8
0
public abstract class ClientBase extends ZKTestCase {
  protected static final Logger LOG = LoggerFactory.getLogger(ClientBase.class);

  public static int CONNECTION_TIMEOUT = 30000;
  static final File BASETEST = new File(System.getProperty("build.test.dir", "build"));

  protected int port = PortAssignment.unique();
  protected String hostPort = "127.0.0.1:" + port;
  protected String ipv6HostPort = "[0:0:0:0:0:0:0:1]:" + port;
  protected int maxCnxns = 0;
  protected ServerCnxnFactory serverFactory = null;
  protected File tmpDir = null;

  long initialFdCount;

  public ClientBase() {
    super();
  }

  /**
   * In general don't use this. Only use in the special case that you want to ignore results (for
   * whatever reason) in your test. Don't use empty watchers in real code!
   */
  protected class NullWatcher implements Watcher {
    public void process(WatchedEvent event) {
      /* nada */
    }
  }

  public static class CountdownWatcher implements Watcher {
    // XXX this doesn't need to be volatile! (Should probably be final)
    volatile CountDownLatch clientConnected;
    volatile boolean connected;

    public CountdownWatcher() {
      reset();
    }

    public synchronized void reset() {
      clientConnected = new CountDownLatch(1);
      connected = false;
    }

    public synchronized void process(WatchedEvent event) {
      if (event.getState() == KeeperState.SyncConnected
          || event.getState() == KeeperState.ConnectedReadOnly) {
        connected = true;
        notifyAll();
        clientConnected.countDown();
      } else {
        connected = false;
        notifyAll();
      }
    }

    public synchronized boolean isConnected() {
      return connected;
    }

    public synchronized void waitForConnected(long timeout)
        throws InterruptedException, TimeoutException {
      long expire = System.currentTimeMillis() + timeout;
      long left = timeout;
      while (!connected && left > 0) {
        wait(left);
        left = expire - System.currentTimeMillis();
      }
      if (!connected) {
        throw new TimeoutException("Did not connect");
      }
    }

    public synchronized void waitForDisconnected(long timeout)
        throws InterruptedException, TimeoutException {
      long expire = System.currentTimeMillis() + timeout;
      long left = timeout;
      while (connected && left > 0) {
        wait(left);
        left = expire - System.currentTimeMillis();
      }
      if (connected) {
        throw new TimeoutException("Did not disconnect");
      }
    }
  }

  protected TestableZooKeeper createClient() throws IOException, InterruptedException {
    return createClient(hostPort);
  }

  protected TestableZooKeeper createClient(String hp) throws IOException, InterruptedException {
    CountdownWatcher watcher = new CountdownWatcher();
    return createClient(watcher, hp);
  }

  protected TestableZooKeeper createClient(CountdownWatcher watcher)
      throws IOException, InterruptedException {
    return createClient(watcher, hostPort);
  }

  private LinkedList<ZooKeeper> allClients;
  private boolean allClientsSetup = false;

  protected TestableZooKeeper createClient(CountdownWatcher watcher, String hp)
      throws IOException, InterruptedException {
    return createClient(watcher, hp, CONNECTION_TIMEOUT);
  }

  protected TestableZooKeeper createClient(CountdownWatcher watcher, String hp, int timeout)
      throws IOException, InterruptedException {
    watcher.reset();
    TestableZooKeeper zk = new TestableZooKeeper(hp, timeout, watcher);
    if (!watcher.clientConnected.await(timeout, TimeUnit.MILLISECONDS)) {
      Assert.fail("Unable to connect to server");
    }
    synchronized (this) {
      if (!allClientsSetup) {
        LOG.error("allClients never setup");
        Assert.fail("allClients never setup");
      }
      if (allClients != null) {
        allClients.add(zk);
        JMXEnv.ensureAll(getHexSessionId(zk.getSessionId()));
      } else {
        // test done - close the zk, not needed
        zk.close();
      }
    }

    return zk;
  }

  public static class HostPort {
    String host;
    int port;

    public HostPort(String host, int port) {
      this.host = host;
      this.port = port;
    }
  }

  public static List<HostPort> parseHostPortList(String hplist) {
    ArrayList<HostPort> alist = new ArrayList<HostPort>();
    for (String hp : hplist.split(",")) {
      int idx = hp.lastIndexOf(':');
      String host = hp.substring(0, idx);
      int port;
      try {
        port = Integer.parseInt(hp.substring(idx + 1));
      } catch (RuntimeException e) {
        throw new RuntimeException("Problem parsing " + hp + e.toString());
      }
      alist.add(new HostPort(host, port));
    }
    return alist;
  }

  public static boolean waitForServerUp(String hp, long timeout) {
    long start = System.currentTimeMillis();
    while (true) {
      try {
        // if there are multiple hostports, just take the first one
        HostPort hpobj = parseHostPortList(hp).get(0);
        String result = send4LetterWord(hpobj.host, hpobj.port, "stat");
        if (result.startsWith("Zookeeper version:") && !result.contains("READ-ONLY")) {
          return true;
        }
      } catch (IOException e) {
        // ignore as this is expected
        LOG.info("server " + hp + " not up " + e);
      }

      if (System.currentTimeMillis() > start + timeout) {
        break;
      }
      try {
        Thread.sleep(250);
      } catch (InterruptedException e) {
        // ignore
      }
    }
    return false;
  }

  public static boolean waitForServerDown(String hp, long timeout) {
    long start = System.currentTimeMillis();
    while (true) {
      try {
        HostPort hpobj = parseHostPortList(hp).get(0);
        send4LetterWord(hpobj.host, hpobj.port, "stat");
      } catch (IOException e) {
        return true;
      }

      if (System.currentTimeMillis() > start + timeout) {
        break;
      }
      try {
        Thread.sleep(250);
      } catch (InterruptedException e) {
        // ignore
      }
    }
    return false;
  }

  public static boolean waitForServerState(QuorumPeer qp, int timeout, String serverState) {
    long start = System.currentTimeMillis();
    while (true) {
      try {
        Thread.sleep(250);
      } catch (InterruptedException e) {
        // ignore
      }
      if (qp.getServerState().equals(serverState)) return true;
      if (System.currentTimeMillis() > start + timeout) {
        return false;
      }
    }
  }

  static void verifyThreadTerminated(Thread thread, long millis) throws InterruptedException {
    thread.join(millis);
    if (thread.isAlive()) {
      LOG.error("Thread " + thread.getName() + " : " + Arrays.toString(thread.getStackTrace()));
      Assert.assertFalse("thread " + thread.getName() + " still alive after join", true);
    }
  }

  public static File createTmpDir() throws IOException {
    return createTmpDir(BASETEST);
  }

  static File createTmpDir(File parentDir) throws IOException {
    File tmpFile = File.createTempFile("test", ".junit", parentDir);
    // don't delete tmpFile - this ensures we don't attempt to create
    // a tmpDir with a duplicate name
    File tmpDir = new File(tmpFile + ".dir");
    Assert.assertFalse(tmpDir.exists()); // never true if tmpfile does it's job
    Assert.assertTrue(tmpDir.mkdirs());

    return tmpDir;
  }

  private static int getPort(String hostPort) {
    String[] split = hostPort.split(":");
    String portstr = split[split.length - 1];
    String[] pc = portstr.split("/");
    if (pc.length > 1) {
      portstr = pc[0];
    }
    return Integer.parseInt(portstr);
  }

  /** Starting the given server instance */
  public static void startServerInstance(File dataDir, ServerCnxnFactory factory, String hostPort)
      throws IOException, InterruptedException {
    final int port = getPort(hostPort);
    LOG.info("STARTING server instance 127.0.0.1:{}", port);
    ZooKeeperServer zks = new ZooKeeperServer(dataDir, dataDir, 3000);
    factory.startup(zks);
    Assert.assertTrue(
        "waiting for server up",
        ClientBase.waitForServerUp("127.0.0.1:" + port, CONNECTION_TIMEOUT));
  }

  /**
   * This method instantiates a new server. Starting of the server instance has been moved to a
   * separate method {@link ClientBase#startServerInstance(File, ServerCnxnFactory, String)}.
   * Because any exception on starting the server would leave the server running and the caller
   * would not be able to shutdown the instance. This may affect other test cases.
   *
   * @return newly created server instance
   * @see <a href="https://issues.apache.org/jira/browse/ZOOKEEPER-1852">ZOOKEEPER-1852</a> for more
   *     information.
   */
  public static ServerCnxnFactory createNewServerInstance(
      ServerCnxnFactory factory, String hostPort, int maxCnxns)
      throws IOException, InterruptedException {
    final int port = getPort(hostPort);
    LOG.info("CREATING server instance 127.0.0.1:{}", port);
    if (factory == null) {
      factory = ServerCnxnFactory.createFactory(port, maxCnxns);
    }
    return factory;
  }

  static void shutdownServerInstance(ServerCnxnFactory factory, String hostPort) {
    if (factory != null) {
      ZKDatabase zkDb = null;
      {
        ZooKeeperServer zs = getServer(factory);
        if (zs != null) {
          zkDb = zs.getZKDatabase();
        }
      }
      factory.shutdown();
      try {
        if (zkDb != null) {
          zkDb.close();
        }
      } catch (IOException ie) {
        LOG.warn("Error closing logs ", ie);
      }
      final int PORT = getPort(hostPort);

      Assert.assertTrue(
          "waiting for server down",
          ClientBase.waitForServerDown("127.0.0.1:" + PORT, CONNECTION_TIMEOUT));
    }
  }

  /** Test specific setup */
  public static void setupTestEnv() {
    // during the tests we run with 100K prealloc in the logs.
    // on windows systems prealloc of 64M was seen to take ~15seconds
    // resulting in test Assert.failure (client timeout on first session).
    // set env and directly in order to handle static init/gc issues
    System.setProperty("zookeeper.preAllocSize", "100");
    FileTxnLog.setPreallocSize(100 * 1024);
  }

  protected void setUpAll() throws Exception {
    allClients = new LinkedList<ZooKeeper>();
    allClientsSetup = true;
  }

  @Before
  public void setUp() throws Exception {
    /* some useful information - log the number of fds used before
     * and after a test is run. Helps to verify we are freeing resources
     * correctly. Unfortunately this only works on unix systems (the
     * only place sun has implemented as part of the mgmt bean api.
     */
    OSMXBean osMbean = new OSMXBean();
    if (osMbean.getUnix() == true) {
      initialFdCount = osMbean.getOpenFileDescriptorCount();
      LOG.info("Initial fdcount is: " + initialFdCount);
    }

    setupTestEnv();

    JMXEnv.setUp();

    setUpAll();

    tmpDir = createTmpDir(BASETEST);

    startServer();

    LOG.info("Client test setup finished");
  }

  protected void startServer() throws Exception {
    LOG.info("STARTING server");
    serverFactory = createNewServerInstance(serverFactory, hostPort, maxCnxns);
    startServerInstance(tmpDir, serverFactory, hostPort);
    // ensure that server and data bean are registered
    Set<ObjectName> children = JMXEnv.ensureParent("InMemoryDataTree", "StandaloneServer_port");
    // Remove beans which are related to zk client sessions. Strong
    // assertions cannot be done for these client sessions because
    // registeration of these beans with server will happen only on their
    // respective reconnection interval
    verifyUnexpectedBeans(children);
  }

  private void verifyUnexpectedBeans(Set<ObjectName> children) {
    if (allClients != null) {
      for (ZooKeeper zkc : allClients) {
        Iterator<ObjectName> childItr = children.iterator();
        while (childItr.hasNext()) {
          ObjectName clientBean = childItr.next();
          if (clientBean.toString().contains(getHexSessionId(zkc.getSessionId()))) {
            LOG.info("found name:" + zkc.getSessionId() + " client bean:" + clientBean.toString());
            childItr.remove();
          }
        }
      }
    }
    for (ObjectName bean : children) {
      LOG.info("unexpected:" + bean.toString());
    }
    TestCase.assertEquals("Unexpected bean exists!", 0, children.size());
  }

  /**
   * Returns a string representation of the given long value session id
   *
   * @param sessionId long value of session id
   * @return string representation of session id
   */
  protected static String getHexSessionId(long sessionId) {
    return "0x" + Long.toHexString(sessionId);
  }

  protected void stopServer() throws Exception {
    LOG.info("STOPPING server");
    shutdownServerInstance(serverFactory, hostPort);
    serverFactory = null;
    // ensure no beans are leftover
    JMXEnv.ensureOnly();
  }

  protected static ZooKeeperServer getServer(ServerCnxnFactory fac) {
    ZooKeeperServer zs = ServerCnxnFactoryAccessor.getZkServer(fac);

    return zs;
  }

  protected void tearDownAll() throws Exception {
    synchronized (this) {
      if (allClients != null)
        for (ZooKeeper zk : allClients) {
          try {
            if (zk != null) zk.close();
          } catch (InterruptedException e) {
            LOG.warn("ignoring interrupt", e);
          }
        }
      allClients = null;
    }
  }

  @After
  public void tearDown() throws Exception {
    LOG.info("tearDown starting");

    tearDownAll();

    stopServer();

    if (tmpDir != null) {
      Assert.assertTrue("delete " + tmpDir.toString(), recursiveDelete(tmpDir));
    }

    // This has to be set to null when the same instance of this class is reused between test cases
    serverFactory = null;

    JMXEnv.tearDown();

    /* some useful information - log the number of fds used before
     * and after a test is run. Helps to verify we are freeing resources
     * correctly. Unfortunately this only works on unix systems (the
     * only place sun has implemented as part of the mgmt bean api.
     */
    OSMXBean osMbean = new OSMXBean();
    if (osMbean.getUnix() == true) {
      long fdCount = osMbean.getOpenFileDescriptorCount();
      String message = "fdcount after test is: " + fdCount + " at start it was " + initialFdCount;
      LOG.info(message);
      if (fdCount > initialFdCount) {
        LOG.info("sleeping for 20 secs");
        // Thread.sleep(60000);
        // assertTrue(message, fdCount <= initialFdCount);
      }
    }
  }

  public static MBeanServerConnection jmxConn() throws IOException {
    return JMXEnv.conn();
  }

  public static boolean recursiveDelete(File d) {
    if (d.isDirectory()) {
      File children[] = d.listFiles();
      for (File f : children) {
        Assert.assertTrue("delete " + f.toString(), recursiveDelete(f));
      }
    }
    return d.delete();
  }

  public static void logAllStackTraces() {
    StringBuilder sb = new StringBuilder();
    sb.append("Starting logAllStackTraces()\n");
    Map<Thread, StackTraceElement[]> threads = Thread.getAllStackTraces();
    for (Entry<Thread, StackTraceElement[]> e : threads.entrySet()) {
      sb.append("Thread " + e.getKey().getName() + "\n");
      for (StackTraceElement elem : e.getValue()) {
        sb.append("\tat " + elem + "\n");
      }
    }
    sb.append("Ending logAllStackTraces()\n");
    LOG.error(sb.toString());
  }

  /*
   * Verify that all of the servers see the same number of nodes
   * at the root
   */
  void verifyRootOfAllServersMatch(String hostPort)
      throws InterruptedException, KeeperException, IOException {
    String parts[] = hostPort.split(",");

    // run through till the counts no longer change on each server
    // max 15 tries, with 2 second sleeps, so approx 30 seconds
    int[] counts = new int[parts.length];
    int failed = 0;
    for (int j = 0; j < 100; j++) {
      int newcounts[] = new int[parts.length];
      int i = 0;
      for (String hp : parts) {
        try {
          ZooKeeper zk = createClient(hp);

          try {
            newcounts[i++] = zk.getChildren("/", false).size();
          } finally {
            zk.close();
          }
        } catch (Throwable t) {
          failed++;
          // if session creation Assert.fails dump the thread stack
          // and try the next server
          logAllStackTraces();
        }
      }

      if (Arrays.equals(newcounts, counts)) {
        LOG.info("Found match with array:" + Arrays.toString(newcounts));
        counts = newcounts;
        break;
      } else {
        counts = newcounts;
        Thread.sleep(10000);
      }

      // don't keep this up too long, will Assert.assert false below
      if (failed > 10) {
        break;
      }
    }

    // verify all the servers reporting same number of nodes
    String logmsg = "node count not consistent{} {}";
    for (int i = 1; i < parts.length; i++) {
      if (counts[i - 1] != counts[i]) {
        LOG.error(logmsg, Integer.valueOf(counts[i - 1]), Integer.valueOf(counts[i]));
      } else {
        LOG.info(logmsg, Integer.valueOf(counts[i - 1]), Integer.valueOf(counts[i]));
      }
    }
  }

  public static String readFile(File file) throws IOException {
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    BufferedInputStream is = new BufferedInputStream(new FileInputStream(file));
    try {
      IOUtils.copyBytes(is, os, 1024, true);
    } finally {
      is.close();
    }
    return os.toString();
  }

  public static String join(String separator, Object[] parts) {
    StringBuilder sb = new StringBuilder();
    boolean first = true;
    for (Object part : parts) {
      if (!first) {
        sb.append(separator);
        first = false;
      }
      sb.append(part);
    }
    return sb.toString();
  }
}
  @Test
  public void testPortChange() throws Exception {
    qu = new QuorumUtil(1); // create 3 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    List<String> joiningServers = new ArrayList<String>();

    int leaderIndex = getLeaderId(qu);
    int followerIndex = leaderIndex == 1 ? 2 : 1;

    // modify follower's client port

    int quorumPort = qu.getPeer(followerIndex).peer.getQuorumAddress().getPort();
    int electionPort = qu.getPeer(followerIndex).peer.getElectionAddress().getPort();
    int oldClientPort = qu.getPeer(followerIndex).peer.getClientPort();
    int newClientPort = PortAssignment.unique();
    joiningServers.add(
        "server."
            + followerIndex
            + "=localhost:"
            + quorumPort
            + ":"
            + electionPort
            + ":participant;localhost:"
            + newClientPort);

    // create a /test znode and check that read/write works before
    // any reconfig is invoked
    testNormalOperation(zkArr[followerIndex], zkArr[leaderIndex]);

    reconfig(zkArr[followerIndex], joiningServers, null, null, -1);

    try {
      for (int i = 0; i < 20; i++) {
        Thread.sleep(1000);
        zkArr[followerIndex].setData("/test", "teststr".getBytes(), -1);
      }
    } catch (KeeperException.ConnectionLossException e) {
      Assert.fail("Existing client disconnected when client port changed!");
    }

    zkArr[followerIndex].close();
    zkArr[followerIndex] =
        new ZooKeeper(
            "127.0.0.1:" + oldClientPort,
            ClientBase.CONNECTION_TIMEOUT,
            new Watcher() {
              public void process(WatchedEvent event) {}
            });
    for (int i = 0; i < 10; i++) {
      try {
        Thread.sleep(1000);
        zkArr[followerIndex].setData("/test", "teststr".getBytes(), -1);
        Assert.fail("New client connected to old client port!");
      } catch (KeeperException.ConnectionLossException e) {
      }
    }

    zkArr[followerIndex].close();
    zkArr[followerIndex] =
        new ZooKeeper(
            "127.0.0.1:" + newClientPort,
            ClientBase.CONNECTION_TIMEOUT,
            new Watcher() {
              public void process(WatchedEvent event) {}
            });

    testNormalOperation(zkArr[followerIndex], zkArr[leaderIndex]);
    testServerHasConfig(zkArr[followerIndex], joiningServers, null);
    Assert.assertEquals(newClientPort, qu.getPeer(followerIndex).peer.getClientPort());

    joiningServers.clear();

    // change leader's leading port - should renounce leadership

    int newQuorumPort = PortAssignment.unique();
    joiningServers.add(
        "server."
            + leaderIndex
            + "=localhost:"
            + newQuorumPort
            + ":"
            + qu.getPeer(leaderIndex).peer.getElectionAddress().getPort()
            + ":participant;localhost:"
            + qu.getPeer(leaderIndex).peer.getClientPort());

    reconfig(zkArr[leaderIndex], joiningServers, null, null, -1);

    testNormalOperation(zkArr[followerIndex], zkArr[leaderIndex]);

    Assert.assertTrue(qu.getPeer(leaderIndex).peer.getQuorumAddress().getPort() == newQuorumPort);
    Assert.assertTrue(getLeaderId(qu) != leaderIndex); // the leader changed

    joiningServers.clear();

    // change everyone's leader election port

    for (int i = 1; i <= 3; i++) {
      joiningServers.add(
          "server."
              + i
              + "=localhost:"
              + qu.getPeer(i).peer.getQuorumAddress().getPort()
              + ":"
              + PortAssignment.unique()
              + ":participant;localhost:"
              + qu.getPeer(i).peer.getClientPort());
    }

    reconfig(zkArr[1], joiningServers, null, null, -1);

    leaderIndex = getLeaderId(qu);
    int follower1 = leaderIndex == 1 ? 2 : 1;
    int follower2 = 1;
    while (follower2 == leaderIndex || follower2 == follower1) follower2++;

    // lets kill the leader and see if a new one is elected

    qu.shutdown(getLeaderId(qu));

    testNormalOperation(zkArr[follower2], zkArr[follower1]);
    testServerHasConfig(zkArr[follower1], joiningServers, null);
    testServerHasConfig(zkArr[follower2], joiningServers, null);

    closeAllHandles(zkArr);
  }