@Test
 public void testInitialConfigHasPositiveVersion() throws Exception {
   qu = new QuorumUtil(1); // create 3 servers
   qu.disableJMXTest = true;
   qu.startAll();
   ZooKeeper[] zkArr = createHandles(qu);
   testNormalOperation(zkArr[1], zkArr[2]);
   for (int i = 1; i < 4; i++) {
     String configStr = testServerHasConfig(zkArr[i], null, null);
     QuorumVerifier qv = qu.getPeer(i).peer.configFromString(configStr);
     long version = qv.getVersion();
     Assert.assertTrue(version == 0x100000000L);
   }
 }
 public static ZooKeeper[] createHandles(QuorumUtil qu) throws IOException {
   // create an extra handle, so we can index the handles from 1 to qu.ALL
   // using the server id.
   ZooKeeper[] zkArr = new ZooKeeper[qu.ALL + 1];
   zkArr[0] = null; // not used.
   for (int i = 1; i <= qu.ALL; i++) {
     // server ids are 1, 2 and 3
     zkArr[i] =
         new ZooKeeper(
             "127.0.0.1:" + qu.getPeer(i).peer.getClientPort(),
             ClientBase.CONNECTION_TIMEOUT,
             new Watcher() {
               public void process(WatchedEvent event) {}
             });
   }
   return zkArr;
 }
  @Test
  public void testBulkReconfig() throws Exception {
    qu = new QuorumUtil(3); // create 7 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    // new config will have three of the servers as followers
    // two of the servers as observers, and all ports different
    ArrayList<String> newServers = new ArrayList<String>();
    for (int i = 1; i <= 5; i++) {
      String server =
          "server."
              + i
              + "=localhost:"
              + PortAssignment.unique()
              + ":"
              + PortAssignment.unique()
              + ":"
              + ((i == 4 || i == 5) ? "observer" : "participant")
              + ";localhost:"
              + qu.getPeer(i).peer.getClientPort();
      newServers.add(server);
    }

    qu.shutdown(3);
    qu.shutdown(6);
    qu.shutdown(7);

    reconfig(zkArr[1], null, null, newServers, -1);
    testNormalOperation(zkArr[1], zkArr[2]);

    testServerHasConfig(zkArr[1], newServers, null);
    testServerHasConfig(zkArr[2], newServers, null);
    testServerHasConfig(zkArr[4], newServers, null);
    testServerHasConfig(zkArr[5], newServers, null);

    qu.shutdown(5);
    qu.shutdown(4);

    testNormalOperation(zkArr[1], zkArr[2]);

    closeAllHandles(zkArr);
  }
  /**
   * Tests verifies the jmx attributes of local and remote peer bean - change participant to
   * observer role
   */
  @Test
  public void testJMXBeanAfterRoleChange() throws Exception {
    qu = new QuorumUtil(1); // create 3 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    // changing a server's role / port is done by "adding" it with the same
    // id but different role / port
    List<String> joiningServers = new ArrayList<String>();

    // assert remotePeerBean.1 of ReplicatedServer_2
    int changingIndex = 1;
    int replica2 = 2;
    QuorumPeer peer2 = qu.getPeer(replica2).peer;
    QuorumServer changingQS2 = peer2.getView().get(new Long(changingIndex));
    String remotePeerBean2 =
        CommonNames.DOMAIN
            + ":name0=ReplicatedServer_id"
            + replica2
            + ",name1=replica."
            + changingIndex;
    assertRemotePeerMXBeanAttributes(changingQS2, remotePeerBean2);

    // assert remotePeerBean.1 of ReplicatedServer_3
    int replica3 = 3;
    QuorumPeer peer3 = qu.getPeer(replica3).peer;
    QuorumServer changingQS3 = peer3.getView().get(new Long(changingIndex));
    String remotePeerBean3 =
        CommonNames.DOMAIN
            + ":name0=ReplicatedServer_id"
            + replica3
            + ",name1=replica."
            + changingIndex;
    assertRemotePeerMXBeanAttributes(changingQS3, remotePeerBean3);

    String newRole = "observer";

    ZooKeeper zk = zkArr[changingIndex];

    // exactly as it is now, except for role change
    joiningServers.add(
        "server."
            + changingIndex
            + "=127.0.0.1:"
            + qu.getPeer(changingIndex).peer.getQuorumAddress().getPort()
            + ":"
            + qu.getPeer(changingIndex).peer.getElectionAddress().getPort()
            + ":"
            + newRole
            + ";127.0.0.1:"
            + qu.getPeer(changingIndex).peer.getClientPort());

    reconfig(zk, joiningServers, null, null, -1);
    testNormalOperation(zkArr[changingIndex], zk);

    Assert.assertTrue(
        qu.getPeer(changingIndex).peer.observer != null
            && qu.getPeer(changingIndex).peer.follower == null
            && qu.getPeer(changingIndex).peer.leader == null);
    Assert.assertTrue(qu.getPeer(changingIndex).peer.getPeerState() == ServerState.OBSERVING);

    QuorumPeer qp = qu.getPeer(changingIndex).peer;
    String localPeerBeanName =
        CommonNames.DOMAIN
            + ":name0=ReplicatedServer_id"
            + changingIndex
            + ",name1=replica."
            + changingIndex;

    // localPeerBean.1 of ReplicatedServer_1
    assertLocalPeerMXBeanAttributes(qp, localPeerBeanName, true);

    // assert remotePeerBean.1 of ReplicatedServer_2
    changingQS2 = peer2.getView().get(new Long(changingIndex));
    assertRemotePeerMXBeanAttributes(changingQS2, remotePeerBean2);

    // assert remotePeerBean.1 of ReplicatedServer_3
    changingQS3 = peer3.getView().get(new Long(changingIndex));
    assertRemotePeerMXBeanAttributes(changingQS3, remotePeerBean3);

    closeAllHandles(zkArr);
  }
  /**
   * Tests verifies the jmx attributes of local and remote peer bean - remove one quorum peer and
   * again adding it back
   */
  @Test
  public void testJMXBeanAfterRemoveAddOne() throws Exception {
    qu = new QuorumUtil(1); // create 3 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    List<String> leavingServers = new ArrayList<String>();
    List<String> joiningServers = new ArrayList<String>();

    // assert remotePeerBean.1 of ReplicatedServer_2
    int leavingIndex = 1;
    int replica2 = 2;
    QuorumPeer peer2 = qu.getPeer(replica2).peer;
    QuorumServer leavingQS2 = peer2.getView().get(new Long(leavingIndex));
    String remotePeerBean2 =
        CommonNames.DOMAIN
            + ":name0=ReplicatedServer_id"
            + replica2
            + ",name1=replica."
            + leavingIndex;
    assertRemotePeerMXBeanAttributes(leavingQS2, remotePeerBean2);

    // assert remotePeerBean.1 of ReplicatedServer_3
    int replica3 = 3;
    QuorumPeer peer3 = qu.getPeer(replica3).peer;
    QuorumServer leavingQS3 = peer3.getView().get(new Long(leavingIndex));
    String remotePeerBean3 =
        CommonNames.DOMAIN
            + ":name0=ReplicatedServer_id"
            + replica3
            + ",name1=replica."
            + leavingIndex;
    assertRemotePeerMXBeanAttributes(leavingQS3, remotePeerBean3);

    ZooKeeper zk = zkArr[leavingIndex];

    leavingServers.add(Integer.toString(leavingIndex));

    // remember this server so we can add it back later
    joiningServers.add(
        "server."
            + leavingIndex
            + "=127.0.0.1:"
            + qu.getPeer(leavingIndex).peer.getQuorumAddress().getPort()
            + ":"
            + qu.getPeer(leavingIndex).peer.getElectionAddress().getPort()
            + ":participant;127.0.0.1:"
            + qu.getPeer(leavingIndex).peer.getClientPort());

    // Remove ReplicatedServer_1 from the ensemble
    reconfig(zk, null, leavingServers, null, -1);

    // localPeerBean.1 of ReplicatedServer_1
    QuorumPeer removedPeer = qu.getPeer(leavingIndex).peer;
    String localPeerBean =
        CommonNames.DOMAIN
            + ":name0=ReplicatedServer_id"
            + leavingIndex
            + ",name1=replica."
            + leavingIndex;
    assertLocalPeerMXBeanAttributes(removedPeer, localPeerBean, false);

    // remotePeerBean.1 shouldn't exists in ReplicatedServer_2
    JMXEnv.ensureNone(remotePeerBean2);
    // remotePeerBean.1 shouldn't exists in ReplicatedServer_3
    JMXEnv.ensureNone(remotePeerBean3);

    // Add ReplicatedServer_1 back to the ensemble
    reconfig(zk, joiningServers, null, null, -1);

    // localPeerBean.1 of ReplicatedServer_1
    assertLocalPeerMXBeanAttributes(removedPeer, localPeerBean, true);

    // assert remotePeerBean.1 of ReplicatedServer_2
    leavingQS2 = peer2.getView().get(new Long(leavingIndex));
    assertRemotePeerMXBeanAttributes(leavingQS2, remotePeerBean2);

    // assert remotePeerBean.1 of ReplicatedServer_3
    leavingQS3 = peer3.getView().get(new Long(leavingIndex));
    assertRemotePeerMXBeanAttributes(leavingQS3, remotePeerBean3);

    closeAllHandles(zkArr);
  }
  @Test
  public void testQuorumSystemChange() throws Exception {
    qu = new QuorumUtil(3); // create 7 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    ArrayList<String> members = new ArrayList<String>();
    members.add("group.1=3:4:5");
    members.add("group.2=1:2");
    members.add("weight.1=0");
    members.add("weight.2=0");
    members.add("weight.3=1");
    members.add("weight.4=1");
    members.add("weight.5=1");

    for (int i = 1; i <= 5; i++) {
      members.add(
          "server."
              + i
              + "=127.0.0.1:"
              + qu.getPeer(i).peer.getQuorumAddress().getPort()
              + ":"
              + qu.getPeer(i).peer.getElectionAddress().getPort()
              + ";"
              + "127.0.0.1:"
              + qu.getPeer(i).peer.getClientPort());
    }

    reconfig(zkArr[1], null, null, members, -1);

    // this should flush the config to servers 2, 3, 4 and 5
    testNormalOperation(zkArr[2], zkArr[3]);
    testNormalOperation(zkArr[4], zkArr[5]);

    for (int i = 1; i <= 5; i++) {
      if (!(qu.getPeer(i).peer.quorumVerifier instanceof QuorumHierarchical))
        Assert.fail("peer " + i + " doesn't think the quorum system is Hieararchical!");
    }

    qu.shutdown(1);
    qu.shutdown(2);
    qu.shutdown(3);
    qu.shutdown(7);
    qu.shutdown(6);

    // servers 4 and 5 should be able to work independently
    testNormalOperation(zkArr[4], zkArr[5]);

    qu.restart(1);
    qu.restart(2);

    members.clear();
    for (int i = 1; i <= 3; i++) {
      members.add(
          "server."
              + i
              + "=127.0.0.1:"
              + qu.getPeer(i).peer.getQuorumAddress().getPort()
              + ":"
              + qu.getPeer(i).peer.getElectionAddress().getPort()
              + ";"
              + "127.0.0.1:"
              + qu.getPeer(i).peer.getClientPort());
    }

    reconfig(zkArr[1], null, null, members, -1);

    // flush the config to server 2
    testNormalOperation(zkArr[1], zkArr[2]);

    qu.shutdown(4);
    qu.shutdown(5);

    // servers 1 and 2 should be able to work independently
    testNormalOperation(zkArr[1], zkArr[2]);

    for (int i = 1; i <= 2; i++) {
      if (!(qu.getPeer(i).peer.quorumVerifier instanceof QuorumMaj))
        Assert.fail("peer " + i + " doesn't think the quorum system is a majority quorum system!");
    }

    closeAllHandles(zkArr);
  }
  @Test
  public void testPortChange() throws Exception {
    qu = new QuorumUtil(1); // create 3 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    List<String> joiningServers = new ArrayList<String>();

    int leaderIndex = getLeaderId(qu);
    int followerIndex = leaderIndex == 1 ? 2 : 1;

    // modify follower's client port

    int quorumPort = qu.getPeer(followerIndex).peer.getQuorumAddress().getPort();
    int electionPort = qu.getPeer(followerIndex).peer.getElectionAddress().getPort();
    int oldClientPort = qu.getPeer(followerIndex).peer.getClientPort();
    int newClientPort = PortAssignment.unique();
    joiningServers.add(
        "server."
            + followerIndex
            + "=localhost:"
            + quorumPort
            + ":"
            + electionPort
            + ":participant;localhost:"
            + newClientPort);

    // create a /test znode and check that read/write works before
    // any reconfig is invoked
    testNormalOperation(zkArr[followerIndex], zkArr[leaderIndex]);

    reconfig(zkArr[followerIndex], joiningServers, null, null, -1);

    try {
      for (int i = 0; i < 20; i++) {
        Thread.sleep(1000);
        zkArr[followerIndex].setData("/test", "teststr".getBytes(), -1);
      }
    } catch (KeeperException.ConnectionLossException e) {
      Assert.fail("Existing client disconnected when client port changed!");
    }

    zkArr[followerIndex].close();
    zkArr[followerIndex] =
        new ZooKeeper(
            "127.0.0.1:" + oldClientPort,
            ClientBase.CONNECTION_TIMEOUT,
            new Watcher() {
              public void process(WatchedEvent event) {}
            });
    for (int i = 0; i < 10; i++) {
      try {
        Thread.sleep(1000);
        zkArr[followerIndex].setData("/test", "teststr".getBytes(), -1);
        Assert.fail("New client connected to old client port!");
      } catch (KeeperException.ConnectionLossException e) {
      }
    }

    zkArr[followerIndex].close();
    zkArr[followerIndex] =
        new ZooKeeper(
            "127.0.0.1:" + newClientPort,
            ClientBase.CONNECTION_TIMEOUT,
            new Watcher() {
              public void process(WatchedEvent event) {}
            });

    testNormalOperation(zkArr[followerIndex], zkArr[leaderIndex]);
    testServerHasConfig(zkArr[followerIndex], joiningServers, null);
    Assert.assertEquals(newClientPort, qu.getPeer(followerIndex).peer.getClientPort());

    joiningServers.clear();

    // change leader's leading port - should renounce leadership

    int newQuorumPort = PortAssignment.unique();
    joiningServers.add(
        "server."
            + leaderIndex
            + "=localhost:"
            + newQuorumPort
            + ":"
            + qu.getPeer(leaderIndex).peer.getElectionAddress().getPort()
            + ":participant;localhost:"
            + qu.getPeer(leaderIndex).peer.getClientPort());

    reconfig(zkArr[leaderIndex], joiningServers, null, null, -1);

    testNormalOperation(zkArr[followerIndex], zkArr[leaderIndex]);

    Assert.assertTrue(qu.getPeer(leaderIndex).peer.getQuorumAddress().getPort() == newQuorumPort);
    Assert.assertTrue(getLeaderId(qu) != leaderIndex); // the leader changed

    joiningServers.clear();

    // change everyone's leader election port

    for (int i = 1; i <= 3; i++) {
      joiningServers.add(
          "server."
              + i
              + "=localhost:"
              + qu.getPeer(i).peer.getQuorumAddress().getPort()
              + ":"
              + PortAssignment.unique()
              + ":participant;localhost:"
              + qu.getPeer(i).peer.getClientPort());
    }

    reconfig(zkArr[1], joiningServers, null, null, -1);

    leaderIndex = getLeaderId(qu);
    int follower1 = leaderIndex == 1 ? 2 : 1;
    int follower2 = 1;
    while (follower2 == leaderIndex || follower2 == follower1) follower2++;

    // lets kill the leader and see if a new one is elected

    qu.shutdown(getLeaderId(qu));

    testNormalOperation(zkArr[follower2], zkArr[follower1]);
    testServerHasConfig(zkArr[follower1], joiningServers, null);
    testServerHasConfig(zkArr[follower2], joiningServers, null);

    closeAllHandles(zkArr);
  }
  @Test
  public void testRoleChange() throws Exception {
    qu = new QuorumUtil(1); // create 3 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    // changing a server's role / port is done by "adding" it with the same
    // id but different role / port
    List<String> joiningServers = new ArrayList<String>();

    int leaderIndex = getLeaderId(qu);

    // during first and second iteration, leavingIndex will correspond to a
    // follower
    // during third and fouth iteration leavingIndex will be the index of
    // the leader
    int changingIndex = (leaderIndex == 1) ? 2 : 1;

    // first convert participant to observer, then observer to participant,
    // and so on
    String newRole = "observer";

    for (int i = 0; i < 4; i++) {
      // some of the operations will be executed by a client connected to
      // the removed server
      // while others are invoked by a client connected to some other
      // server.
      // when we're removing the leader, zk1 will be the client connected
      // to removed server
      ZooKeeper zk1 =
          (changingIndex == leaderIndex) ? zkArr[leaderIndex] : zkArr[(leaderIndex % qu.ALL) + 1];

      // exactly as it is now, except for role change
      joiningServers.add(
          "server."
              + changingIndex
              + "=localhost:"
              + qu.getPeer(changingIndex).peer.getQuorumAddress().getPort()
              + ":"
              + qu.getPeer(changingIndex).peer.getElectionAddress().getPort()
              + ":"
              + newRole
              + ";localhost:"
              + qu.getPeer(changingIndex).peer.getClientPort());

      reconfig(zk1, joiningServers, null, null, -1);
      testNormalOperation(zkArr[changingIndex], zk1);

      if (newRole.equals("observer")) {
        Assert.assertTrue(
            qu.getPeer(changingIndex).peer.observer != null
                && qu.getPeer(changingIndex).peer.follower == null
                && qu.getPeer(changingIndex).peer.leader == null);
        Assert.assertTrue(qu.getPeer(changingIndex).peer.getPeerState() == ServerState.OBSERVING);
      } else {
        Assert.assertTrue(
            qu.getPeer(changingIndex).peer.observer == null
                && (qu.getPeer(changingIndex).peer.follower != null
                    || qu.getPeer(changingIndex).peer.leader != null));
        Assert.assertTrue(
            qu.getPeer(changingIndex).peer.getPeerState() == ServerState.FOLLOWING
                || qu.getPeer(changingIndex).peer.getPeerState() == ServerState.LEADING);
      }

      joiningServers.clear();

      if (newRole.equals("observer")) {
        newRole = "participant";
      } else {
        // lets change leader to observer
        newRole = "observer";
        leaderIndex = getLeaderId(qu);
        changingIndex = leaderIndex;
      }
    }
    closeAllHandles(zkArr);
  }
  /**
   * 1. removes and adds back two servers (incl leader). One of the servers is added back as
   * observer 2. tests that reconfig fails if quorum of new config is not up 3. tests that a server
   * that's not up during reconfig learns the new config when it comes up
   *
   * @throws Exception
   */
  @Test
  public void testRemoveAddTwo() throws Exception {
    qu = new QuorumUtil(2); // create 5 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    List<String> leavingServers = new ArrayList<String>();
    List<String> joiningServers = new ArrayList<String>();

    int leaderIndex = getLeaderId(qu);

    // lets remove the leader and some other server
    int leavingIndex1 = leaderIndex;
    int leavingIndex2 = (leaderIndex == 1) ? 2 : 1;

    // find some server that's staying
    int stayingIndex1 = 1, stayingIndex2 = 1, stayingIndex3 = 1;
    while (stayingIndex1 == leavingIndex1 || stayingIndex1 == leavingIndex2) stayingIndex1++;

    while (stayingIndex2 == leavingIndex1
        || stayingIndex2 == leavingIndex2
        || stayingIndex2 == stayingIndex1) stayingIndex2++;

    while (stayingIndex3 == leavingIndex1
        || stayingIndex3 == leavingIndex2
        || stayingIndex3 == stayingIndex1
        || stayingIndex3 == stayingIndex2) stayingIndex3++;

    leavingServers.add(Integer.toString(leavingIndex1));
    leavingServers.add(Integer.toString(leavingIndex2));

    // remember these servers so we can add them back later
    joiningServers.add(
        "server."
            + leavingIndex1
            + "=localhost:"
            + qu.getPeer(leavingIndex1).peer.getQuorumAddress().getPort()
            + ":"
            + qu.getPeer(leavingIndex1).peer.getElectionAddress().getPort()
            + ":participant;localhost:"
            + qu.getPeer(leavingIndex1).peer.getClientPort());

    // this server will be added back as an observer
    joiningServers.add(
        "server."
            + leavingIndex2
            + "=localhost:"
            + qu.getPeer(leavingIndex2).peer.getQuorumAddress().getPort()
            + ":"
            + qu.getPeer(leavingIndex2).peer.getElectionAddress().getPort()
            + ":observer;localhost:"
            + qu.getPeer(leavingIndex2).peer.getClientPort());

    qu.shutdown(leavingIndex1);
    qu.shutdown(leavingIndex2);

    // 3 servers still up so this should work
    reconfig(zkArr[stayingIndex2], null, leavingServers, null, -1);

    qu.shutdown(stayingIndex2);

    // the following commands would not work in the original
    // cluster of 5, but now that we've removed 2 servers
    // we have a cluster of 3 servers and one of them is allowed to fail

    testServerHasConfig(zkArr[stayingIndex1], null, leavingServers);
    testServerHasConfig(zkArr[stayingIndex3], null, leavingServers);
    testNormalOperation(zkArr[stayingIndex1], zkArr[stayingIndex3]);

    // this is a test that a reconfig will only succeed
    // if there is a quorum up in new config. Below there is no
    // quorum so it should fail

    // the sleep is necessary so that the leader figures out
    // that the switched off servers are down
    Thread.sleep(10000);

    try {
      reconfig(zkArr[stayingIndex1], joiningServers, null, null, -1);
      Assert.fail(
          "reconfig completed successfully even though there is no quorum up in new config!");
    } catch (KeeperException.NewConfigNoQuorum e) {

    }

    // now start the third server so that new config has quorum
    qu.restart(stayingIndex2);

    reconfig(zkArr[stayingIndex1], joiningServers, null, null, -1);
    testNormalOperation(zkArr[stayingIndex2], zkArr[stayingIndex3]);
    testServerHasConfig(zkArr[stayingIndex2], joiningServers, null);

    // this server wasn't around during the configuration change
    // we should check that it is able to connect, finds out
    // about the change and becomes an observer.

    qu.restart(leavingIndex2);
    Assert.assertTrue(qu.getPeer(leavingIndex2).peer.getPeerState() == ServerState.OBSERVING);
    testNormalOperation(zkArr[stayingIndex2], zkArr[leavingIndex2]);
    testServerHasConfig(zkArr[leavingIndex2], joiningServers, null);

    closeAllHandles(zkArr);
  }
  @Test
  public void testRemoveAddOne() throws Exception {
    qu = new QuorumUtil(1); // create 3 servers
    qu.disableJMXTest = true;
    qu.startAll();
    ZooKeeper[] zkArr = createHandles(qu);

    List<String> leavingServers = new ArrayList<String>();
    List<String> joiningServers = new ArrayList<String>();

    int leaderIndex = getLeaderId(qu);

    // during first iteration, leavingIndex will correspond to a follower
    // during second iteration leavingIndex will be the index of the leader
    int leavingIndex = (leaderIndex == 1) ? 2 : 1;

    for (int i = 0; i < 2; i++) {
      // some of the operations will be executed by a client connected to
      // the removed server
      // while others are invoked by a client connected to some other
      // server.
      // when we're removing the leader, zk1 will be the client connected
      // to removed server
      ZooKeeper zk1 =
          (leavingIndex == leaderIndex) ? zkArr[leaderIndex] : zkArr[(leaderIndex % qu.ALL) + 1];
      ZooKeeper zk2 =
          (leavingIndex == leaderIndex) ? zkArr[(leaderIndex % qu.ALL) + 1] : zkArr[leaderIndex];

      leavingServers.add(Integer.toString(leavingIndex));

      // remember this server so we can add it back later
      joiningServers.add(
          "server."
              + leavingIndex
              + "=localhost:"
              + qu.getPeer(leavingIndex).peer.getQuorumAddress().getPort()
              + ":"
              + qu.getPeer(leavingIndex).peer.getElectionAddress().getPort()
              + ":participant;localhost:"
              + qu.getPeer(leavingIndex).peer.getClientPort());

      String configStr = reconfig(zk1, null, leavingServers, null, -1);
      testServerHasConfig(zk2, null, leavingServers);
      testNormalOperation(zk2, zk1);

      QuorumVerifier qv = qu.getPeer(1).peer.configFromString(configStr);
      long version = qv.getVersion();

      // checks that conditioning on version works properly
      try {
        reconfig(zk2, joiningServers, null, null, version + 1);
        Assert.fail("reconfig succeeded even though version condition was incorrect!");
      } catch (KeeperException.BadVersionException e) {

      }

      reconfig(zk2, joiningServers, null, null, version);

      testNormalOperation(zk1, zk2);
      testServerHasConfig(zk1, joiningServers, null);

      // second iteration of the loop will remove the leader
      // and add it back (as follower)
      leavingIndex = leaderIndex = getLeaderId(qu);
      leavingServers.clear();
      joiningServers.clear();
    }

    closeAllHandles(zkArr);
  }
 private int getLeaderId(QuorumUtil qu) {
   int leaderId = 1;
   while (qu.getPeer(leaderId).peer.leader == null) leaderId++;
   return leaderId;
 }