/**
   * Tests the behavior of Zookeeper upon a restart. ZK should clean up old coordinates.
   *
   * @throws Exception
   */
  @Test
  public void testZookeeperRestarts() throws Exception {
    final CountDownLatch connectedLatch1 = new CountDownLatch(1);
    final CountDownLatch connectedLatch2 = new CountDownLatch(3);
    TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
    assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
    log.info("Killing zookeeper");
    forwarder.terminate();

    ezk.shutdown();
    ezk.del();
    ezk.init();
    Thread.sleep(2000);
    forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);

    int timeoutSecs = 30;
    while (--timeoutSecs > 0) {
      Thread.sleep(1000);
    }
    Coordinate c = Coordinate.parse("1.service.user.cell");
    cn.createCoordinate(c);

    Thread.sleep(9000);
    assertEquals(
        listener.events.get(listener.events.size() - 1), CoordinateListener.Event.COORDINATE_OK);
  }
  private TestCoordinateListener setUpListenerEnvironment(
      CountDownLatch connectedLatch1, CountDownLatch connectedLatch2) throws Exception {
    forwarderPort = Net.getFreePort();
    forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
    final Coordinate c = Coordinate.parse("1.service.user.cell");

    cn = makeLocalZkCloudname(forwarderPort);
    try {
      cn.createCoordinate(c);
    } catch (CoordinateException e) {
      fail(e.toString());
    }
    final TestCoordinateListener listener =
        new TestCoordinateListener(connectedLatch1, connectedLatch2);
    ServiceHandle serviceHandle = cn.claim(c);
    serviceHandle.registerCoordinateListener(listener);

    return listener;
  }
  /**
   * Tests that one process claims a coordinate, then another process tries to claim the same
   * coordinate. The first coordinate looses connection to ZooKeeper and the other process gets the
   * coordinate.
   *
   * @throws Exception
   */
  @Test
  public void testFastHardRestart() throws Exception {
    final Coordinate c = Coordinate.parse("1.service.user.cell");
    final CountDownLatch claimLatch1 = new CountDownLatch(1);
    forwarderPort = Net.getFreePort();
    forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
    Cloudname cn1 =
        new ZkCloudname.Builder().setConnectString("localhost:" + forwarderPort).build().connect();
    cn1.createCoordinate(c);

    ServiceHandle handle1 = cn1.claim(c);
    handle1.registerCoordinateListener(
        new CoordinateListener() {

          @Override
          public void onCoordinateEvent(Event event, String message) {
            if (event == Event.COORDINATE_OK) {
              claimLatch1.countDown();
            }
          }
        });
    assertTrue(claimLatch1.await(5, TimeUnit.SECONDS));

    Cloudname cn2 =
        new ZkCloudname.Builder().setConnectString("localhost:" + zkport).build().connect();

    ServiceHandle handle2 = cn2.claim(c);

    forwarder.terminate();

    assertTrue(handle2.waitForCoordinateOkSeconds(20));

    ServiceStatus status = new ServiceStatus(ServiceState.RUNNING, "updated status");
    handle2.setStatus(status);

    Cloudname cn3 =
        new ZkCloudname.Builder().setConnectString("localhost:" + zkport).build().connect();
    ServiceStatus statusRetrieved = cn3.getStatus(c);
    assertEquals("updated status", statusRetrieved.getMessage());
  }