@Test
  public void testCoordinateListenerConnectionDiesReconnectAfterTimeout() throws Exception {
    final CountDownLatch connectedLatch1 = new CountDownLatch(2);
    final CountDownLatch connectedLatch2 = new CountDownLatch(6);
    TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
    assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
    assertEquals(
        CoordinateListener.Event.COORDINATE_OK, listener.events.get(listener.events.size() - 1));
    log.info("Killing connection");
    forwarder.terminate();

    log.info("Connection down.");

    Thread.sleep(9000);
    log.info("Recreating connection soon" + forwarderPort + "->" + zkport);
    Thread.sleep(1000);
    assertEquals(
        CoordinateListener.Event.NO_CONNECTION_TO_STORAGE,
        listener.events.get(listener.events.size() - 1));
    forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
    assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));

    for (int c = 0; c < 100; c++) {
      if (CoordinateListener.Event.COORDINATE_OK
          == listener.events.get(listener.events.size() - 1)) {
        break;
      }
      Thread.sleep(300);
    }
    Thread.sleep(4500);
    assertEquals(
        CoordinateListener.Event.COORDINATE_OK, listener.events.get(listener.events.size() - 1));

    forwarder.terminate();
  }
  /**
   * Tests the behavior of Zookeeper upon a restart. ZK should clean up old coordinates.
   *
   * @throws Exception
   */
  @Test
  public void testZookeeperRestarts() throws Exception {
    final CountDownLatch connectedLatch1 = new CountDownLatch(1);
    final CountDownLatch connectedLatch2 = new CountDownLatch(3);
    TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
    assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
    log.info("Killing zookeeper");
    forwarder.terminate();

    ezk.shutdown();
    ezk.del();
    ezk.init();
    Thread.sleep(2000);
    forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);

    int timeoutSecs = 30;
    while (--timeoutSecs > 0) {
      Thread.sleep(1000);
    }
    Coordinate c = Coordinate.parse("1.service.user.cell");
    cn.createCoordinate(c);

    Thread.sleep(9000);
    assertEquals(
        listener.events.get(listener.events.size() - 1), CoordinateListener.Event.COORDINATE_OK);
  }
  @Test
  public void testCoordinateListenerStolenCoordinate() throws Exception {

    final CountDownLatch connectedLatch1 = new CountDownLatch(1);
    final CountDownLatch connectedLatch2 = new CountDownLatch(2);
    TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
    assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
    log.info("Killing zookeeper");
    assertTrue(zk.getState() == ZooKeeper.States.CONNECTED);

    log.info("Killing connection");
    forwarder.terminate();

    zk.delete("/cn/cell/user/service/1/status", -1);
    Util.mkdir(zk, "/cn/cell/user/service/1/status", ZooDefs.Ids.OPEN_ACL_UNSAFE);

    forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);

    assertTrue(connectedLatch2.await(6, TimeUnit.SECONDS));

    int i = 0;
    int q = -1;
    while (true) {
      if (q != listener.events.size()) {
        q = listener.events.size();
      }
      if (listener.events.get(listener.events.size() - 1) == CoordinateListener.Event.NOT_OWNER) {
        break;
      }

      Thread.sleep(10);
      ++i;
      if (i > 1000) {
        fail("Did not get NOT_OWNER");
      }
    }

    // cn2.close();
    // We use the same path for the new ezk, so it reads up the old state, and hence the coordinate
    // is ok.
    //        assertEquals(CoordinateListener.Event.COORDINATE_OK, listener.events.get(2));
    forwarder.terminate();
  }
 @Test
 public void testCoordinateListenerInitialEvent() throws Exception {
   final CountDownLatch connectedLatch1 = new CountDownLatch(2);
   final CountDownLatch connectedLatch2 = new CountDownLatch(2);
   final TestCoordinateListener listener =
       setUpListenerEnvironment(connectedLatch1, connectedLatch2);
   assertTrue(connectedLatch1.await(15, TimeUnit.SECONDS));
   assertEquals(2, listener.events.size());
   assertEquals(CoordinateListener.Event.COORDINATE_OK, listener.events.get(1));
   forwarder.terminate();
 }
 @Test
 public void testCoordinateListenerCoordinateLost() throws Exception {
   final CountDownLatch connectedLatch1 = new CountDownLatch(1);
   final CountDownLatch connectedLatch2 = new CountDownLatch(3);
   TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
   assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
   log.info("Deleting coordinate");
   forwarder.terminate();
   zk.delete("/cn/cell/user/service/1/status", -1);
   zk.delete("/cn/cell/user/service/1/config", -1);
   zk.delete("/cn/cell/user/service/1", -1);
   forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
   assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
   int i = 0;
   while (listener.events.get(listener.events.size() - 1) != CoordinateListener.Event.NOT_OWNER) {
     Thread.sleep(30);
     ++i;
     if (i > 100) {
       fail("Did not get COORDINATE_VANISHED");
     }
   }
   forwarder.terminate();
 }
  @Test
  public void testCoordinateListenerConnectionDiesReconnect() throws Exception {
    final CountDownLatch connectedLatch1 = new CountDownLatch(2);
    final CountDownLatch connectedLatch2 = new CountDownLatch(4);
    TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
    assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));

    log.info("Killing connection");
    forwarder.terminate();
    log.info("Recreating connection" + forwarderPort + "->" + zkport);

    forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
    assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
    assertEquals(
        CoordinateListener.Event.COORDINATE_OK, listener.events.get(listener.events.size() - 1));
  }
  @Test
  public void testCoordinateListenerCoordinateCorrupted() throws Exception {
    final CountDownLatch connectedLatch1 = new CountDownLatch(2);
    final CountDownLatch connectedLatch2 = new CountDownLatch(3);
    final TestCoordinateListener listener =
        setUpListenerEnvironment(connectedLatch1, connectedLatch2);
    assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
    log.info("Corrupting coordinate.");
    byte[] garbageBytes = "sdfgsdfgsfgdsdfgsdfgsdfg".getBytes("UTF-16LE");

    zk.setData("/cn/cell/user/service/1/status", garbageBytes, -1);
    assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
    assertEquals(3, listener.events.size());
    assertEquals(CoordinateListener.Event.COORDINATE_OUT_OF_SYNC, listener.events.get(2));
    forwarder.terminate();
  }
 @Test
 public void testCoordinateListenerConnectionDies() throws Exception {
   final CountDownLatch connectedLatch1 = new CountDownLatch(1);
   final CountDownLatch connectedLatch2 = new CountDownLatch(2);
   final TestCoordinateListener listener =
       setUpListenerEnvironment(connectedLatch1, connectedLatch2);
   assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
   log.info("Killing zookeeper");
   ezk.shutdown();
   forwarder.terminate();
   assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
   final int size = listener.events.size();
   assertTrue(size > 1);
   log.info("status " + listener.events.toString());
   assertEquals(CoordinateListener.Event.NO_CONNECTION_TO_STORAGE, listener.events.get(size - 1));
 }
  @Test
  public void testCoordinateListenerCoordinateOutOfSync() throws Exception {
    final CountDownLatch connectedLatch1 = new CountDownLatch(2);
    final CountDownLatch connectedLatch2 = new CountDownLatch(4);
    final TestCoordinateListener listener =
        setUpListenerEnvironment(connectedLatch1, connectedLatch2);
    assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));

    log.info("Writing different coordinate.");
    String source = "\"{\\\"state\\\":\\\"STARTING\\\",\\\"message\\\":\\\"Lost hamster.\\\"}\" {}";
    byte[] byteArray = source.getBytes(Util.CHARSET_NAME);

    zk.setData("/cn/cell/user/service/1/status", byteArray, -1);
    log.info("Done writing different coordinate.");
    assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));

    assertEquals(CoordinateListener.Event.NOT_OWNER, listener.events.get(3));
    forwarder.terminate();
  }
  /**
   * Tests that one process claims a coordinate, then another process tries to claim the same
   * coordinate. The first coordinate looses connection to ZooKeeper and the other process gets the
   * coordinate.
   *
   * @throws Exception
   */
  @Test
  public void testFastHardRestart() throws Exception {
    final Coordinate c = Coordinate.parse("1.service.user.cell");
    final CountDownLatch claimLatch1 = new CountDownLatch(1);
    forwarderPort = Net.getFreePort();
    forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
    Cloudname cn1 =
        new ZkCloudname.Builder().setConnectString("localhost:" + forwarderPort).build().connect();
    cn1.createCoordinate(c);

    ServiceHandle handle1 = cn1.claim(c);
    handle1.registerCoordinateListener(
        new CoordinateListener() {

          @Override
          public void onCoordinateEvent(Event event, String message) {
            if (event == Event.COORDINATE_OK) {
              claimLatch1.countDown();
            }
          }
        });
    assertTrue(claimLatch1.await(5, TimeUnit.SECONDS));

    Cloudname cn2 =
        new ZkCloudname.Builder().setConnectString("localhost:" + zkport).build().connect();

    ServiceHandle handle2 = cn2.claim(c);

    forwarder.terminate();

    assertTrue(handle2.waitForCoordinateOkSeconds(20));

    ServiceStatus status = new ServiceStatus(ServiceState.RUNNING, "updated status");
    handle2.setStatus(status);

    Cloudname cn3 =
        new ZkCloudname.Builder().setConnectString("localhost:" + zkport).build().connect();
    ServiceStatus statusRetrieved = cn3.getStatus(c);
    assertEquals("updated status", statusRetrieved.getMessage());
  }