/**
* Tests the behavior of Zookeeper upon a restart. ZK should clean up old coordinates.
*
* @throws Exception
*/
@Test
public void testZookeeperRestarts() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(1);
final CountDownLatch connectedLatch2 = new CountDownLatch(3);
TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Killing zookeeper");
forwarder.terminate();
ezk.shutdown();
ezk.del();
ezk.init();
Thread.sleep(2000);
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
int timeoutSecs = 30;
while (--timeoutSecs > 0) {
Thread.sleep(1000);
}
Coordinate c = Coordinate.parse("1.service.user.cell");
cn.createCoordinate(c);
Thread.sleep(9000);
assertEquals(
listener.events.get(listener.events.size() - 1), CoordinateListener.Event.COORDINATE_OK);
}
@Test
public void testCoordinateListenerConnectionDiesReconnectAfterTimeout() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(2);
final CountDownLatch connectedLatch2 = new CountDownLatch(6);
TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
assertEquals(
CoordinateListener.Event.COORDINATE_OK, listener.events.get(listener.events.size() - 1));
log.info("Killing connection");
forwarder.terminate();
log.info("Connection down.");
Thread.sleep(9000);
log.info("Recreating connection soon" + forwarderPort + "->" + zkport);
Thread.sleep(1000);
assertEquals(
CoordinateListener.Event.NO_CONNECTION_TO_STORAGE,
listener.events.get(listener.events.size() - 1));
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
for (int c = 0; c < 100; c++) {
if (CoordinateListener.Event.COORDINATE_OK
== listener.events.get(listener.events.size() - 1)) {
break;
}
Thread.sleep(300);
}
Thread.sleep(4500);
assertEquals(
CoordinateListener.Event.COORDINATE_OK, listener.events.get(listener.events.size() - 1));
forwarder.terminate();
}
@Test
public void testCoordinateListenerStolenCoordinate() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(1);
final CountDownLatch connectedLatch2 = new CountDownLatch(2);
TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Killing zookeeper");
assertTrue(zk.getState() == ZooKeeper.States.CONNECTED);
log.info("Killing connection");
forwarder.terminate();
zk.delete("/cn/cell/user/service/1/status", -1);
Util.mkdir(zk, "/cn/cell/user/service/1/status", ZooDefs.Ids.OPEN_ACL_UNSAFE);
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
assertTrue(connectedLatch2.await(6, TimeUnit.SECONDS));
int i = 0;
int q = -1;
while (true) {
if (q != listener.events.size()) {
q = listener.events.size();
}
if (listener.events.get(listener.events.size() - 1) == CoordinateListener.Event.NOT_OWNER) {
break;
}
Thread.sleep(10);
++i;
if (i > 1000) {
fail("Did not get NOT_OWNER");
}
}
// cn2.close();
// We use the same path for the new ezk, so it reads up the old state, and hence the coordinate
// is ok.
// assertEquals(CoordinateListener.Event.COORDINATE_OK, listener.events.get(2));
forwarder.terminate();
}
@Test
public void testCoordinateListenerCoordinateLost() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(1);
final CountDownLatch connectedLatch2 = new CountDownLatch(3);
TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Deleting coordinate");
forwarder.terminate();
zk.delete("/cn/cell/user/service/1/status", -1);
zk.delete("/cn/cell/user/service/1/config", -1);
zk.delete("/cn/cell/user/service/1", -1);
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
int i = 0;
while (listener.events.get(listener.events.size() - 1) != CoordinateListener.Event.NOT_OWNER) {
Thread.sleep(30);
++i;
if (i > 100) {
fail("Did not get COORDINATE_VANISHED");
}
}
forwarder.terminate();
}