@Test
public void testCoordinateListenerConnectionDiesReconnectAfterTimeout() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(2);
final CountDownLatch connectedLatch2 = new CountDownLatch(6);
TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
assertEquals(
CoordinateListener.Event.COORDINATE_OK, listener.events.get(listener.events.size() - 1));
log.info("Killing connection");
forwarder.terminate();
log.info("Connection down.");
Thread.sleep(9000);
log.info("Recreating connection soon" + forwarderPort + "->" + zkport);
Thread.sleep(1000);
assertEquals(
CoordinateListener.Event.NO_CONNECTION_TO_STORAGE,
listener.events.get(listener.events.size() - 1));
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
for (int c = 0; c < 100; c++) {
if (CoordinateListener.Event.COORDINATE_OK
== listener.events.get(listener.events.size() - 1)) {
break;
}
Thread.sleep(300);
}
Thread.sleep(4500);
assertEquals(
CoordinateListener.Event.COORDINATE_OK, listener.events.get(listener.events.size() - 1));
forwarder.terminate();
}
/**
* Tests the behavior of Zookeeper upon a restart. ZK should clean up old coordinates.
*
* @throws Exception
*/
@Test
public void testZookeeperRestarts() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(1);
final CountDownLatch connectedLatch2 = new CountDownLatch(3);
TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Killing zookeeper");
forwarder.terminate();
ezk.shutdown();
ezk.del();
ezk.init();
Thread.sleep(2000);
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
int timeoutSecs = 30;
while (--timeoutSecs > 0) {
Thread.sleep(1000);
}
Coordinate c = Coordinate.parse("1.service.user.cell");
cn.createCoordinate(c);
Thread.sleep(9000);
assertEquals(
listener.events.get(listener.events.size() - 1), CoordinateListener.Event.COORDINATE_OK);
}
@Test
public void testCoordinateListenerStolenCoordinate() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(1);
final CountDownLatch connectedLatch2 = new CountDownLatch(2);
TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Killing zookeeper");
assertTrue(zk.getState() == ZooKeeper.States.CONNECTED);
log.info("Killing connection");
forwarder.terminate();
zk.delete("/cn/cell/user/service/1/status", -1);
Util.mkdir(zk, "/cn/cell/user/service/1/status", ZooDefs.Ids.OPEN_ACL_UNSAFE);
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
assertTrue(connectedLatch2.await(6, TimeUnit.SECONDS));
int i = 0;
int q = -1;
while (true) {
if (q != listener.events.size()) {
q = listener.events.size();
}
if (listener.events.get(listener.events.size() - 1) == CoordinateListener.Event.NOT_OWNER) {
break;
}
Thread.sleep(10);
++i;
if (i > 1000) {
fail("Did not get NOT_OWNER");
}
}
// cn2.close();
// We use the same path for the new ezk, so it reads up the old state, and hence the coordinate
// is ok.
// assertEquals(CoordinateListener.Event.COORDINATE_OK, listener.events.get(2));
forwarder.terminate();
}
@Test
public void testCoordinateListenerInitialEvent() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(2);
final CountDownLatch connectedLatch2 = new CountDownLatch(2);
final TestCoordinateListener listener =
setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(15, TimeUnit.SECONDS));
assertEquals(2, listener.events.size());
assertEquals(CoordinateListener.Event.COORDINATE_OK, listener.events.get(1));
forwarder.terminate();
}
@Test
public void testCoordinateListenerCoordinateLost() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(1);
final CountDownLatch connectedLatch2 = new CountDownLatch(3);
TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Deleting coordinate");
forwarder.terminate();
zk.delete("/cn/cell/user/service/1/status", -1);
zk.delete("/cn/cell/user/service/1/config", -1);
zk.delete("/cn/cell/user/service/1", -1);
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
int i = 0;
while (listener.events.get(listener.events.size() - 1) != CoordinateListener.Event.NOT_OWNER) {
Thread.sleep(30);
++i;
if (i > 100) {
fail("Did not get COORDINATE_VANISHED");
}
}
forwarder.terminate();
}
@Test
public void testCoordinateListenerConnectionDiesReconnect() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(2);
final CountDownLatch connectedLatch2 = new CountDownLatch(4);
TestCoordinateListener listener = setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Killing connection");
forwarder.terminate();
log.info("Recreating connection" + forwarderPort + "->" + zkport);
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
assertEquals(
CoordinateListener.Event.COORDINATE_OK, listener.events.get(listener.events.size() - 1));
}
@Test
public void testCoordinateListenerCoordinateCorrupted() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(2);
final CountDownLatch connectedLatch2 = new CountDownLatch(3);
final TestCoordinateListener listener =
setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Corrupting coordinate.");
byte[] garbageBytes = "sdfgsdfgsfgdsdfgsdfgsdfg".getBytes("UTF-16LE");
zk.setData("/cn/cell/user/service/1/status", garbageBytes, -1);
assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
assertEquals(3, listener.events.size());
assertEquals(CoordinateListener.Event.COORDINATE_OUT_OF_SYNC, listener.events.get(2));
forwarder.terminate();
}
@Test
public void testCoordinateListenerConnectionDies() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(1);
final CountDownLatch connectedLatch2 = new CountDownLatch(2);
final TestCoordinateListener listener =
setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Killing zookeeper");
ezk.shutdown();
forwarder.terminate();
assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
final int size = listener.events.size();
assertTrue(size > 1);
log.info("status " + listener.events.toString());
assertEquals(CoordinateListener.Event.NO_CONNECTION_TO_STORAGE, listener.events.get(size - 1));
}
@Test
public void testCoordinateListenerCoordinateOutOfSync() throws Exception {
final CountDownLatch connectedLatch1 = new CountDownLatch(2);
final CountDownLatch connectedLatch2 = new CountDownLatch(4);
final TestCoordinateListener listener =
setUpListenerEnvironment(connectedLatch1, connectedLatch2);
assertTrue(connectedLatch1.await(20, TimeUnit.SECONDS));
log.info("Writing different coordinate.");
String source = "\"{\\\"state\\\":\\\"STARTING\\\",\\\"message\\\":\\\"Lost hamster.\\\"}\" {}";
byte[] byteArray = source.getBytes(Util.CHARSET_NAME);
zk.setData("/cn/cell/user/service/1/status", byteArray, -1);
log.info("Done writing different coordinate.");
assertTrue(connectedLatch2.await(20, TimeUnit.SECONDS));
assertEquals(CoordinateListener.Event.NOT_OWNER, listener.events.get(3));
forwarder.terminate();
}
/**
* Tests that one process claims a coordinate, then another process tries to claim the same
* coordinate. The first coordinate looses connection to ZooKeeper and the other process gets the
* coordinate.
*
* @throws Exception
*/
@Test
public void testFastHardRestart() throws Exception {
final Coordinate c = Coordinate.parse("1.service.user.cell");
final CountDownLatch claimLatch1 = new CountDownLatch(1);
forwarderPort = Net.getFreePort();
forwarder = new PortForwarder(forwarderPort, "127.0.0.1", zkport);
Cloudname cn1 =
new ZkCloudname.Builder().setConnectString("localhost:" + forwarderPort).build().connect();
cn1.createCoordinate(c);
ServiceHandle handle1 = cn1.claim(c);
handle1.registerCoordinateListener(
new CoordinateListener() {
@Override
public void onCoordinateEvent(Event event, String message) {
if (event == Event.COORDINATE_OK) {
claimLatch1.countDown();
}
}
});
assertTrue(claimLatch1.await(5, TimeUnit.SECONDS));
Cloudname cn2 =
new ZkCloudname.Builder().setConnectString("localhost:" + zkport).build().connect();
ServiceHandle handle2 = cn2.claim(c);
forwarder.terminate();
assertTrue(handle2.waitForCoordinateOkSeconds(20));
ServiceStatus status = new ServiceStatus(ServiceState.RUNNING, "updated status");
handle2.setStatus(status);
Cloudname cn3 =
new ZkCloudname.Builder().setConnectString("localhost:" + zkport).build().connect();
ServiceStatus statusRetrieved = cn3.getStatus(c);
assertEquals("updated status", statusRetrieved.getMessage());
}