@Test
public void testNamespace() throws Exception {
Timing timing = new Timing();
ChildReaper reaper = null;
CuratorFramework client =
CuratorFrameworkFactory.builder()
.connectString(server.getConnectString())
.sessionTimeoutMs(timing.session())
.connectionTimeoutMs(timing.connection())
.retryPolicy(new RetryOneTime(1))
.namespace("foo")
.build();
try {
client.start();
for (int i = 0; i < 10; ++i) {
client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
}
reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
reaper.start();
timing.forWaiting().sleepABit();
Stat stat = client.checkExists().forPath("/test");
Assert.assertEquals(stat.getNumChildren(), 0);
stat = client.usingNamespace(null).checkExists().forPath("/foo/test");
Assert.assertNotNull(stat);
Assert.assertEquals(stat.getNumChildren(), 0);
} finally {
Closeables.closeQuietly(reaper);
Closeables.closeQuietly(client);
}
}
@Test
public void testCheckVersion() throws Exception {
CuratorFramework client =
CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
client.start();
try {
client.create().forPath("/foo");
Stat stat = client.setData().forPath("/foo", "new".getBytes()); // up the version
try {
client
.inTransaction()
.check()
.withVersion(stat.getVersion() + 1)
.forPath("/foo") // force a bad version
.and()
.create()
.forPath("/bar")
.and()
.commit();
Assert.fail();
} catch (KeeperException.BadVersionException correct) {
// correct
}
Assert.assertNull(client.checkExists().forPath("/bar"));
} finally {
client.close();
}
}
@Test
public void testSomeNodes() throws Exception {
Timing timing = new Timing();
ChildReaper reaper = null;
CuratorFramework client =
CuratorFrameworkFactory.newClient(
server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1));
try {
client.start();
Random r = new Random();
int nonEmptyNodes = 0;
for (int i = 0; i < 10; ++i) {
client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
if (r.nextBoolean()) {
client.create().forPath("/test/" + Integer.toString(i) + "/foo");
++nonEmptyNodes;
}
}
reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
reaper.start();
timing.forWaiting().sleepABit();
Stat stat = client.checkExists().forPath("/test");
Assert.assertEquals(stat.getNumChildren(), nonEmptyNodes);
} finally {
Closeables.closeQuietly(reaper);
Closeables.closeQuietly(client);
}
}
@Test
public void testBasic() throws Exception {
CuratorFramework client =
CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
client.start();
try {
Collection<CuratorTransactionResult> results =
client
.inTransaction()
.create()
.forPath("/foo")
.and()
.create()
.forPath("/foo/bar", "snafu".getBytes())
.and()
.commit();
Assert.assertTrue(client.checkExists().forPath("/foo/bar") != null);
Assert.assertEquals(client.getData().forPath("/foo/bar"), "snafu".getBytes());
CuratorTransactionResult fooResult =
Iterables.find(
results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/foo"));
CuratorTransactionResult fooBarResult =
Iterables.find(
results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/foo/bar"));
Assert.assertNotNull(fooResult);
Assert.assertNotNull(fooBarResult);
Assert.assertNotSame(fooResult, fooBarResult);
Assert.assertEquals(fooResult.getResultPath(), "/foo");
Assert.assertEquals(fooBarResult.getResultPath(), "/foo/bar");
} finally {
client.close();
}
}
@Before
public void setup() throws Exception {
zkServer = new TestingServer(2222);
zkClient =
CuratorFrameworkFactory.newClient(
zkServer.getConnectString(), new ExponentialBackoffRetry(300, 5));
dragonZK = new DragonZooKeeper(zkClient, "/dragon");
}
public static IZkConnection createZkConnection(String connectString) {
Timing timing = new Timing();
CuratorFramework client =
CuratorFrameworkFactory.newClient(
connectString, timing.session(), timing.connection(), new RetryOneTime(1));
client.start();
try {
return new CuratorZKClientBridge(client);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Before
public void setUp() throws Exception {
server = new TestingServer();
String connectionString = server.getConnectString();
Map conf = new HashMap();
conf.put(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT, 1000);
conf.put(Config.STORM_ZOOKEEPER_RETRY_TIMES, 4);
conf.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL, 5);
ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
zookeeper = CuratorFrameworkFactory.newClient(connectionString, retryPolicy);
dynamicBrokersReader = new DynamicBrokersReader(conf, connectionString, masterPath, topic);
zookeeper.start();
}
@Test
public void testOverSubscribed() throws Exception {
final Timing timing = new Timing();
final CuratorFramework client =
CuratorFrameworkFactory.newClient(
server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1));
ExecutorService service = Executors.newCachedThreadPool();
ExecutorCompletionService<Void> completionService =
new ExecutorCompletionService<Void>(service);
try {
client.start();
final Semaphore semaphore = new Semaphore(0);
final CountDownLatch latch = new CountDownLatch(1);
for (int i = 0; i < (QTY + 1); ++i) {
completionService.submit(
new Callable<Void>() {
@Override
public Void call() throws Exception {
DistributedDoubleBarrier barrier =
new DistributedDoubleBarrier(client, "/barrier", QTY) {
@Override
protected List<String> getChildrenForEntering() throws Exception {
semaphore.release();
Assert.assertTrue(timing.awaitLatch(latch));
return super.getChildrenForEntering();
}
};
Assert.assertTrue(barrier.enter(timing.seconds(), TimeUnit.SECONDS));
Assert.assertTrue(barrier.leave(timing.seconds(), TimeUnit.SECONDS));
return null;
}
});
}
Assert.assertTrue(
semaphore.tryAcquire(
QTY + 1,
timing.seconds(),
TimeUnit.SECONDS)); // wait until all QTY+1 barriers are trying to enter
latch.countDown();
for (int i = 0; i < (QTY + 1); ++i) {
completionService.take().get(); // to check for assertions
}
} finally {
service.shutdown();
Closeables.close(client, true);
}
}
public KafkaTestBroker() {
try {
server = new TestingServer();
String zookeeperConnectionString = server.getConnectString();
ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
zookeeper = CuratorFrameworkFactory.newClient(zookeeperConnectionString, retryPolicy);
zookeeper.start();
port = InstanceSpec.getRandomPort();
kafka.server.KafkaConfig config = buildKafkaConfig(zookeeperConnectionString);
kafka = new KafkaServerStartable(config);
kafka.startup();
} catch (Exception ex) {
throw new RuntimeException("Could not start test broker", ex);
}
}
/**
* Create and configure the ZK sync object with the KinesisSpoutConfig.
*
* @param config the configuration for the spout.
*/
ZookeeperShardState(final KinesisSpoutConfig config) {
this.config = config;
this.rand = new Random();
try {
zk =
CuratorFrameworkFactory.newClient(
config.getZookeeperConnectionString(),
new ExponentialBackoffRetry(BASE_SLEEP_TIME_MS, MAX_NUM_RETRIES));
} catch (IOException e) {
LOG.error("Could not connect to ZooKeeper", e);
throw new KinesisSpoutException(e);
}
zk.start();
}
@Test
public void testWithNamespace() throws Exception {
CuratorFramework client =
CuratorFrameworkFactory.builder()
.connectString(server.getConnectString())
.retryPolicy(new RetryOneTime(1))
.namespace("galt")
.build();
client.start();
try {
Collection<CuratorTransactionResult> results =
client
.inTransaction()
.create()
.forPath("/foo", "one".getBytes())
.and()
.create()
.withMode(CreateMode.PERSISTENT_SEQUENTIAL)
.forPath("/test-", "one".getBytes())
.and()
.setData()
.forPath("/foo", "two".getBytes())
.and()
.create()
.forPath("/foo/bar")
.and()
.delete()
.forPath("/foo/bar")
.and()
.commit();
Assert.assertTrue(client.checkExists().forPath("/foo") != null);
Assert.assertTrue(client.nonNamespaceView().checkExists().forPath("/galt/foo") != null);
Assert.assertEquals(client.getData().forPath("/foo"), "two".getBytes());
Assert.assertTrue(client.checkExists().forPath("/foo/bar") == null);
CuratorTransactionResult ephemeralResult =
Iterables.find(
results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/test-"));
Assert.assertNotNull(ephemeralResult);
Assert.assertNotEquals(ephemeralResult.getResultPath(), "/test-");
Assert.assertTrue(ephemeralResult.getResultPath().startsWith("/test-"));
} finally {
client.close();
}
}
private void initZooKeeper() {
getLogger().info("Init ZooKeeper");
this.zooKeeperClient =
CuratorFrameworkFactory.builder()
.connectString("elab.kr:2181")
.retryPolicy(new ExponentialBackoffRetry(1000, 3))
.aclProvider(new ServiceACLProvider())
.sessionTimeoutMs(10 * 1000)
.build();
this.zooKeeperClient.start();
try {
this.zooKeeperClient.getZookeeperClient().blockUntilConnectedOrTimedOut();
} catch (InterruptedException e) {
getLogger().fatal("Not Connected ZooKeeper", e);
}
this.registerService();
}
@Test
public void testInCluster() throws Exception {
final int PARTICIPANT_QTY = 3;
List<ClientAndLatch> clients = Lists.newArrayList();
Timing timing = new Timing();
TestingCluster cluster = new TestingCluster(PARTICIPANT_QTY);
try {
cluster.start();
List<InstanceSpec> instances = Lists.newArrayList(cluster.getInstances());
for (int i = 0; i < PARTICIPANT_QTY; ++i) {
CuratorFramework client =
CuratorFrameworkFactory.newClient(
instances.get(i).getConnectString(),
timing.session(),
timing.connection(),
new RetryOneTime(1));
LeaderLatch latch = new LeaderLatch(client, "/latch");
clients.add(new ClientAndLatch(client, latch, i));
client.start();
latch.start();
}
ClientAndLatch leader = waitForALeader(clients, timing);
Assert.assertNotNull(leader);
cluster.killServer(instances.get(leader.index));
Thread.sleep(timing.multiple(2).session());
leader = waitForALeader(clients, timing);
Assert.assertNotNull(leader);
Assert.assertEquals(getLeaders(clients).size(), 1);
} finally {
for (ClientAndLatch client : clients) {
Closeables.close(client.latch, true);
Closeables.close(client.client, true);
}
Closeables.close(cluster, true);
}
}
/** Amoeba启动的时候，初始化zookeeper客户端，另外，给zookeeper预热 */
public void init() throws Exception {
Properties props = Utils.readGlobalSeqConfigProps();
String zkHosts = props.getProperty("zkHosts", "127.0.0.1:2181");
Integer connTimeOut = Integer.valueOf(props.getProperty("zkConnTimeOut", "60"));
log.info(String.format("zookeeper config: connect string= %s", zkHosts));
log.info(String.format("zookeeper config: connect timeout= %d seconds", connTimeOut));
client =
CuratorFrameworkFactory.builder()
.connectString(zkHosts)
.retryPolicy(new ExponentialBackoffRetry(BASE_SLEEP_TIME, RETRY_TIMES))
.connectionTimeoutMs(connTimeOut * 1000)
.build(); // 时间要由秒转成毫秒
client.start();
// 只是为client热身而调用
client.checkExists().forPath(ROOT_PATH);
}
public static CuratorFramework newCurator(
Map conf, List<String> servers, Object port, String root) {
List<String> serverPorts = new ArrayList<String>();
for (String zkServer : (List<String>) servers) {
serverPorts.add(zkServer + ":" + Utils.getInt(port));
}
String zkStr = StringUtils.join(serverPorts, ",") + root;
try {
CuratorFramework ret =
CuratorFrameworkFactory.newClient(
zkStr,
Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)),
15000,
new RetryNTimes(
Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL))));
return ret;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("unchecked")
private CuratorFramework newCurator(Map stateConf) {
Integer port = (Integer) stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_PORT);
String zkServerPorts = "";
for (String server : (List<String>) stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS)) {
zkServerPorts += server + ":" + port + ",";
}
// LOG.info("@@@@@@@@@@@@@@@" + zkServerPorts);
try {
return CuratorFrameworkFactory.newClient(
zkServerPorts,
Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)),
20000,
new RetryNTimes(
Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL))));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Test
public void testBasic() throws Exception {
final Timing timing = new Timing();
final List<Closeable> closeables = Lists.newArrayList();
final CuratorFramework client =
CuratorFrameworkFactory.newClient(
server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1));
try {
closeables.add(client);
client.start();
final CountDownLatch postEnterLatch = new CountDownLatch(QTY);
final CountDownLatch postLeaveLatch = new CountDownLatch(QTY);
final AtomicInteger count = new AtomicInteger(0);
final AtomicInteger max = new AtomicInteger(0);
List<Future<Void>> futures = Lists.newArrayList();
ExecutorService service = Executors.newCachedThreadPool();
for (int i = 0; i < QTY; ++i) {
Future<Void> future =
service.submit(
new Callable<Void>() {
@Override
public Void call() throws Exception {
DistributedDoubleBarrier barrier =
new DistributedDoubleBarrier(client, "/barrier", QTY);
Assert.assertTrue(barrier.enter(timing.seconds(), TimeUnit.SECONDS));
synchronized (TestDistributedDoubleBarrier.this) {
int thisCount = count.incrementAndGet();
if (thisCount > max.get()) {
max.set(thisCount);
}
}
postEnterLatch.countDown();
Assert.assertTrue(timing.awaitLatch(postEnterLatch));
Assert.assertEquals(count.get(), QTY);
Assert.assertTrue(barrier.leave(10, TimeUnit.SECONDS));
count.decrementAndGet();
postLeaveLatch.countDown();
Assert.assertTrue(timing.awaitLatch(postLeaveLatch));
return null;
}
});
futures.add(future);
}
for (Future<Void> f : futures) {
f.get();
}
Assert.assertEquals(count.get(), 0);
Assert.assertEquals(max.get(), QTY);
} finally {
for (Closeable c : closeables) {
Closeables.close(c, true);
}
}
}
private void buildZKClient(final String zkFullPath) {
final String[] zkFullPathItems = zkFullPath.split(ZK_FULL_PATH_DELIMITER);
zkHost_ = zkFullPathItems[1];
zkNamespace_ =
zkFullPathItems[2].substring(
0, zkFullPathItems[2].lastIndexOf(ZK_FULL_PATH_NAMESPACE_SEPERATOR));
zkNodePath_ =
zkFullPathItems[2].substring(
zkFullPathItems[2].lastIndexOf(ZK_FULL_PATH_NAMESPACE_SEPERATOR));
if (zkClient_ == null) {
try {
zkClient_ =
CuratorFrameworkFactory.builder()
.connectString(zkHost_)
.namespace(zkNamespace_)
.retryPolicy(new ExponentialBackoffRetry(1000, 60))
.connectionTimeoutMs(2 * 3600 * 1000)
.build();
zkClient_
.getConnectionStateListenable()
.addListener(
new ConnectionStateListener() {
@Override
public void stateChanged(CuratorFramework client, ConnectionState newState) {
// TODO Auto-generated method stub
if (newState == ConnectionState.LOST) {
try {
synchronized (this) {
while (!client.getZookeeperClient().blockUntilConnectedOrTimedOut()) {
if (logger_ != null) {
if (logger_.isInfoEnabled()) logger_.info("waitting reconnected...");
}
wait(1000);
}
}
if (logger_ != null) {
if (logger_.isInfoEnabled()) logger_.info("to zookeeper reconnected.");
}
doNodesDiscovery();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
if (logger_ != null) {
if (logger_.isInfoEnabled())
logger_.info(
String.format(
"To re connect to zookeeper to generate an error, cause:%s",
e));
}
}
}
}
});
zkClient_.start();
} catch (Throwable t) {
zkClient_.close();
throw new IllegalStateException("building zookeeper client failed.");
}
}
}