@Test
public void testNamespace() throws Exception {
Timing timing = new Timing();
ChildReaper reaper = null;
CuratorFramework client =
CuratorFrameworkFactory.builder()
.connectString(server.getConnectString())
.sessionTimeoutMs(timing.session())
.connectionTimeoutMs(timing.connection())
.retryPolicy(new RetryOneTime(1))
.namespace("foo")
.build();
try {
client.start();
for (int i = 0; i < 10; ++i) {
client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
}
reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
reaper.start();
timing.forWaiting().sleepABit();
Stat stat = client.checkExists().forPath("/test");
Assert.assertEquals(stat.getNumChildren(), 0);
stat = client.usingNamespace(null).checkExists().forPath("/foo/test");
Assert.assertNotNull(stat);
Assert.assertEquals(stat.getNumChildren(), 0);
} finally {
Closeables.closeQuietly(reaper);
Closeables.closeQuietly(client);
}
}
/**
* Try to atomically update a node in ZooKeeper, creating it if it doesn't exist. This is meant to
* be used within an optimistic concurrency model.
*
* @param pathSuffix suffix to use to build path in ZooKeeper.
* @param f function used to initialize the node, or transform the data already there.
* @return true if node was created/updated, false if a concurrent modification occurred and
* succeeded while trying to update/create the node.
* @throws Exception
*/
private boolean tryAtomicUpdate(final String pathSuffix, final NodeFunction f) throws Exception {
final String path = buildZookeeperPath(pathSuffix);
final Stat stat = zk.checkExists().forPath(path);
if (stat == null) {
try {
zk.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.PERSISTENT)
.forPath(path, f.initialize());
} catch (KeeperException.NodeExistsException e) {
LOG.debug("Concurrent creation of " + path + ", retrying", e);
return false;
}
} else {
Mod<byte[]> newVal = f.apply(zk.getData().forPath(path));
if (newVal.hasModification()) {
try {
zk.setData().withVersion(stat.getVersion()).forPath(path, newVal.get());
} catch (KeeperException.BadVersionException e) {
LOG.debug("Concurrent update to " + path + ", retrying.", e);
return false;
}
}
}
return true;
}
@Test
public void testUpdateShuffleNode() throws Exception {
JobId jobId = JobId.newJobId(createTestAppId(), 123456);
DragonZooKeeper.NodeData node = new DragonZooKeeper.NodeData();
node.nodeId = createTestNodeId();
node.taskId = TaskId.newTaskId(jobId, 34565, TaskType.MAP);
dragonZK.createShufflePath(jobId);
dragonZK.createShuffleNode(jobId, newArrayList(node));
node.nodeId.setHost("NewServerHost");
dragonZK.updateShuffleNode(jobId, newArrayList(node));
Stat stat =
zkClient
.checkExists()
.forPath(
"/dragon/job_458495849584_89894985_123456/shuffle/task_458495849584_89894985_123456_m_34565");
assertNotNull(stat);
byte[] data =
zkClient
.getData()
.forPath(
"/dragon/job_458495849584_89894985_123456/shuffle/task_458495849584_89894985_123456_m_34565");
assertEquals("NewServerHost:2345", new String(data));
}
@Test
public void testSomeNodes() throws Exception {
Timing timing = new Timing();
ChildReaper reaper = null;
CuratorFramework client =
CuratorFrameworkFactory.newClient(
server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1));
try {
client.start();
Random r = new Random();
int nonEmptyNodes = 0;
for (int i = 0; i < 10; ++i) {
client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
if (r.nextBoolean()) {
client.create().forPath("/test/" + Integer.toString(i) + "/foo");
++nonEmptyNodes;
}
}
reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
reaper.start();
timing.forWaiting().sleepABit();
Stat stat = client.checkExists().forPath("/test");
Assert.assertEquals(stat.getNumChildren(), nonEmptyNodes);
} finally {
Closeables.closeQuietly(reaper);
Closeables.closeQuietly(client);
}
}
public void shutdown() throws IOException {
kafka.shutdown();
if (zookeeper.getState().equals(CuratorFrameworkState.STARTED)) {
zookeeper.close();
}
server.close();
FileUtils.deleteQuietly(new File(getLogDir()));
}
/**
* Add this instance to the leadership election and attempt to acquire leadership.
*
* @throws Exception errors
*/
public void start() throws Exception {
Preconditions.checkState(state.compareAndSet(State.LATENT, State.STARTED), "Already started");
client.getConnectionStateListenable().addListener(listener);
client.newNamespaceAwareEnsurePath(latchPath).ensure(client.getZookeeperClient());
internalStart();
}
private boolean internalLockLoop(long startMillis, Long millisToWait, String ourPath)
throws Exception {
if (revocable.get() != null) {
client.getData().usingWatcher(revocableWatcher).forPath(ourPath);
}
boolean haveTheLock = false;
boolean doDelete = false;
try {
while (client.isStarted() && !haveTheLock) {
List<String> children = getSortedChildren();
String sequenceNodeName =
ourPath.substring(basePath.length() + 1); // +1 to include the slash
PredicateResults predicateResults =
driver.getsTheLock(client, children, sequenceNodeName, maxLeases);
if (predicateResults.getsTheLock()) {
haveTheLock = true;
} else {
String previousSequencePath = basePath + "/" + predicateResults.getPathToWatch();
synchronized (this) {
Stat stat = client.checkExists().usingWatcher(watcher).forPath(previousSequencePath);
if (stat != null) {
if (millisToWait != null) {
millisToWait -= (System.currentTimeMillis() - startMillis);
startMillis = System.currentTimeMillis();
if (millisToWait <= 0) {
doDelete = true; // timed out - delete our node
break;
}
wait(millisToWait);
} else {
wait();
}
}
}
// else it may have been deleted (i.e. lock released). Try to acquire again
}
}
} catch (KeeperException e) {
// ignore this and let the retry policy handle it
throw e;
} catch (Exception e) {
doDelete = true;
throw e;
} finally {
if (doDelete) {
client.delete().guaranteed().forPath(ourPath);
}
}
return haveTheLock;
}
/**
* Forcibly sets the value any guarantees of atomicity.
*
* @param newValue the new value
* @throws Exception ZooKeeper errors
*/
public void forceSet(byte[] newValue) throws Exception {
try {
client.setData().forPath(path, newValue);
} catch (KeeperException.NoNodeException dummy) {
try {
client.create().forPath(path, newValue);
} catch (KeeperException.NodeExistsException dummy2) {
client.setData().forPath(path, newValue);
}
}
}
String attemptLock(long time, TimeUnit unit, byte[] lockNodeBytes) throws Exception {
final long startMillis = System.currentTimeMillis();
final Long millisToWait = (unit != null) ? unit.toMillis(time) : null;
final byte[] localLockNodeBytes = (revocable.get() != null) ? new byte[0] : lockNodeBytes;
int retryCount = 0;
ensurePath.ensure(client.getZookeeperClient());
String ourPath = null;
boolean hasTheLock = false;
boolean isDone = false;
while (!isDone) {
isDone = true;
try {
if (localLockNodeBytes != null) {
ourPath =
client
.create()
.withProtection()
.withMode(CreateMode.EPHEMERAL_SEQUENTIAL)
.forPath(path, localLockNodeBytes);
} else {
ourPath =
client
.create()
.withProtection()
.withMode(CreateMode.EPHEMERAL_SEQUENTIAL)
.forPath(path);
}
hasTheLock = internalLockLoop(startMillis, millisToWait, ourPath);
} catch (KeeperException.NoNodeException e) {
// gets thrown by StandardLockInternalsDriver when it can't find the lock node
// this can happen when the session expires, etc. So, if the retry allows, just try it all
// again
if (client
.getZookeeperClient()
.getRetryPolicy()
.allowRetry(retryCount++, System.currentTimeMillis() - startMillis)) {
isDone = false;
if (ourPath != null) {
client.delete().inBackground().forPath(ourPath); // just in case
}
} else {
throw e;
}
}
}
if (hasTheLock) {
return ourPath;
}
return null;
}
public static IZkConnection createZkConnection(String connectString) {
Timing timing = new Timing();
CuratorFramework client =
CuratorFrameworkFactory.newClient(
connectString, timing.session(), timing.connection(), new RetryOneTime(1));
client.start();
try {
return new CuratorZKClientBridge(client);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* The shared value must be started before it can be used. Call {@link #close()} when you are
* finished with the shared value
*
* @throws Exception ZK errors, interruptions, etc.
*/
public void start() throws Exception {
Preconditions.checkState(state.compareAndSet(State.LATENT, State.STARTED), "already started");
client.getConnectionStateListenable().addListener(connectionStateListener);
try {
client.create().creatingParentsIfNeeded().forPath(path, seedValue);
} catch (KeeperException.NodeExistsException ignore) {
// ignore
}
readValue();
}
/**
* Remove this instance from the leadership election. If this instance is the leader, leadership
* is released. IMPORTANT: the only way to release leadership is by calling close(). All
* LeaderLatch instances must eventually be closed.
*
* @throws IOException errors
*/
@Override
public void close() throws IOException {
Preconditions.checkState(state.compareAndSet(State.STARTED, State.CLOSED), "Not started");
try {
client.delete().guaranteed().inBackground().forPath(ourPath);
} catch (Exception e) {
throw new IOException(e);
} finally {
client.getConnectionStateListenable().removeListener(listener);
setLeadership(false);
}
}
@Test
public void testOverSubscribed() throws Exception {
final Timing timing = new Timing();
final CuratorFramework client =
CuratorFrameworkFactory.newClient(
server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1));
ExecutorService service = Executors.newCachedThreadPool();
ExecutorCompletionService<Void> completionService =
new ExecutorCompletionService<Void>(service);
try {
client.start();
final Semaphore semaphore = new Semaphore(0);
final CountDownLatch latch = new CountDownLatch(1);
for (int i = 0; i < (QTY + 1); ++i) {
completionService.submit(
new Callable<Void>() {
@Override
public Void call() throws Exception {
DistributedDoubleBarrier barrier =
new DistributedDoubleBarrier(client, "/barrier", QTY) {
@Override
protected List<String> getChildrenForEntering() throws Exception {
semaphore.release();
Assert.assertTrue(timing.awaitLatch(latch));
return super.getChildrenForEntering();
}
};
Assert.assertTrue(barrier.enter(timing.seconds(), TimeUnit.SECONDS));
Assert.assertTrue(barrier.leave(timing.seconds(), TimeUnit.SECONDS));
return null;
}
});
}
Assert.assertTrue(
semaphore.tryAcquire(
QTY + 1,
timing.seconds(),
TimeUnit.SECONDS)); // wait until all QTY+1 barriers are trying to enter
latch.countDown();
for (int i = 0; i < (QTY + 1); ++i) {
completionService.take().get(); // to check for assertions
}
} finally {
service.shutdown();
Closeables.close(client, true);
}
}
private void internalStart() throws Exception {
hasLeadership.set(false);
if (ourPath != null) {
client.delete().guaranteed().inBackground().forPath(ourPath);
}
ourPath =
client
.create()
.withProtection()
.withMode(CreateMode.EPHEMERAL_SEQUENTIAL)
.forPath(ZKPaths.makePath(latchPath, LOCK_NAME), LeaderSelector.getIdBytes(id));
checkForLeadership();
}
private List<ServerNode> doNodesDiscovery() throws Exception {
if (logger_ != null) {
if (logger_.isInfoEnabled()) logger_.info("nodes discovery start...");
}
List<String> serverNodes = registerPathNodeDataChangeWatcher(zkNodePath_);
if (serverNodes == null) serverNodes = new ArrayList<>();
List<ServerNode> newestServerNodes = new ArrayList<>();
for (final String nodeKey : serverNodes) {
final byte[] nodeValueBytes =
zkClient_.getData().forPath(String.format("%s/%s", zkNodePath_, nodeKey));
if (nodeValueBytes == null || nodeValueBytes.length <= 0) continue;
final String nodeValueInfo = new String(nodeValueBytes, ZK_CHAR_SET);
final ServerNodeInfo serverNode = JSONObject.parseObject(nodeValueInfo, ServerNodeInfo.class);
newestServerNodes.add(serverNode);
}
if (serverNodes != null) serverNodes.clear();
synchronized (this.serverNodes_) {
this.serverNodes_.clear();
this.serverNodes_.addAll(newestServerNodes);
if (logger_ != null) {
if (logger_.isInfoEnabled())
logger_.info(
String.format(
"current discovery's server nodes count (%d)", this.serverNodes_.size()));
}
}
if (logger_ != null) {
if (logger_.isInfoEnabled()) logger_.info("nodes discovery end...");
}
return this.nodes();
}
/**
* Change the shared value value irrespective of its previous state
*
* @param newValue new value
* @throws Exception ZK errors, interruptions, etc.
*/
public void setValue(byte[] newValue) throws Exception {
Preconditions.checkState(state.get() == State.STARTED, "not started");
client.setData().forPath(path, newValue);
stat.setVersion(stat.getVersion() + 1);
value = Arrays.copyOf(newValue, newValue.length);
}
@VisibleForTesting
void reset() throws Exception {
setLeadership(false);
setNode(null);
BackgroundCallback callback =
new BackgroundCallback() {
@Override
public void processResult(CuratorFramework client, CuratorEvent event) throws Exception {
if (debugResetWaitLatch != null) {
debugResetWaitLatch.await();
debugResetWaitLatch = null;
}
if (event.getResultCode() == KeeperException.Code.OK.intValue()) {
setNode(event.getName());
getChildren();
} else {
log.error("getChildren() failed. rc = " + event.getResultCode());
}
}
};
client
.create()
.creatingParentsIfNeeded()
.withProtection()
.withMode(CreateMode.EPHEMERAL_SEQUENTIAL)
.inBackground(callback)
.forPath(ZKPaths.makePath(latchPath, LOCK_NAME), LeaderSelector.getIdBytes(id));
}
/**
* Add this instance to the leadership election and attempt to acquire leadership.
*
* @throws Exception errors
*/
public void start() throws Exception {
Preconditions.checkState(
state.compareAndSet(State.LATENT, State.STARTED), "Cannot be started more than once");
client.getConnectionStateListenable().addListener(listener);
reset();
}
@Test
public void testCheckVersion() throws Exception {
CuratorFramework client =
CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
client.start();
try {
client.create().forPath("/foo");
Stat stat = client.setData().forPath("/foo", "new".getBytes()); // up the version
try {
client
.inTransaction()
.check()
.withVersion(stat.getVersion() + 1)
.forPath("/foo") // force a bad version
.and()
.create()
.forPath("/bar")
.and()
.commit();
Assert.fail();
} catch (KeeperException.BadVersionException correct) {
// correct
}
Assert.assertNull(client.checkExists().forPath("/bar"));
} finally {
client.close();
}
}
/**
* Attempt to delete the lock node so that sequence numbers get reset
*
* @throws Exception errors
*/
public void clean() throws Exception {
try {
client.delete().forPath(basePath);
} catch (KeeperException.BadVersionException ignore) {
// ignore - another thread/process got the lock
} catch (KeeperException.NotEmptyException ignore) {
// ignore - other threads/processes are waiting
}
}
/**
* 提取获取全局序列操作通用流程
*
* @param schema
* @param seqName
* @param count
* @param operation
* @return
*/
private long commonGetSeqVal(
String schema, String seqName, Long count, FETCH_OPERATION operation) {
long id = -1;
// 节点路径
String idPath = String.format("%s/%s/%s", ROOT_PATH, schema, seqName);
try {
if (checkNodeExisted(idPath, client)) {
client.sync(idPath, null); // 获取前，先从leader那里同步 !! 异步操作，但是有序 !!
DistributedAtomicLong dal = buildDAL(idPath, true);
switch (operation) {
// 获取当前全局序列值
case CURR:
AtomicValue<Long> currValue = dal.get();
if (currValue.succeeded()) {
id = currValue.postValue();
} else {
throw new AmoebaRuntimeException("fetch from id server error");
}
break;
// 获取下一个全局序列
case NEXT:
AtomicValue<Long> nextValue = dal.increment();
if (nextValue.succeeded()) {
id = nextValue.postValue();
} else {
throw new AmoebaRuntimeException("fetch from id server error");
}
break;
// 批量获取全局序列
case BATCH:
AtomicValue<Long> startValue = dal.get();
id = startValue.postValue() + count;
if (startValue.succeeded()) {
dal.forceSet(id);
}
break;
default:
throw new AmoebaRuntimeException("not support this fetch method");
}
} else {
throw new AmoebaRuntimeException(String.format("sequence %s is not existed", seqName));
}
} catch (Exception e) {
throw new AmoebaRuntimeException(e.getMessage());
}
return id;
}
@Test
public void testRegisterNodeManager() throws Exception {
NodeId nodeId = createTestNodeId();
dragonZK.registerNodeManager(nodeId);
Stat stat = zkClient.checkExists().forPath("/dragon/nodemanagers/ServerHost:2345");
assertNotNull(stat);
}
@Test
public void testInCluster() throws Exception {
final int PARTICIPANT_QTY = 3;
List<ClientAndLatch> clients = Lists.newArrayList();
Timing timing = new Timing();
TestingCluster cluster = new TestingCluster(PARTICIPANT_QTY);
try {
cluster.start();
List<InstanceSpec> instances = Lists.newArrayList(cluster.getInstances());
for (int i = 0; i < PARTICIPANT_QTY; ++i) {
CuratorFramework client =
CuratorFrameworkFactory.newClient(
instances.get(i).getConnectString(),
timing.session(),
timing.connection(),
new RetryOneTime(1));
LeaderLatch latch = new LeaderLatch(client, "/latch");
clients.add(new ClientAndLatch(client, latch, i));
client.start();
latch.start();
}
ClientAndLatch leader = waitForALeader(clients, timing);
Assert.assertNotNull(leader);
cluster.killServer(instances.get(leader.index));
Thread.sleep(timing.multiple(2).session());
leader = waitForALeader(clients, timing);
Assert.assertNotNull(leader);
Assert.assertEquals(getLeaders(clients).size(), 1);
} finally {
for (ClientAndLatch client : clients) {
Closeables.close(client.latch, true);
Closeables.close(client.client, true);
}
Closeables.close(cluster, true);
}
}
private byte[] get(final String pathSuffix) throws Exception {
return RetryLoop.callWithRetry(
zk.getZookeeperClient(),
new Callable<byte[]>() {
@Override
public byte[] call() throws Exception {
return zk.getData().forPath(buildZookeeperPath(pathSuffix));
}
});
}
private boolean getCurrentValue(MutableAtomicValue<byte[]> result, Stat stat) throws Exception {
boolean createIt = false;
try {
result.preValue = client.getData().storingStatIn(stat).forPath(path);
} catch (KeeperException.NoNodeException e) {
result.preValue = null;
createIt = true;
}
return createIt;
}
// 检查节点是否已经存在
private synchronized boolean checkNodeExisted(String path, CuratorFramework client)
throws Exception {
Stat stat = client.checkExists().forPath(path);
if (stat != null) {
return true;
}
return false;
}
private void watch(final String pathSuffix, final Watcher callback) throws Exception {
RetryLoop.callWithRetry(
zk.getZookeeperClient(),
new Callable<Void>() {
@Override
public Void call() throws Exception {
zk.checkExists().usingWatcher(callback).forPath(buildZookeeperPath(pathSuffix));
return null;
}
});
}
@Test
public void testCreateShufflePath() throws Exception {
ApplicationId appId = createTestAppId();
JobId jobId = JobId.newJobId(appId, 123456);
dragonZK.createShufflePath(jobId);
Stat stat = zkClient.checkExists().forPath("/dragon/job_458495849584_89894985_123456/shuffle");
assertNotNull(stat);
}
/** Amoeba启动的时候，初始化zookeeper客户端，另外，给zookeeper预热 */
public void init() throws Exception {
Properties props = Utils.readGlobalSeqConfigProps();
String zkHosts = props.getProperty("zkHosts", "127.0.0.1:2181");
Integer connTimeOut = Integer.valueOf(props.getProperty("zkConnTimeOut", "60"));
log.info(String.format("zookeeper config: connect string= %s", zkHosts));
log.info(String.format("zookeeper config: connect timeout= %d seconds", connTimeOut));
client =
CuratorFrameworkFactory.builder()
.connectString(zkHosts)
.retryPolicy(new ExponentialBackoffRetry(BASE_SLEEP_TIME, RETRY_TIMES))
.connectionTimeoutMs(connTimeOut * 1000)
.build(); // 时间要由秒转成毫秒
client.start();
// 只是为client热身而调用
client.checkExists().forPath(ROOT_PATH);
}
private void getChildren() throws Exception {
BackgroundCallback callback =
new BackgroundCallback() {
@Override
public void processResult(CuratorFramework client, CuratorEvent event) throws Exception {
if (event.getResultCode() == KeeperException.Code.OK.intValue()) {
checkLeadership(event.getChildren());
}
}
};
client.getChildren().inBackground(callback).forPath(latchPath);
}