@Test public void testCheckVersion() throws Exception { CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1)); client.start(); try { client.create().forPath("/foo"); Stat stat = client.setData().forPath("/foo", "new".getBytes()); // up the version try { client .inTransaction() .check() .withVersion(stat.getVersion() + 1) .forPath("/foo") // force a bad version .and() .create() .forPath("/bar") .and() .commit(); Assert.fail(); } catch (KeeperException.BadVersionException correct) { // correct } Assert.assertNull(client.checkExists().forPath("/bar")); } finally { client.close(); } }
@Test public void testNamespace() throws Exception { Timing timing = new Timing(); ChildReaper reaper = null; CuratorFramework client = CuratorFrameworkFactory.builder() .connectString(server.getConnectString()) .sessionTimeoutMs(timing.session()) .connectionTimeoutMs(timing.connection()) .retryPolicy(new RetryOneTime(1)) .namespace("foo") .build(); try { client.start(); for (int i = 0; i < 10; ++i) { client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i)); } reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1); reaper.start(); timing.forWaiting().sleepABit(); Stat stat = client.checkExists().forPath("/test"); Assert.assertEquals(stat.getNumChildren(), 0); stat = client.usingNamespace(null).checkExists().forPath("/foo/test"); Assert.assertNotNull(stat); Assert.assertEquals(stat.getNumChildren(), 0); } finally { Closeables.closeQuietly(reaper); Closeables.closeQuietly(client); } }
@Test public void testBasic() throws Exception { CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1)); client.start(); try { Collection<CuratorTransactionResult> results = client .inTransaction() .create() .forPath("/foo") .and() .create() .forPath("/foo/bar", "snafu".getBytes()) .and() .commit(); Assert.assertTrue(client.checkExists().forPath("/foo/bar") != null); Assert.assertEquals(client.getData().forPath("/foo/bar"), "snafu".getBytes()); CuratorTransactionResult fooResult = Iterables.find( results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/foo")); CuratorTransactionResult fooBarResult = Iterables.find( results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/foo/bar")); Assert.assertNotNull(fooResult); Assert.assertNotNull(fooBarResult); Assert.assertNotSame(fooResult, fooBarResult); Assert.assertEquals(fooResult.getResultPath(), "/foo"); Assert.assertEquals(fooBarResult.getResultPath(), "/foo/bar"); } finally { client.close(); } }
@Test public void testUpdateShuffleNode() throws Exception { JobId jobId = JobId.newJobId(createTestAppId(), 123456); DragonZooKeeper.NodeData node = new DragonZooKeeper.NodeData(); node.nodeId = createTestNodeId(); node.taskId = TaskId.newTaskId(jobId, 34565, TaskType.MAP); dragonZK.createShufflePath(jobId); dragonZK.createShuffleNode(jobId, newArrayList(node)); node.nodeId.setHost("NewServerHost"); dragonZK.updateShuffleNode(jobId, newArrayList(node)); Stat stat = zkClient .checkExists() .forPath( "/dragon/job_458495849584_89894985_123456/shuffle/task_458495849584_89894985_123456_m_34565"); assertNotNull(stat); byte[] data = zkClient .getData() .forPath( "/dragon/job_458495849584_89894985_123456/shuffle/task_458495849584_89894985_123456_m_34565"); assertEquals("NewServerHost:2345", new String(data)); }
@Test public void testSomeNodes() throws Exception { Timing timing = new Timing(); ChildReaper reaper = null; CuratorFramework client = CuratorFrameworkFactory.newClient( server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1)); try { client.start(); Random r = new Random(); int nonEmptyNodes = 0; for (int i = 0; i < 10; ++i) { client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i)); if (r.nextBoolean()) { client.create().forPath("/test/" + Integer.toString(i) + "/foo"); ++nonEmptyNodes; } } reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1); reaper.start(); timing.forWaiting().sleepABit(); Stat stat = client.checkExists().forPath("/test"); Assert.assertEquals(stat.getNumChildren(), nonEmptyNodes); } finally { Closeables.closeQuietly(reaper); Closeables.closeQuietly(client); } }
/** * Try to atomically update a node in ZooKeeper, creating it if it doesn't exist. This is meant to * be used within an optimistic concurrency model. * * @param pathSuffix suffix to use to build path in ZooKeeper. * @param f function used to initialize the node, or transform the data already there. * @return true if node was created/updated, false if a concurrent modification occurred and * succeeded while trying to update/create the node. * @throws Exception */ private boolean tryAtomicUpdate(final String pathSuffix, final NodeFunction f) throws Exception { final String path = buildZookeeperPath(pathSuffix); final Stat stat = zk.checkExists().forPath(path); if (stat == null) { try { zk.create() .creatingParentsIfNeeded() .withMode(CreateMode.PERSISTENT) .forPath(path, f.initialize()); } catch (KeeperException.NodeExistsException e) { LOG.debug("Concurrent creation of " + path + ", retrying", e); return false; } } else { Mod<byte[]> newVal = f.apply(zk.getData().forPath(path)); if (newVal.hasModification()) { try { zk.setData().withVersion(stat.getVersion()).forPath(path, newVal.get()); } catch (KeeperException.BadVersionException e) { LOG.debug("Concurrent update to " + path + ", retrying.", e); return false; } } } return true; }
@Test public void testWithNamespace() throws Exception { CuratorFramework client = CuratorFrameworkFactory.builder() .connectString(server.getConnectString()) .retryPolicy(new RetryOneTime(1)) .namespace("galt") .build(); client.start(); try { Collection<CuratorTransactionResult> results = client .inTransaction() .create() .forPath("/foo", "one".getBytes()) .and() .create() .withMode(CreateMode.PERSISTENT_SEQUENTIAL) .forPath("/test-", "one".getBytes()) .and() .setData() .forPath("/foo", "two".getBytes()) .and() .create() .forPath("/foo/bar") .and() .delete() .forPath("/foo/bar") .and() .commit(); Assert.assertTrue(client.checkExists().forPath("/foo") != null); Assert.assertTrue(client.nonNamespaceView().checkExists().forPath("/galt/foo") != null); Assert.assertEquals(client.getData().forPath("/foo"), "two".getBytes()); Assert.assertTrue(client.checkExists().forPath("/foo/bar") == null); CuratorTransactionResult ephemeralResult = Iterables.find( results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/test-")); Assert.assertNotNull(ephemeralResult); Assert.assertNotEquals(ephemeralResult.getResultPath(), "/test-"); Assert.assertTrue(ephemeralResult.getResultPath().startsWith("/test-")); } finally { client.close(); } }
@Test public void testRegisterNodeManager() throws Exception { NodeId nodeId = createTestNodeId(); dragonZK.registerNodeManager(nodeId); Stat stat = zkClient.checkExists().forPath("/dragon/nodemanagers/ServerHost:2345"); assertNotNull(stat); }
// 检查节点是否已经存在 private synchronized boolean checkNodeExisted(String path, CuratorFramework client) throws Exception { Stat stat = client.checkExists().forPath(path); if (stat != null) { return true; } return false; }
@Test public void testCreateShufflePath() throws Exception { ApplicationId appId = createTestAppId(); JobId jobId = JobId.newJobId(appId, 123456); dragonZK.createShufflePath(jobId); Stat stat = zkClient.checkExists().forPath("/dragon/job_458495849584_89894985_123456/shuffle"); assertNotNull(stat); }
private boolean internalLockLoop(long startMillis, Long millisToWait, String ourPath) throws Exception { if (revocable.get() != null) { client.getData().usingWatcher(revocableWatcher).forPath(ourPath); } boolean haveTheLock = false; boolean doDelete = false; try { while (client.isStarted() && !haveTheLock) { List<String> children = getSortedChildren(); String sequenceNodeName = ourPath.substring(basePath.length() + 1); // +1 to include the slash PredicateResults predicateResults = driver.getsTheLock(client, children, sequenceNodeName, maxLeases); if (predicateResults.getsTheLock()) { haveTheLock = true; } else { String previousSequencePath = basePath + "/" + predicateResults.getPathToWatch(); synchronized (this) { Stat stat = client.checkExists().usingWatcher(watcher).forPath(previousSequencePath); if (stat != null) { if (millisToWait != null) { millisToWait -= (System.currentTimeMillis() - startMillis); startMillis = System.currentTimeMillis(); if (millisToWait <= 0) { doDelete = true; // timed out - delete our node break; } wait(millisToWait); } else { wait(); } } } // else it may have been deleted (i.e. lock released). Try to acquire again } } } catch (KeeperException e) { // ignore this and let the retry policy handle it throw e; } catch (Exception e) { doDelete = true; throw e; } finally { if (doDelete) { client.delete().guaranteed().forPath(ourPath); } } return haveTheLock; }
private void checkLeadership(List<String> children) throws Exception { final String localOurPath = ourPath.get(); List<String> sortedChildren = LockInternals.getSortedChildren(LOCK_NAME, sorter, children); int ourIndex = (localOurPath != null) ? sortedChildren.indexOf(ZKPaths.getNodeFromPath(localOurPath)) : -1; if (ourIndex < 0) { log.error("Can't find our node. Resetting. Index: " + ourIndex); reset(); } else if (ourIndex == 0) { setLeadership(true); } else { String watchPath = sortedChildren.get(ourIndex - 1); Watcher watcher = new Watcher() { @Override public void process(WatchedEvent event) { if ((state.get() == State.STARTED) && (event.getType() == Event.EventType.NodeDeleted) && (localOurPath != null)) { try { getChildren(); } catch (Exception ex) { log.error("An error occurred checking the leadership.", ex); } } } }; BackgroundCallback callback = new BackgroundCallback() { @Override public void processResult(CuratorFramework client, CuratorEvent event) throws Exception { if (event.getResultCode() == KeeperException.Code.NONODE.intValue()) { // previous node is gone - reset reset(); } } }; client .checkExists() .usingWatcher(watcher) .inBackground(callback) .forPath(ZKPaths.makePath(latchPath, watchPath)); } }
/** Amoeba启动的时候,初始化zookeeper客户端,另外,给zookeeper预热 */ public void init() throws Exception { Properties props = Utils.readGlobalSeqConfigProps(); String zkHosts = props.getProperty("zkHosts", "127.0.0.1:2181"); Integer connTimeOut = Integer.valueOf(props.getProperty("zkConnTimeOut", "60")); log.info(String.format("zookeeper config: connect string= %s", zkHosts)); log.info(String.format("zookeeper config: connect timeout= %d seconds", connTimeOut)); client = CuratorFrameworkFactory.builder() .connectString(zkHosts) .retryPolicy(new ExponentialBackoffRetry(BASE_SLEEP_TIME, RETRY_TIMES)) .connectionTimeoutMs(connTimeOut * 1000) .build(); // 时间要由秒转成毫秒 client.start(); // 只是为client热身而调用 client.checkExists().forPath(ROOT_PATH); }
private void checkForLeadership() throws Exception { List<String> sortedChildren = LockInternals.getSortedChildren(client, latchPath, LOCK_NAME, sorter); if (sortedChildren.size() == 0) { throw new Exception("no children - unexpected state"); } int ourIndex = sortedChildren.indexOf(ZKPaths.getNodeFromPath(ourPath)); if (ourIndex == 0) { setLeadership(true); } else { final String ourPathWhenWatched = ourPath; // protected against a lost/suspended connection and an old watcher - I'm not // sure if this is possible but it can't hurt String watchPath = sortedChildren.get(ourIndex - 1); Watcher watcher = new Watcher() { @Override public void process(WatchedEvent event) { if ((event.getType() == Event.EventType.NodeDeleted) && (ourPath != null) && ourPath.equals(ourPathWhenWatched)) { try { checkForLeadership(); } catch (Exception ex) { log.error("An error ocurred checking the leadership.", ex); } } } }; if (client.checkExists().usingWatcher(watcher).forPath(ZKPaths.makePath(latchPath, watchPath)) == null) { // the previous Participant may be down, so we need to reevaluate the list // to get the actual previous Participant or get the leadership checkForLeadership(); } } }
@Test public void testGetShuffleNodeByTaskId() throws Exception { JobId jobId = JobId.newJobId(createTestAppId(), 123456); TaskId taskId = TaskId.newTaskId(jobId, 34565, TaskType.MAP); DragonZooKeeper.NodeData node = new DragonZooKeeper.NodeData(); node.nodeId = createTestNodeId(); node.taskId = taskId; if (zkClient.checkExists().forPath("/dragon/job_458495849584_89894985_123456/shuffle") == null) { dragonZK.createShufflePath(jobId); } PathChildrenCache shuffleNodeCache = new PathChildrenCache(zkClient, dragonZK.getShufflePath(jobId), true); dragonZK.setShuffleNodeCache(shuffleNodeCache); dragonZK.createShuffleNode(jobId, newArrayList(node)); Thread.sleep(1000); NodeId nodeId = dragonZK.getShuffleNodeByTaskId(jobId, taskId); assertNotNull(nodeId); assertEquals(node.nodeId, nodeId); }