public void addServer(
String zookeeperInstance,
String joiningServerIp,
String joiningServerZookeeperPort,
Integer myId,
FutureCallback callback) {
AsyncCallback.DataCallback dataCallback = getDataCallback(callback);
try (CuratorFramework zkClient = connectToZookeeper(zookeeperInstance)) {
List<String> joiningServers = new ArrayList<>();
String serverStr =
"server." + myId + "=" + joiningServerIp + ":2888:3888;" + joiningServerZookeeperPort;
logger.info("Adding server: " + serverStr);
joiningServers.add(serverStr);
LinkedList<Integer> results = new LinkedList<Integer>();
zkClient
.getZookeeperClient()
.getZooKeeper()
.reconfig(joiningServers, null, null, -1, dataCallback, results);
} catch (Exception e) {
logger.error("Rethrowing error ", e);
throw new RuntimeException(e);
}
}
public static long getOffsetFromZooKeeper(
CuratorFramework curatorClient, String groupId, String topic, int partition)
throws Exception {
ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic);
String path = topicDirs.consumerOffsetDir() + "/" + partition;
curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient());
byte[] data = curatorClient.getData().forPath(path);
if (data == null) {
return OFFSET_NOT_SET;
} else {
String asString = new String(data);
if (asString.length() == 0) {
return OFFSET_NOT_SET;
} else {
try {
return Long.parseLong(asString);
} catch (NumberFormatException e) {
throw new Exception(
String.format(
"The offset in ZooKeeper for group '%s', topic '%s', partition %d is a malformed string: %s",
groupId, topic, partition, asString));
}
}
}
}
/**
* 连接
*
* @return
*/
public boolean connect() {
try {
client
.getConnectionStateListenable()
.addListener(
new ConnectionStateListener() {
public void stateChanged(CuratorFramework framework, ConnectionState state) {
logger.info("Zookeeper状态：" + state.name());
if (state.equals(ConnectionState.RECONNECTED)) {
logger.info("Zookeeper状态：检查节点");
for (String key : backup.keySet()) {
String value = backup.get(key);
createNode(key, value);
}
}
}
});
client.start();
client.getZookeeperClient().blockUntilConnectedOrTimedOut();
createRoomProcessorNode();
return true;
} catch (Exception e) {
logger.error("ZooKeeperWrapper.connect出现异常", e);
return false;
}
}
@Before
public void setUp() throws Exception {
sfb = new ZKServerFactoryBean();
delete(sfb.getDataDir());
delete(sfb.getDataLogDir());
sfb.afterPropertiesSet();
CuratorFrameworkFactory.Builder builder =
CuratorFrameworkFactory.builder()
.connectString("localhost:" + sfb.getClientPortAddress().getPort())
.retryPolicy(new RetryOneTime(1000))
.connectionTimeoutMs(360000);
curator = builder.build();
curator.start();
curator.getZookeeperClient().blockUntilConnectedOrTimedOut();
// setup a local and remote git repo
basedir = System.getProperty("basedir", ".");
File root = new File(basedir + "/target/git").getCanonicalFile();
delete(root);
new File(root, "remote").mkdirs();
remote = Git.init().setDirectory(new File(root, "remote")).call();
remote.commit().setMessage("First Commit").setCommitter("fabric", "user@fabric").call();
String remoteUrl = "file://" + new File(root, "remote").getCanonicalPath();
new File(root, "local").mkdirs();
git = Git.init().setDirectory(new File(root, "local")).call();
git.commit().setMessage("First Commit").setCommitter("fabric", "user@fabric").call();
StoredConfig config = git.getRepository().getConfig();
config.setString("remote", "origin", "url", remoteUrl);
config.setString("remote", "origin", "fetch", "+refs/heads/*:refs/remotes/origin/*");
config.save();
DefaultRuntimeProperties sysprops = new DefaultRuntimeProperties();
sysprops.setProperty(SystemProperties.KARAF_DATA, "target/data");
FabricGitServiceImpl gitService = new FabricGitServiceImpl();
gitService.bindRuntimeProperties(sysprops);
gitService.activate();
gitService.setGitForTesting(git);
DataStoreTemplateRegistry registrationHandler = new DataStoreTemplateRegistry();
registrationHandler.activateComponent();
dataStore = new CachingGitDataStore();
dataStore.bindCurator(curator);
dataStore.bindGitService(gitService);
dataStore.bindRegistrationHandler(registrationHandler);
dataStore.bindRuntimeProperties(sysprops);
dataStore.bindConfigurer(
new Configurer() {
@Override
public <T> void configure(Map<String, ?> configuration, T target) throws Exception {}
});
Map<String, String> datastoreProperties = new HashMap<String, String>();
datastoreProperties.put(GitDataStore.GIT_REMOTE_URL, remoteUrl);
dataStore.activate(datastoreProperties);
}
public static void setOffsetInZooKeeper(
CuratorFramework curatorClient, String groupId, String topic, int partition, long offset)
throws Exception {
ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic);
String path = topicDirs.consumerOffsetDir() + "/" + partition;
curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient());
byte[] data = Long.toString(offset).getBytes();
curatorClient.setData().forPath(path, data);
}
public static void main(String[] args) {
/*
* 在curator中，CuratorFramework作为zookeeper的client接口
*/
String connectString = ZKCommons.ZK_SERVER_IP + ":" + ZKCommons.ZK_SERVER_PORT;
RetryPolicy retryPolicy = new ExponentialBackoffRetry(5000, 3);
// CuratorFrameworkFactory.newClient(connectString, 5000, 5000, retryPolicy);
final CuratorFramework client =
CuratorFrameworkFactory.builder()
.connectString(connectString)
.sessionTimeoutMs(5000)
.connectionTimeoutMs(5000)
.retryPolicy(retryPolicy)
.build();
client.start();
try {
AfterConnectionEstablished.execute(
client,
new Runnable() {
@Override
public void run() {
try {
Stat stat = new Stat();
byte[] data = client.getData().storingStatIn(stat).forPath("/");
System.out.println(new String(data));
System.out.println(stat);
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("do something after get connect");
}
});
} catch (Exception e1) {
e1.printStackTrace();
}
try {
States stat = client.getZookeeperClient().getZooKeeper().getState();
// CuratorFrameworkState stat = client.getState();
System.out.println(stat);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
Thread.sleep(Integer.MAX_VALUE);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Test
public void testZkClientLosingSession() throws Exception {
// Cut the session in the server through the client
long sessionId = zkClient.getZookeeperClient().getZooKeeper().getSessionId();
byte[] sessionPasswd = zkClient.getZookeeperClient().getZooKeeper().getSessionPasswd();
ZooKeeper zk = new ZooKeeper(ZK_CLUSTER, 1000, null, sessionId, sessionPasswd);
zk.close();
LOG.info("ZKClient session closed");
// Iterate updating the timestamp and check the final value
long previousMaxTimestamp = INITIAL_MAX_TS_VALUE;
for (int i = 0; i < ITERATION_COUNT; i++) {
long newMaxTimestamp = previousMaxTimestamp + 1_000_000;
storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
LOG.info("Updating timestamp. Previous/New {}/{}", previousMaxTimestamp, newMaxTimestamp);
previousMaxTimestamp = newMaxTimestamp;
}
assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
}
public static void main(String[] args) throws Exception {
client.start();
ZooKeeper zookeeper = client.getZookeeperClient().getZooKeeper();
System.out.println(ZKPaths.fixForNamespace(path, "sub"));
System.out.println(ZKPaths.makePath(path, "sub"));
System.out.println(ZKPaths.getNodeFromPath("/curator_zkpath_sample/sub1"));
PathAndNode pn = ZKPaths.getPathAndNode("/curator_zkpath_sample/sub1");
System.out.println(pn.getPath());
System.out.println(pn.getNode());
String dir1 = path + "/child1";
String dir2 = path + "/child2";
ZKPaths.mkdirs(zookeeper, dir1);
ZKPaths.mkdirs(zookeeper, dir2);
System.out.println(ZKPaths.getSortedChildren(zookeeper, path));
ZKPaths.deleteChildren(client.getZookeeperClient().getZooKeeper(), path, true);
}
/**
* Ensure the existence of the given path.
*
* @param client curator client
* @param path path to create, if needed
*/
public static void ensurePath(CuratorFramework client, String path) {
EnsurePath ensurePath = ensurePaths.get(path);
if (ensurePath == null) {
ensurePaths.putIfAbsent(path, client.newNamespaceAwareEnsurePath(path));
ensurePath = ensurePaths.get(path);
}
try {
ensurePath.ensure(client.getZookeeperClient());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
protected List<String> setupDefaultMaster(final int offset, String... args) throws Exception {
// TODO (dano): Move this bootstrapping to something reusable
final CuratorFramework curator = zk.curatorWithSuperAuth();
curator.newNamespaceAwareEnsurePath(Paths.configHosts()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.configJobs()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.configJobRefs()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.statusHosts()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.statusMasters()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.historyJobs()).ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(Paths.configId(zkClusterId))
.ensure(curator.getZookeeperClient());
final List<String> argsList =
Lists.newArrayList(
"-vvvv",
"--no-log-setup",
"--http",
"http://0.0.0.0:" + (masterPort() + offset),
"--admin",
"http://0.0.0.0:" + (masterAdminPort() + offset),
"--domain",
"",
"--zk",
zk.connectString(),
"--zk-enable-acls",
"--zk-acl-agent-user",
AGENT_USER,
"--zk-acl-agent-digest",
AGENT_DIGEST,
"--zk-acl-master-user",
MASTER_USER,
"--zk-acl-master-password",
MASTER_PASSWORD);
final String name;
if (asList(args).contains("--name")) {
name = args[asList(args).indexOf("--name") + 1];
} else {
name = TEST_MASTER + offset;
argsList.addAll(asList("--name", TEST_MASTER));
}
final String stateDir = masterStateDirs.resolve(name).toString();
argsList.addAll(asList("--state-dir", stateDir));
argsList.addAll(asList(args));
return argsList;
}
public void invokeCreate(String nameSpace, String path, String value, CreateMode model)
throws Exception {
Assert.notNull(path, "path不能为空！");
Assert.hasText(nameSpace, "nameSpace不能为空");
ZKPaths.mkdirs(
curatorFramework.getZookeeperClient().getZooKeeper(),
ZKPaths.fixForNamespace(nameSpace, path),
false);
curatorFramework
.usingNamespace(nameSpace)
.create()
.withMode(model)
.inBackground()
.forPath(path, value.getBytes());
}
public void removeServer(String zookeeperInstance, Integer myId, FutureCallback callback) {
AsyncCallback.DataCallback dataCallback = getDataCallback(callback);
try (CuratorFramework zkClient = connectToZookeeper(zookeeperInstance)) {
List<String> leavingServers = new ArrayList<>();
leavingServers.add(myId.toString());
logger.info("Removing server: " + myId.toString());
LinkedList<Integer> results = new LinkedList<Integer>();
zkClient
.getZookeeperClient()
.getZooKeeper()
.reconfig(null, leavingServers, null, -1, dataCallback, results);
} catch (Exception e) {
logger.error("Rethrowing error ", e);
throw new RuntimeException(e);
}
}
public boolean invokeExistsSync(String nameSpace, final String path) {
Assert.notNull(path, "path不能为空！");
Assert.hasText(nameSpace, "nameSpace不能为空");
try {
return curatorFramework.usingNamespace(nameSpace).checkExists().forPath(path) == null;
} catch (NoNodeException ex) {
logger.warn(
"ZK服务端({})没有配置节点：{}：{}",
curatorFramework.getZookeeperClient().getCurrentConnectionString(),
nameSpace,
path);
} catch (ConnectionLossException ex) {
logger.error("ZK服务器连接失败！", ex);
} catch (Exception ex) {
rethrowRuntimeException(ex);
}
return false;
}
public String getDataSync(String nameSpace, String path) {
Assert.hasText(nameSpace, "nameSpace不能为空");
try {
byte[] data = curatorFramework.usingNamespace(nameSpace).getData().forPath(path);
String dataStr = new String(data, "UTF-8");
logger.info("节点：{}:{} 的值为: {}", nameSpace, path, dataStr);
return dataStr;
} catch (NoNodeException ex) {
logger.warn(
"ZK服务端({})没有配置节点：{}：{}",
curatorFramework.getZookeeperClient().getCurrentConnectionString(),
nameSpace,
path);
} catch (ConnectionLossException ex) {
logger.error("ZK服务器连接失败！", ex);
} catch (Exception ex) {
rethrowRuntimeException(ex);
}
return null;
}
private void invokeGetChildrenData(
String nameSpace, String path, BackgroundCallback callback, Watcher watcher) {
Assert.hasText(nameSpace, "nameSpace不能为空");
try {
curatorFramework
.usingNamespace(nameSpace)
.getChildren()
.usingWatcher(watcher)
.inBackground(callback)
.forPath(path);
} catch (NoNodeException ex) {
logger.warn(
"ZK服务端({})没有配置节点：{}：{}",
curatorFramework.getZookeeperClient().getCurrentConnectionString(),
nameSpace,
path);
} catch (ConnectionLossException ex) {
logger.error("ZK服务器连接失败！", ex);
} catch (Exception ex) {
rethrowRuntimeException(ex);
}
}
@Test
public void testLocalFabricCluster() throws Exception {
fabricService = getFabricService();
// Test autostartup.
assertNotNull(fabricService);
CuratorFramework curator = getCurator();
curator.getZookeeperClient().blockUntilConnectedOrTimedOut();
Provision.containerAlive(
Arrays.<Container>asList(new ContainerImpl(null, "root", fabricService)),
PROVISION_TIMEOUT);
Container[] containers = fabricService.getContainers();
assertNotNull(containers);
assertEquals("Expected to find 1 container", 1, containers.length);
assertEquals("Expected to find the root container", "root", containers[0].getId());
// Test that a generated password exists
// We don't inject the configuration admin as it causes issues when the tracker gets closed.
ConfigurationAdmin configurationAdmin = getOsgiService(ConfigurationAdmin.class);
org.osgi.service.cm.Configuration configuration =
configurationAdmin.getConfiguration(Constants.ZOOKEEPER_CLIENT_PID);
Dictionary<String, Object> dictionary = configuration.getProperties();
assertNotNull("Expected a generated zookeeper password", dictionary.get("zookeeper.password"));
assertTrue(String.valueOf(dictionary.get("zookeeper.url")).endsWith("2182"));
}
@LifecycleStart
public void start() throws IOException {
log.info("Starting zkCoordinator for server[%s]", me);
synchronized (lock) {
if (started) {
return;
}
final String loadQueueLocation = ZKPaths.makePath(zkPaths.getLoadQueuePath(), me.getName());
final String servedSegmentsLocation =
ZKPaths.makePath(zkPaths.getServedSegmentsPath(), me.getName());
final String liveSegmentsLocation =
ZKPaths.makePath(zkPaths.getLiveSegmentsPath(), me.getName());
loadQueueCache =
new PathChildrenCache(
curator,
loadQueueLocation,
true,
true,
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ZkCoordinator-%s").build());
try {
config.getInfoDir().mkdirs();
curator.newNamespaceAwareEnsurePath(loadQueueLocation).ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(servedSegmentsLocation)
.ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(liveSegmentsLocation)
.ensure(curator.getZookeeperClient());
loadCache();
loadQueueCache
.getListenable()
.addListener(
new PathChildrenCacheListener() {
@Override
public void childEvent(CuratorFramework client, PathChildrenCacheEvent event)
throws Exception {
final ChildData child = event.getData();
switch (event.getType()) {
case CHILD_ADDED:
final String path = child.getPath();
final DataSegmentChangeRequest segment =
jsonMapper.readValue(child.getData(), DataSegmentChangeRequest.class);
log.info("New node[%s] with segmentClass[%s]", path, segment.getClass());
try {
segment.go(ZkCoordinator.this);
curator.delete().guaranteed().forPath(path);
log.info("Completed processing for node[%s]", path);
} catch (Exception e) {
try {
curator.delete().guaranteed().forPath(path);
} catch (Exception e1) {
log.info(
e1, "Failed to delete node[%s], but ignoring exception.", path);
}
log.makeAlert(e, "Segment load/unload: uncaught exception.")
.addData("node", path)
.addData("nodeProperties", segment)
.emit();
}
break;
case CHILD_REMOVED:
log.info("%s was removed", event.getData().getPath());
break;
default:
log.info("Ignoring event[%s]", event);
}
}
});
loadQueueCache.start();
} catch (Exception e) {
Throwables.propagateIfPossible(e, IOException.class);
throw Throwables.propagate(e);
}
started = true;
}
}
public boolean isLeader() {
return curatorClient.getZookeeperClient().isConnected() && leaderLatch.hasLeadership();
}
@LifecycleStart
public void start() throws IOException {
synchronized (lock) {
if (started) {
return;
}
log.info("Starting zkCoordinator for server[%s]", me.getName());
final String loadQueueLocation = ZKPaths.makePath(zkPaths.getLoadQueuePath(), me.getName());
final String servedSegmentsLocation =
ZKPaths.makePath(zkPaths.getServedSegmentsPath(), me.getName());
final String liveSegmentsLocation =
ZKPaths.makePath(zkPaths.getLiveSegmentsPath(), me.getName());
loadQueueCache = new PathChildrenCache(curator, loadQueueLocation, true, true, loadingExec);
try {
curator.newNamespaceAwareEnsurePath(loadQueueLocation).ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(servedSegmentsLocation)
.ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(liveSegmentsLocation)
.ensure(curator.getZookeeperClient());
loadLocalCache();
loadQueueCache
.getListenable()
.addListener(
new PathChildrenCacheListener() {
@Override
public void childEvent(CuratorFramework client, PathChildrenCacheEvent event)
throws Exception {
final ChildData child = event.getData();
switch (event.getType()) {
case CHILD_ADDED:
final String path = child.getPath();
final DataSegmentChangeRequest request =
jsonMapper.readValue(child.getData(), DataSegmentChangeRequest.class);
log.info("New request[%s] with zNode[%s].", request.asString(), path);
try {
request.go(
getDataSegmentChangeHandler(),
new DataSegmentChangeCallback() {
boolean hasRun = false;
@Override
public void execute() {
try {
if (!hasRun) {
curator.delete().guaranteed().forPath(path);
log.info("Completed request [%s]", request.asString());
hasRun = true;
}
} catch (Exception e) {
try {
curator.delete().guaranteed().forPath(path);
} catch (Exception e1) {
log.error(
e1,
"Failed to delete zNode[%s], but ignoring exception.",
path);
}
log.error(e, "Exception while removing zNode[%s]", path);
throw Throwables.propagate(e);
}
}
});
} catch (Exception e) {
try {
curator.delete().guaranteed().forPath(path);
} catch (Exception e1) {
log.error(
e1, "Failed to delete zNode[%s], but ignoring exception.", path);
}
log.makeAlert(e, "Segment load/unload: uncaught exception.")
.addData("node", path)
.addData("nodeProperties", request)
.emit();
}
break;
case CHILD_REMOVED:
log.info("zNode[%s] was removed", event.getData().getPath());
break;
default:
log.info("Ignoring event[%s]", event);
}
}
});
loadQueueCache.start();
} catch (Exception e) {
Throwables.propagateIfPossible(e, IOException.class);
throw Throwables.propagate(e);
}
started = true;
}
}
