@Override
public void createTestTopic(
String topic, int numberOfPartitions, int replicationFactor, Properties topicConfig) {
// create topic with one client
LOG.info("Creating topic {}", topic);
ZkClient creator = createZkClient();
AdminUtils.createTopic(creator, topic, numberOfPartitions, replicationFactor, topicConfig);
creator.close();
// validate that the topic has been created
final long deadline = System.currentTimeMillis() + 30000;
do {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
// restore interrupted state
}
List<KafkaTopicPartitionLeader> partitions =
FlinkKafkaConsumer08.getPartitionsForTopic(
Collections.singletonList(topic), standardProps);
if (partitions != null && partitions.size() > 0) {
return;
}
} while (System.currentTimeMillis() < deadline);
fail("Test topic could not be created");
}
@SuppressWarnings("unchecked")
public T poll() throws Exception {
try {
List<String> list = zkClient.getChildren(root);
if (list.size() == 0) {
return null;
}
Collections.sort(
list,
new Comparator<String>() {
public int compare(String lhs, String rhs) {
return getNodeNumber(lhs, Node_NAME).compareTo(getNodeNumber(rhs, Node_NAME));
}
});
for (String nodeName : list) {
String nodeFullPath = root.concat("/").concat(nodeName);
try {
T node = (T) zkClient.readData(nodeFullPath);
zkClient.delete(nodeFullPath);
return node;
} catch (ZkNoNodeException e) {
// ignore
}
}
return null;
} catch (Exception e) {
throw ExceptionUtil.convertToRuntimeException(e);
}
}
@Override
public void deleteTestTopic(String topic) {
LOG.info("Deleting topic {}", topic);
ZkClient zk = createZkClient();
AdminUtils.deleteTopic(zk, topic);
zk.close();
}
@Export
public void clean() {
List<String> groups = zkClient.getChildren(ZkPaths.rootPath());
for (String group : groups) {
if (zkClient.countChildren(ZkPaths.groupPath(group)) == 0) {
zkClient.delete(ZkPaths.groupPath(group));
}
}
}
@SuppressWarnings("unchecked")
public List<T> getElements() {
List<String> paths = _zkClient.getChildren(_elementsPath);
List<T> elements = new ArrayList<T>();
for (String path : paths) {
elements.add((T) _zkClient.readData(path));
}
return elements;
}
public ZooKeeperQueue(ZkClient zkClient, String rootPath, boolean isBlocking) {
_zkClient = zkClient;
_rootPath = rootPath;
_isBlocking = isBlocking;
_elementsPath = rootPath + "/queue";
if (!_zkClient.exists(rootPath)) {
_zkClient.createPersistent(rootPath, true);
_zkClient.createPersistent(_elementsPath, true);
}
}
/**
* Creates a Kafka topic if needed, or try to increase its partition count to the desired number.
*/
private Collection<Partition> ensureTopicCreated(
final String topicName, final int numPartitions, int replicationFactor) {
final int sessionTimeoutMs = 10000;
final int connectionTimeoutMs = 10000;
final ZkClient zkClient =
new ZkClient(zkAddress, sessionTimeoutMs, connectionTimeoutMs, utf8Serializer);
try {
// The following is basically copy/paste from AdminUtils.createTopic() with
// createOrUpdateTopicPartitionAssignmentPathInZK(..., update=true)
final Properties topicConfig = new Properties();
Seq<Object> brokerList = ZkUtils.getSortedBrokerList(zkClient);
final scala.collection.Map<Object, Seq<Object>> replicaAssignment =
AdminUtils.assignReplicasToBrokers(brokerList, numPartitions, replicationFactor, -1, -1);
retryOperations.execute(
new RetryCallback<Object, RuntimeException>() {
@Override
public Object doWithRetry(RetryContext context) throws RuntimeException {
AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(
zkClient, topicName, replicaAssignment, topicConfig, true);
return null;
}
});
try {
Collection<Partition> partitions =
retryOperations.execute(
new RetryCallback<Collection<Partition>, Exception>() {
@Override
public Collection<Partition> doWithRetry(RetryContext context) throws Exception {
connectionFactory.refreshMetadata(Collections.singleton(topicName));
Collection<Partition> partitions = connectionFactory.getPartitions(topicName);
if (partitions.size() < numPartitions) {
throw new IllegalStateException(
"The number of expected partitions was: "
+ numPartitions
+ ", but "
+ partitions.size()
+ " have been found instead");
}
connectionFactory.getLeaders(partitions);
return partitions;
}
});
return partitions;
} catch (Exception e) {
logger.error("Cannot initialize Binder", e);
throw new BinderException("Cannot initialize binder:", e);
}
} finally {
zkClient.close();
}
}
public LinkedHashMap<String, List<String>> listAll() {
List<String> groups = zkClient.getChildren(ZkPaths.rootPath());
LinkedHashMap<String, List<String>> result = Maps.newLinkedHashMap();
for (String group : groups) {
List<String> cells = zkClient.getChildren(ZkPaths.groupPath(group));
if (cells != null && !cells.isEmpty()) { // 过滤空 group
result.put(group, cells);
}
}
return result;
}
public static void main(String[] args) {
ZkClient zc = new ZkClient("localhost:2181", 1000);
// 创建监控节点
zc.create("/monitor/client/clientA", "clientA", CreateMode.EPHEMERAL);
try {
System.in.read();
} catch (Exception e) {
e.printStackTrace();
}
}
@Export
public List<String> listGroups() {
List<String> groups = zkClient.getChildren(ZkPaths.rootPath());
List<String> notEmptyGroups = Lists.newArrayList();
for (String group : groups) {
if (zkClient.countChildren(ZkPaths.groupPath(group)) > 0) {
notEmptyGroups.add(group);
}
}
return notEmptyGroups;
}
public static void writeData(String path, Object object, boolean createPathIfNotExist) {
try {
zkClient.writeData(path, JsonUtils.marshalToByte(object, SerializerFeature.WriteClassName));
} catch (ZkException e) {
if (createPathIfNotExist) {
createIfNotExist(path);
zkClient.writeData(path, JsonUtils.marshalToByte(object, SerializerFeature.WriteClassName));
} else {
throw e;
}
}
}
/** 使用正确的密码删除节点 */
static void deleteParent() throws Exception {
try {
zkClient = new ZkClient(SERVER_LIST, 50000);
// zkClient.addAuthInfo(authentication_type, correctAuthentication.getBytes());
if (zkClient.exists(PATH)) {
zkClient.delete(PATH);
System.out.println(PATH + "删除成功");
}
} catch (Exception e) {
e.printStackTrace();
}
}
public boolean offer(T element) throws Exception {
String nodeFullPath = root.concat("/").concat(Node_NAME);
try {
zkClient.createPersistentSequential(nodeFullPath, element);
} catch (ZkNoNodeException e) {
zkClient.createPersistent(root);
offer(element);
} catch (Exception e) {
throw ExceptionUtil.convertToRuntimeException(e);
}
return true;
}
/** 不使用密码 删除节点 */
static void deleteNodeByNoAuthentication() throws Exception {
String prefix = "[不使用任何授权信息]";
try {
System.out.println(prefix + "删除节点：" + PATH_DEL);
zkClient = new ZkClient(SERVER_LIST, 50000);
if (zkClient.exists(PATH_DEL)) {
zkClient.delete(PATH_DEL);
System.out.println(prefix + "删除成功");
}
} catch (Exception e) {
System.err.println(prefix + "删除失败，原因是：" + e.getMessage());
}
}
public static void main(String[] args) throws Exception {
// 连接zookeeper服务器
ZkClient zkClient = new ZkClient("127.0.0.1:2181");
// 读取本地spring配置
String filePath = "c:/applicationContext.xml";
// zookeeper保存spring配置的znode
String zkPath = ZOOKEEPER_NODE_PATH + "/app.xml";
ZkObject object = new ZkObject();
object.setData(readFile(filePath).trim());
System.out.println(object.getData());
// 上传配置
zkClient.writeData(zkPath, object);
zkClient.close();
System.out.println("配置上传完成");
}
/** 更新数据：不采用密码 */
static void updateDataByNoAuthentication() {
String prefix = "[不使用任何授权信息]";
System.out.println(prefix + "更新数据： " + PATH);
try {
zkClient = new ZkClient(SERVER_LIST, 50000);
if (zkClient.exists(PATH)) {
zkClient.writeData(PATH, prefix);
System.out.println(prefix + "更新成功");
}
} catch (Exception e) {
System.err.println(prefix + "更新失败，原因是：" + e.getMessage());
}
}
/** 更新数据：采用正确的密码 */
static void updateDataByCorrectAuthentication() {
String prefix = "[使用正确的授权信息]";
System.out.println(prefix + "更新数据：" + PATH);
try {
zkClient = new ZkClient(SERVER_LIST, 50000);
// zkClient.addAuthInfo(authentication_type, correctAuthentication.getBytes());
if (zkClient.exists(PATH)) {
zkClient.writeData(PATH, prefix);
System.out.println(prefix + "更新成功");
}
} catch (Exception e) {
System.err.println(prefix + "更新失败，原因是：" + e.getMessage());
}
}
/** 使用正确的密码删除节点 */
static void deleteNodeByCorrectAuthentication() throws Exception {
String prefix = "[使用正确的授权信息]";
try {
System.out.println(prefix + "删除节点：" + PATH_DEL);
zkClient = new ZkClient(SERVER_LIST, 50000);
// zkClient.addAuthInfo(authentication_type, correctAuthentication.getBytes());
if (zkClient.exists(PATH_DEL)) {
zkClient.delete(PATH_DEL);
System.out.println(prefix + "删除成功");
}
} catch (Exception e) {
System.out.println(prefix + "删除失败，原因是：" + e.getMessage());
}
}
public List<String> getChildren(String path) {
try {
return client.getChildren(path);
} catch (ZkNoNodeException e) {
return null;
}
}
public static void createIfNotExist(String path) {
try {
zkClient.createPersistent(path, true);
} catch (Exception e) {
//
}
}
public static <T> T readData(String path, Class<T> dataClass) {
byte[] data = zkClient.readData(path, true);
if (data != null && data.length != 0) {
return JsonUtils.unmarshalFromByte(data, dataClass);
}
return null;
}
/*
* (non-Javadoc)
*
* @see com.taobao.metamorphosis.client.SessionFactory#close()
*/
@Override
public void shutdown() throws MetaClientException {
if (shutdown) {
return;
}
shutdown = true;
// if (this.diamondManager != null) {
// this.diamondManager.close();
// }
recoverManager.shutdown();
// this.localMessageStorageManager.shutdown();
for (final Shutdownable child : children) {
child.shutdown();
}
try {
remotingClient.stop();
} catch (final NotifyRemotingException e) {
throw new NetworkException("Stop remoting client failed", e);
}
if (zkClient != null) {
zkClient.close();
}
if (!isHutdownHookCalled) {
Runtime.getRuntime().removeShutdownHook(shutdownHook);
}
}
@Override
public void delete(String path) {
try {
client.delete(path);
} catch (ZkNoNodeException e) {
}
}
@Override
public void createEphemeral(String path, Object data) {
try {
client.createEphemeral(path, data);
} catch (ZkNodeExistsException e) {
}
}
public String enqueue(T element) {
try {
String sequential = _zkClient.createPersistentSequential(getElementRoughPath(), element);
String elementId = sequential.substring(sequential.lastIndexOf('/') + 1);
return elementId;
} catch (Exception e) {
throw ExceptionUtil.convertToRuntimeException(e);
}
}
/** 获取数据：不采用密码 */
static void getDataByNoAuthentication() {
String prefix = "[不使用任何授权信息]";
try {
System.out.println(prefix + "获取数据：" + PATH);
zkClient = new ZkClient(SERVER_LIST, 50000);
System.out.println(prefix + "成功获取数据：" + zkClient.readData(PATH));
} catch (Exception e) {
System.err.println(prefix + "获取数据失败，原因：" + e.getMessage());
}
}
private List<String> getConnectedHosts() {
List<String> ips = zkClient.getChildren(WATCH_PATH);
for (int i = 0; i < ips.size(); i++) {
if (!isConnected(ips.get(i))) {
ips.remove(i);
i--;
}
}
return ips;
}
public ZookeeprTickCondition(String workPath, ZkClient zkClient) {
this.zkClient = zkClient;
this.workPath = ZookeeperConstant.ROOT + "/" + workPath;
zkClient.createPersistent(this.workPath, true);
zkClient.subscribeChildChanges(
this.workPath,
new IZkChildListener() {
@Override
public void handleChildChange(String parentPath, List<String> currentChilds)
throws Exception {
if (currentChilds != null && !currentChilds.isEmpty()) {
return;
}
tryOccupy();
}
});
}
@Override
public int getLeaderToShutDown(String topic) throws Exception {
ZkClient zkClient = createZkClient();
PartitionMetadata firstPart = null;
do {
if (firstPart != null) {
LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
// not the first try. Sleep a bit
Thread.sleep(150);
}
Seq<PartitionMetadata> partitionMetadata =
AdminUtils.fetchTopicMetadataFromZk(topic, zkClient).partitionsMetadata();
firstPart = partitionMetadata.head();
} while (firstPart.errorCode() != 0);
zkClient.close();
return firstPart.leader().get().id();
}
public static void main(String[] args) throws Exception {
List<ACL> acls = new ArrayList<ACL>(1);
for (ACL ids_acl : Ids.CREATOR_ALL_ACL) {
acls.add(ids_acl);
}
try {
zkClient = new ZkClient(SERVER_LIST, 50000);
// zkClient.addAuthInfo(authentication_type, correctAuthentication.getBytes());
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
// zkClient.createPersistent(PATH, acls, "init content");
System.out.println(
"使用授权key：" + correctAuthentication + "创建节点：" + PATH + ", 初始内容是: init content");
} catch (Exception e) {
e.printStackTrace();
}
try {
// zkClient.createPersistent(PATH_DEL, acls, "待删节点");
System.out.println(
"使用授权key：" + correctAuthentication + "创建节点：" + PATH_DEL + ", 初始内容是: init content");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// 获取数据
// getDataByNoAuthentication();
// getDataByBadAuthentication();
// getDataByCorrectAuthentication();
// 更新数据
// updateDataByNoAuthentication();
// updateDataByBadAuthentication();
// updateDataByCorrectAuthentication();
// 获取数据
// getDataByNoAuthentication();
// getDataByBadAuthentication();
// getDataByCorrectAuthentication();
// 删除数据
deleteNodeByBadAuthentication();
deleteNodeByNoAuthentication();
deleteNodeByCorrectAuthentication();
deleteParent();
zkClient.close();
}