private void startWatchingTopicEvents() {
ZkTopicEventListener topicEventListener = new ZkTopicEventListener();
ZkUtils.makeSurePersistentPathExists(zkClient, ZkUtils.BrokerTopicsPath);
List<String> topics =
zkClient.subscribeChildChanges(ZkUtils.BrokerTopicsPath, topicEventListener);
// cal to bootstrap topic list
try {
topicEventListener.handleChildChange(ZkUtils.BrokerTopicsPath, topics);
} catch (Exception e) {
e.printStackTrace();
}
}
private <T> Map<String, List<MessageStream<T>>> consume(
Map<String, Integer> topicCountMap, Decoder<T> decoder) {
if (topicCountMap == null) {
throw new IllegalArgumentException("topicCountMap is null");
}
//
ZkGroupDirs dirs = new ZkGroupDirs(config.getGroupId());
Map<String, List<MessageStream<T>>> ret = new HashMap<String, List<MessageStream<T>>>();
String consumerUuid = config.getConsumerId();
if (consumerUuid == null) {
consumerUuid = generateConsumerId();
}
logger.info(
format(
"create message stream by consumerid [%s] with groupid [%s]",
consumerUuid, config.getGroupId()));
//
// consumerIdString => groupid_consumerid
final String consumerIdString = config.getGroupId() + "_" + consumerUuid;
final TopicCount topicCount = new TopicCount(consumerIdString, topicCountMap);
// 查询一个主题消费者数
// 遍历主题和消费者数
for (Map.Entry<String, Set<String>> e : topicCount.getConsumerThreadIdsPerTopic().entrySet()) {
final String topic = e.getKey();
final Set<String> threadIdSet = e.getValue();
final List<MessageStream<T>> streamList = new ArrayList<MessageStream<T>>();
for (String threadId : threadIdSet) {
LinkedBlockingQueue<FetchedDataChunk> stream =
new LinkedBlockingQueue<FetchedDataChunk>(config.getMaxQueuedChunks());
queues.put(new StringTuple(topic, threadId), stream);
streamList.add(new MessageStream<T>(topic, stream, config.getConsumerTimeoutMs(), decoder));
}
ret.put(topic, streamList);
logger.debug("adding topic " + topic + " and stream to map.");
}
//
// listener to consumer and partition changes
ZKRebalancerListener<T> loadBalancerListener =
new ZKRebalancerListener<T>(config.getGroupId(), consumerIdString, ret);
this.rebalancerListeners.add(loadBalancerListener);
loadBalancerListener.start();
registerConsumerInZK(dirs, consumerIdString, topicCount);
//
// register listener for session expired event
zkClient.subscribeStateChanges(
new ZKSessionExpireListener<T>(dirs, consumerIdString, topicCount, loadBalancerListener));
zkClient.subscribeChildChanges(dirs.consumerRegistryDir, loadBalancerListener);
//
for (String topic : ret.keySet()) {
// register on broker partition path changes
final String partitionPath = ZkUtils.BrokerTopicsPath + "/" + topic;
zkClient.subscribeChildChanges(partitionPath, loadBalancerListener);
}
// explicitly grigger load balancing for this consumer
loadBalancerListener.syncedRebalance();
return ret;
}