コード例 #1
0
 public void close() throws IOException {
   synchronized (lock) {
     if (zkClient == null) {
       logger.info("cannot shutdown already shutdown topic event watcher ");
       return;
     }
     stopWatchingTopicEvents();
     zkClient.close();
     zkClient = null;
   }
 }
コード例 #2
0
  private void startWatchingTopicEvents() {
    ZkTopicEventListener topicEventListener = new ZkTopicEventListener();
    ZkUtils.makeSurePersistentPathExists(zkClient, ZkUtils.BrokerTopicsPath);
    List<String> topics =
        zkClient.subscribeChildChanges(ZkUtils.BrokerTopicsPath, topicEventListener);

    // cal to bootstrap topic list
    try {
      topicEventListener.handleChildChange(ZkUtils.BrokerTopicsPath, topics);
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
コード例 #3
0
  private <T> Map<String, List<MessageStream<T>>> consume(
      Map<String, Integer> topicCountMap, Decoder<T> decoder) {
    if (topicCountMap == null) {
      throw new IllegalArgumentException("topicCountMap is null");
    }
    //
    ZkGroupDirs dirs = new ZkGroupDirs(config.getGroupId());
    Map<String, List<MessageStream<T>>> ret = new HashMap<String, List<MessageStream<T>>>();
    String consumerUuid = config.getConsumerId();
    if (consumerUuid == null) {
      consumerUuid = generateConsumerId();
    }
    logger.info(
        format(
            "create message stream by consumerid [%s] with groupid [%s]",
            consumerUuid, config.getGroupId()));
    //
    // consumerIdString => groupid_consumerid
    final String consumerIdString = config.getGroupId() + "_" + consumerUuid;
    final TopicCount topicCount = new TopicCount(consumerIdString, topicCountMap);

    // 查询一个主题消费者数
    // 遍历主题和消费者数
    for (Map.Entry<String, Set<String>> e : topicCount.getConsumerThreadIdsPerTopic().entrySet()) {
      final String topic = e.getKey();
      final Set<String> threadIdSet = e.getValue();
      final List<MessageStream<T>> streamList = new ArrayList<MessageStream<T>>();
      for (String threadId : threadIdSet) {

        LinkedBlockingQueue<FetchedDataChunk> stream =
            new LinkedBlockingQueue<FetchedDataChunk>(config.getMaxQueuedChunks());
        queues.put(new StringTuple(topic, threadId), stream);
        streamList.add(new MessageStream<T>(topic, stream, config.getConsumerTimeoutMs(), decoder));
      }
      ret.put(topic, streamList);
      logger.debug("adding topic " + topic + " and stream to map.");
    }
    //
    // listener to consumer and partition changes
    ZKRebalancerListener<T> loadBalancerListener =
        new ZKRebalancerListener<T>(config.getGroupId(), consumerIdString, ret);

    this.rebalancerListeners.add(loadBalancerListener);
    loadBalancerListener.start();
    registerConsumerInZK(dirs, consumerIdString, topicCount);
    //
    // register listener for session expired event
    zkClient.subscribeStateChanges(
        new ZKSessionExpireListener<T>(dirs, consumerIdString, topicCount, loadBalancerListener));
    zkClient.subscribeChildChanges(dirs.consumerRegistryDir, loadBalancerListener);
    //
    for (String topic : ret.keySet()) {
      // register on broker partition path changes
      final String partitionPath = ZkUtils.BrokerTopicsPath + "/" + topic;
      zkClient.subscribeChildChanges(partitionPath, loadBalancerListener);
    }

    // explicitly grigger load balancing for this consumer
    loadBalancerListener.syncedRebalance();
    return ret;
  }