public void commitOffsets() { Xnd.logConsumer("AutoCommitTask自动提交执行..."); if (zkClient == null) { logger.error("zk client is null. Cannot commit offsets"); return; } for (Entry<String, Pool<Partition, PartitionTopicInfo>> e : topicRegistry.entrySet()) { ZkGroupTopicDirs topicDirs = new ZkGroupTopicDirs(config.getGroupId(), e.getKey()); // for (PartitionTopicInfo info : e.getValue().values()) { final long lastChanged = info.getConsumedOffsetChanged().get(); if (lastChanged == 0) { logger.trace("consume offset not changed"); continue; } final long newOffset = info.getConsumedOffset(); // path: /consumers/<group>/offsets/<topic>/<brokerid-partition> final String path = topicDirs.consumerOffsetDir + "/" + info.partition.getName(); try { ZkUtils.updatePersistentPath(zkClient, path, "" + newOffset); } catch (Throwable t) { logger.warn("exception during commitOffsets, path=" + path + ",offset=" + newOffset, t); } finally { info.resetComsumedOffsetChanged(lastChanged); if (logger.isDebugEnabled()) { logger.debug("Committed [" + path + "] for topic " + info); } } } // } }
private <T> Map<String, List<MessageStream<T>>> consume( Map<String, Integer> topicCountMap, Decoder<T> decoder) { if (topicCountMap == null) { throw new IllegalArgumentException("topicCountMap is null"); } // ZkGroupDirs dirs = new ZkGroupDirs(config.getGroupId()); Map<String, List<MessageStream<T>>> ret = new HashMap<String, List<MessageStream<T>>>(); String consumerUuid = config.getConsumerId(); if (consumerUuid == null) { consumerUuid = generateConsumerId(); } logger.info( format( "create message stream by consumerid [%s] with groupid [%s]", consumerUuid, config.getGroupId())); // // consumerIdString => groupid_consumerid final String consumerIdString = config.getGroupId() + "_" + consumerUuid; final TopicCount topicCount = new TopicCount(consumerIdString, topicCountMap); // 查询一个主题消费者数 // 遍历主题和消费者数 for (Map.Entry<String, Set<String>> e : topicCount.getConsumerThreadIdsPerTopic().entrySet()) { final String topic = e.getKey(); final Set<String> threadIdSet = e.getValue(); final List<MessageStream<T>> streamList = new ArrayList<MessageStream<T>>(); for (String threadId : threadIdSet) { LinkedBlockingQueue<FetchedDataChunk> stream = new LinkedBlockingQueue<FetchedDataChunk>(config.getMaxQueuedChunks()); queues.put(new StringTuple(topic, threadId), stream); streamList.add(new MessageStream<T>(topic, stream, config.getConsumerTimeoutMs(), decoder)); } ret.put(topic, streamList); logger.debug("adding topic " + topic + " and stream to map."); } // // listener to consumer and partition changes ZKRebalancerListener<T> loadBalancerListener = new ZKRebalancerListener<T>(config.getGroupId(), consumerIdString, ret); this.rebalancerListeners.add(loadBalancerListener); loadBalancerListener.start(); registerConsumerInZK(dirs, consumerIdString, topicCount); // // register listener for session expired event zkClient.subscribeStateChanges( new ZKSessionExpireListener<T>(dirs, consumerIdString, topicCount, loadBalancerListener)); zkClient.subscribeChildChanges(dirs.consumerRegistryDir, loadBalancerListener); // for (String topic : ret.keySet()) { // register on broker partition path changes final String partitionPath = ZkUtils.BrokerTopicsPath + "/" + topic; zkClient.subscribeChildChanges(partitionPath, loadBalancerListener); } // explicitly grigger load balancing for this consumer loadBalancerListener.syncedRebalance(); return ret; }