/** * Ensures an initialized kafka {@link ConsumerConnector} is present. * * @param config The storm configuration passed to {@link #open(Map, TopologyContext, * SpoutOutputCollector)}. * @throws IllegalArgumentException When a required configuration parameter is missing or a sanity * check fails. */ protected void createConsumer(final Map<String, Object> config) { final Properties consumerConfig = createKafkaConfig(config); LOG.info( "connecting kafka client to zookeeper at {} as client group {}", consumerConfig.getProperty("zookeeper.connect"), consumerConfig.getProperty("group.id")); _consumer = Consumer.createJavaConsumerConnector(new ConsumerConfig(consumerConfig)); }
private static List<KafkaStream<String, String>> openKafkaStream( String zkConnect, String group, String topic) { consumer = Consumer.createJavaConsumerConnector(getConsumerConfig(zkConnect, group)); StringDecoder decoder = new StringDecoder(null); Map<String, Integer> topicCountMap = Maps.of(topic, 1); Map<String, List<KafkaStream<String, String>>> consumerMap = consumer.createMessageStreams(topicCountMap, decoder, decoder); return consumerMap.get(topic); }
Kafka_consumer() { Properties props = new Properties(); props.put("zookeeper.connect", "localhost:2181"); props.put("group.id", "0"); props.put("zookeeper.session.timeout.ms", "4000"); props.put("zookeeper.sync.time.ms", "200"); props.put("auto.commit.interval.ms", "1000"); ConsumerConfig config = new ConsumerConfig(props); // 配置consumer的值 consumer = kafka.consumer.Consumer.createJavaConsumerConnector(config); }
public KafkaMessageConsumerConnector(String group, String zkConnectUrls) { Properties props = new Properties(); props.put("group.id", group); props.put("zookeeper.connect", zkConnectUrls); props.put("zookeeper.session.timeout.ms", "400"); props.put("zookeeper.sync.time.ms", "200"); props.put("auto.commit.interval.ms", "1000"); // props.put("auto.commit.enable", "false"); props.put("auto.offset.reset", "smallest"); ConsumerConfig config = new ConsumerConfig(props); consumer = kafka.consumer.Consumer.createJavaConsumerConnector(config); }
public static VertxKafkaConsumer create( final KafkaConfiguration configuration, final KafkaHandler handler) { final Properties properties = createProperties( configuration.getZookeeper(), configuration.getGroupId(), configuration.getZookeeperTimeout()); final ConsumerConfig config = new ConsumerConfig(properties); final ConsumerConnector connector = Consumer.createJavaConsumerConnector(config); return new VertxKafkaConsumer(connector, configuration, handler); }
private void start(String topic) { consumer = Consumer.createJavaConsumerConnector(config); /* We tell Kafka how many threads will read each topic. We have one topic and one thread */ Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(topic, new Integer(1)); /* We will use a decoder to get Kafka to convert messages to Strings * valid property will be deserializer.encoding with the charset to use. * default is UTF8 which works for us */ StringDecoder decoder = new StringDecoder(new VerifiableProperties()); /* Kafka will give us a list of streams of messages for each topic. In this case, its just one topic with a list of a single stream */ stream = consumer.createMessageStreams(topicCountMap, decoder, decoder).get(topic).get(0); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { logger.info("Opened"); this.collector = collector; logger.info(" topic = " + kafkaSpoutConfig.kafkaConsumerConfiguration.getTopic()); this.spoutName = String.format("%s-%d", context.getThisComponentId(), context.getThisTaskId()); Properties kafkaProperties = KafkaConsumerProperties.createKafkaProperties(kafkaSpoutConfig.kafkaConsumerConfiguration); // Have to use a different consumer.id for each spout so use the storm taskId. Otherwise, // zookeeper complains about a conflicted ephemeral node when there is more than one spout // reading from a topic kafkaProperties.setProperty("consumer.id", String.valueOf(context.getThisTaskId())); ConsumerConfig consumerConfig = new ConsumerConfig(kafkaProperties); this.consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig); }
@Override public Firehose connect() throws IOException { final ConsumerConnector connector = Consumer.createJavaConsumerConnector(new ConsumerConfig(consumerProps)); final Map<String, List<KafkaStream<Message>>> streams = connector.createMessageStreams(ImmutableMap.of(feed, 1)); final List<KafkaStream<Message>> streamList = streams.get(feed); if (streamList == null || streamList.size() != 1) { return null; } final KafkaStream<Message> stream = streamList.get(0); return new DefaultFirehose(connector, stream, parser); }
public AvroClicksSessionizer( String zookeeper, String groupId, String inputTopic, String outputTopic, String url, int sessionLengthMs) { this.consumer = kafka.consumer.Consumer.createJavaConsumerConnector( new ConsumerConfig(createConsumerConfig(zookeeper, groupId, url))); this.producer = getProducer(url); this.zookeeper = zookeeper; this.groupId = groupId; this.inputTopic = inputTopic; this.outputTopic = outputTopic; this.url = url; this.sessionLengthMs = sessionLengthMs; }
public KafkaConsumerConnector(String zk, String groupName) { // Get group id which should be unique for table so as to keep offsets clean for multiple // runs. String groupId = "voltdb-" + groupName; // TODO: Should get this from properties file or something as override? Properties props = new Properties(); props.put("zookeeper.connect", zk); props.put("group.id", groupId); props.put("zookeeper.session.timeout.ms", "400"); props.put("zookeeper.sync.time.ms", "200"); props.put("auto.commit.interval.ms", "1000"); props.put("auto.commit.enable", "true"); props.put("auto.offset.reset", "smallest"); props.put("rebalance.backoff.ms", "10000"); m_consumerConfig = new ConsumerConfig(props); m_consumer = kafka.consumer.Consumer.createJavaConsumerConnector(m_consumerConfig); }
public void open() { Properties props = new Properties(); try { InputStream inputStream = ConfigUtil.getConfigReader().getResourceAsStream("/kafka.properties"); props.load(inputStream); // 判断一下,那些topic是需要广播接收 String broadcast_topics = props.getProperty("broadcast.topics"); if (null != broadcast_topics && !broadcast_topics.equals("")) { if (broadcast_topics.indexOf(topic) >= 0) { String groupId = props.getProperty("group.id"); props.setProperty("group.id", groupId + "_" + Tools.getHostAddress()); } } consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(props)); FetchMessageThread thread = new FetchMessageThread(topic, consumer, handler); thread.start(); } catch (IOException e) { e.printStackTrace(); logger.error(String.format("读取kafka配置文件失败。 ErrorMsg:[%s]", e.getMessage()), e); } }
public KafkaStream<byte[], byte[]> createMessageStream(String topic) throws Exception { Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(topic, 1); final ConsumerConnector consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(propsConsumer)); Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap); Runtime.getRuntime() .addShutdownHook( new Thread() { @Override public synchronized void start() { super.start(); LOG.info("Pigeon Shutdown - Closing kafka consumer on topic [" + topic + "]"); consumer.shutdown(); } }); return consumerMap.get(topic).get(0); }
@Override public void open(Configuration parameters) throws Exception { super.open(parameters); ConsumerConnector consumer = Consumer.createJavaConsumerConnector(this.consumerConfig); // we request only one stream per consumer instance. Kafka will make sure that each consumer // group // will see each message only once. Map<String, Integer> topicCountMap = Collections.singletonMap(topicName, 1); Map<String, List<KafkaStream<byte[], byte[]>>> streams = consumer.createMessageStreams(topicCountMap); if (streams.size() != 1) { throw new RuntimeException("Expected only one message stream but got " + streams.size()); } List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName); if (kafkaStreams == null) { throw new RuntimeException( "Requested stream not available. Available streams: " + streams.toString()); } if (kafkaStreams.size() != 1) { throw new RuntimeException( "Requested 1 stream from Kafka, bot got " + kafkaStreams.size() + " streams"); } LOG.info( "Opening Consumer instance for topic '{}' on group '{}'", topicName, consumerConfig.groupId()); this.iteratorToRead = kafkaStreams.get(0).iterator(); this.consumer = consumer; zkClient = new ZkClient( consumerConfig.zkConnect(), consumerConfig.zkSessionTimeoutMs(), consumerConfig.zkConnectionTimeoutMs(), new KafkaZKStringSerializer()); // most likely the number of offsets we're going to store here will be lower than the number of // partitions. int numPartitions = getNumberOfPartitions(); LOG.debug("The topic {} has {} partitions", topicName, numPartitions); this.lastOffsets = getRuntimeContext().getOperatorState("offset", new long[numPartitions], false); this.commitedOffsets = new long[numPartitions]; // check if there are offsets to restore if (!Arrays.equals(lastOffsets.getState(), new long[numPartitions])) { if (lastOffsets.getState().length != numPartitions) { throw new IllegalStateException( "There are " + lastOffsets.getState().length + " offsets to restore for topic " + topicName + " but " + "there are only " + numPartitions + " in the topic"); } LOG.info("Setting restored offsets {} in ZooKeeper", Arrays.toString(lastOffsets.getState())); setOffsetsInZooKeeper(lastOffsets.getState()); } else { // initialize empty offsets Arrays.fill(this.lastOffsets.getState(), -1); } Arrays.fill(this.commitedOffsets, 0); // just to make it clear running = true; }
public Consumer(String topic) { consumer = kafka.consumer.Consumer.createJavaConsumerConnector(createConsumerConfig()); this.topic = topic; }