@Override
public Binding<MessageChannel> bindProducer(
final String name, MessageChannel moduleOutputChannel, Properties properties) {
Assert.isInstanceOf(SubscribableChannel.class, moduleOutputChannel);
KafkaPropertiesAccessor producerPropertiesAccessor = new KafkaPropertiesAccessor(properties);
validateProducerProperties(name, properties, SUPPORTED_PRODUCER_PROPERTIES);
if (logger.isInfoEnabled()) {
logger.info("Using kafka topic for outbound: " + name);
}
final String topicName = escapeTopicName(name);
int numPartitions = producerPropertiesAccessor.getNumberOfKafkaPartitionsForProducer();
Collection<Partition> partitions =
ensureTopicCreated(topicName, numPartitions, defaultReplicationFactor);
ProducerMetadata<byte[], byte[]> producerMetadata =
new ProducerMetadata<>(
topicName, byte[].class, byte[].class, BYTE_ARRAY_SERIALIZER, BYTE_ARRAY_SERIALIZER);
producerMetadata.setCompressionType(
ProducerMetadata.CompressionType.valueOf(
producerPropertiesAccessor.getCompressionCodec(this.defaultCompressionCodec)));
producerMetadata.setBatchBytes(producerPropertiesAccessor.getBatchSize(this.defaultBatchSize));
Properties additionalProps = new Properties();
additionalProps.put(
ProducerConfig.ACKS_CONFIG,
String.valueOf(producerPropertiesAccessor.getRequiredAcks(this.defaultRequiredAcks)));
additionalProps.put(
ProducerConfig.LINGER_MS_CONFIG,
String.valueOf(producerPropertiesAccessor.getBatchTimeout(this.defaultBatchTimeout)));
ProducerFactoryBean<byte[], byte[]> producerFB =
new ProducerFactoryBean<>(producerMetadata, brokers, additionalProps);
try {
final ProducerConfiguration<byte[], byte[]> producerConfiguration =
new ProducerConfiguration<>(producerMetadata, producerFB.getObject());
MessageHandler handler =
new SendingHandler(
topicName, producerPropertiesAccessor, partitions.size(), producerConfiguration);
EventDrivenConsumer consumer =
new EventDrivenConsumer((SubscribableChannel) moduleOutputChannel, handler);
consumer.setBeanFactory(this.getBeanFactory());
consumer.setBeanName("outbound." + name);
consumer.afterPropertiesSet();
Binding<MessageChannel> producerBinding =
Binding.forProducer(name, moduleOutputChannel, consumer, producerPropertiesAccessor);
addBinding(producerBinding);
producerBinding.start();
return producerBinding;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private void doRegisterProducer(
final String name,
MessageChannel moduleOutputChannel,
AmqpOutboundEndpoint delegate,
String replyTo,
RabbitPropertiesAccessor properties) {
Assert.isInstanceOf(SubscribableChannel.class, moduleOutputChannel);
MessageHandler handler = new SendingHandler(delegate, replyTo, properties);
EventDrivenConsumer consumer =
new EventDrivenConsumer((SubscribableChannel) moduleOutputChannel, handler);
consumer.setBeanFactory(getBeanFactory());
consumer.setBeanName("outbound." + name);
consumer.afterPropertiesSet();
Binding producerBinding = Binding.forProducer(name, moduleOutputChannel, consumer, properties);
addBinding(producerBinding);
producerBinding.start();
}
private Binding<MessageChannel> createKafkaConsumer(
String name,
final MessageChannel moduleInputChannel,
Properties properties,
String group,
long referencePoint) {
validateConsumerProperties(groupedName(name, group), properties, SUPPORTED_CONSUMER_PROPERTIES);
KafkaPropertiesAccessor accessor = new KafkaPropertiesAccessor(properties);
int maxConcurrency = accessor.getConcurrency(defaultConcurrency);
String topic = escapeTopicName(name);
int numPartitions = accessor.getNumberOfKafkaPartitionsForConsumer();
Collection<Partition> allPartitions =
ensureTopicCreated(topic, numPartitions, defaultReplicationFactor);
Decoder<byte[]> valueDecoder = new DefaultDecoder(null);
Decoder<byte[]> keyDecoder = new DefaultDecoder(null);
Collection<Partition> listenedPartitions;
int moduleCount = accessor.getCount();
if (moduleCount == 1) {
listenedPartitions = allPartitions;
} else {
listenedPartitions = new ArrayList<Partition>();
for (Partition partition : allPartitions) {
// divide partitions across modules
if (accessor.getPartitionIndex() != -1) {
if ((partition.getId() % moduleCount) == accessor.getPartitionIndex()) {
listenedPartitions.add(partition);
}
} else {
int moduleSequence = accessor.getSequence();
if (moduleCount == 0) {
throw new IllegalArgumentException(
"The Kafka transport does not support 0-count modules");
} else {
// sequence numbers are zero-based
if ((partition.getId() % moduleCount) == (moduleSequence - 1)) {
listenedPartitions.add(partition);
}
}
}
}
}
ReceivingHandler rh = new ReceivingHandler();
rh.setOutputChannel(moduleInputChannel);
final FixedSubscriberChannel bridge = new FixedSubscriberChannel(rh);
bridge.setBeanName("bridge." + name);
final KafkaMessageListenerContainer messageListenerContainer =
createMessageListenerContainer(
accessor, group, maxConcurrency, listenedPartitions, referencePoint);
final KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter =
new KafkaMessageDrivenChannelAdapter(messageListenerContainer);
kafkaMessageDrivenChannelAdapter.setBeanFactory(this.getBeanFactory());
kafkaMessageDrivenChannelAdapter.setKeyDecoder(keyDecoder);
kafkaMessageDrivenChannelAdapter.setPayloadDecoder(valueDecoder);
kafkaMessageDrivenChannelAdapter.setOutputChannel(bridge);
kafkaMessageDrivenChannelAdapter.setAutoCommitOffset(
accessor.getDefaultAutoCommitEnabled(this.defaultAutoCommitEnabled));
kafkaMessageDrivenChannelAdapter.afterPropertiesSet();
kafkaMessageDrivenChannelAdapter.start();
EventDrivenConsumer edc =
new EventDrivenConsumer(bridge, rh) {
@Override
protected void doStop() {
// stop the offset manager and the channel adapter before unbinding
// this means that the upstream channel adapter has a chance to stop
kafkaMessageDrivenChannelAdapter.stop();
if (messageListenerContainer.getOffsetManager() instanceof DisposableBean) {
try {
((DisposableBean) messageListenerContainer.getOffsetManager()).destroy();
} catch (Exception e) {
logger.error("Error while closing the offset manager", e);
}
}
super.doStop();
}
};
String groupedName = groupedName(name, group);
edc.setBeanName("inbound." + groupedName);
Binding<MessageChannel> consumerBinding =
Binding.forConsumer(name, group, edc, moduleInputChannel, accessor);
addBinding(consumerBinding);
consumerBinding.start();
return consumerBinding;
}