@Override
public void run() {
try {
for (Entry<Pair<Topic, ConsumerGroup>, Long> entry : m_limits.entrySet()) {
Topic topic = entry.getKey().getKey();
ConsumerGroup group = entry.getKey().getValue();
long limit = entry.getValue();
Map<Integer, Long> backlogs = calculateBacklog(topic, group);
long totalBacklog = 0;
for (Long backlog : backlogs.values()) {
totalBacklog += backlog;
}
if (totalBacklog >= limit) {
m_result.addMonitorEvent(
new ConsumeLargeBacklogEvent(topic.getName(), group.getName(), backlogs));
}
}
} catch (Exception e) {
m_exceptions.add(e);
} finally {
m_latch.countDown();
}
}
private Partition findPartition(String topicName, int partitionId) {
Topic topic = findTopic(topicName);
if (topic != null) {
return topic.findPartition(partitionId);
}
throw new RuntimeException("Can not find topic: " + topicName);
}
@Test
public void testSendWithNoEndpoint() throws Exception {
Meta meta = m_metaHolder.getMeta();
reset(m_metaHolder);
Topic topic = meta.findTopic(TEST_TOPIC);
for (Partition p : topic.getPartitions()) {
p.setEndpoint(null);
}
when(m_metaHolder.getMeta()).thenReturn(meta);
brokerActionsWhenReceivedSendMessageCmd( //
MessageSendAnswer.NoOp);
List<Pair<String, String>> appProperties = Arrays.asList(new Pair<String, String>("a", "A"));
Future<SendResult> future =
sendAsync(TEST_TOPIC, "pKey", "body", "rKey", appProperties, false, null);
try {
future.get(
lookup(ProducerConfig.class).getBrokerSenderSendTimeoutMillis() + 200L,
TimeUnit.MILLISECONDS);
fail();
} catch (TimeoutException e) {
// do nothing
} catch (Exception e) {
fail();
}
assertFalse(future.isDone());
List<Command> brokerReceivedCmds = getBrokerReceivedCmds();
assertEquals(0, brokerReceivedCmds.size());
}
private Map<Integer, Long> calculateBacklog(Topic topic, ConsumerGroup group) {
Map<Integer, Long> backlogs = new HashMap<Integer, Long>();
for (Partition partition : topic.getPartitions()) {
long pBacklog = doCalculateBacklog(topic.getName(), partition.getId(), 0, group.getId());
long npBacklog = doCalculateBacklog(topic.getName(), partition.getId(), 1, group.getId());
backlogs.put(partition.getId(), pBacklog + npBacklog);
}
return backlogs;
}
@Override
public SubscribeHandle start(List<Subscriber> subscribers) {
CompositeSubscribeHandle handle = new CompositeSubscribeHandle();
for (Subscriber s : subscribers) {
List<Topic> topics = m_metaService.listTopicsByPattern(s.getTopicPattern());
if (topics == null || topics.isEmpty()) {
throw new IllegalArgumentException(
String.format("Can not find any topics matching pattern %s", s.getTopicPattern()));
}
log.info(
"Found topics({}) matching pattern({}), groupId={}.",
CollectionUtil.collect(
topics,
new Transformer() {
@Override
public Object transform(Object topic) {
return ((Topic) topic).getName();
}
}),
s.getTopicPattern(),
s.getGroupId());
for (Topic topic : topics) {
ConsumerContext context =
new ConsumerContext(
topic, s.getGroupId(), s.getConsumer(), s.getMessageClass(), s.getConsumerType());
// FIXME validate all, if fail in any validator, exit
if (validate(topic, context)) {
try {
String endpointType = m_metaService.findEndpointTypeByTopic(topic.getName());
ConsumerBootstrap consumerBootstrap =
m_consumerManager.findConsumerBootStrap(endpointType);
handle.addSubscribeHandle(consumerBootstrap.start(context));
} catch (RuntimeException e) {
log.error(
"Failed to start consumer for topic {}(consumer: groupId={}, sessionId={})",
topic.getName(),
context.getGroupId(),
context.getSessionId(),
e);
throw e;
}
}
}
}
return handle;
}
private boolean validate(Topic topic, ConsumerContext context) {
if (Endpoint.BROKER.equals(topic.getEndpointType())) {
if (!m_metaService.containsConsumerGroup(topic.getName(), context.getGroupId())) {
log.error(
"Consumer group {} not found for topic {}, please add consumer group in Hermes-Portal first.",
context.getGroupId(),
topic.getName());
return false;
}
}
return true;
}