コード例 #1
0
  @Override
  public void run() {
    try {
      for (Entry<Pair<Topic, ConsumerGroup>, Long> entry : m_limits.entrySet()) {
        Topic topic = entry.getKey().getKey();
        ConsumerGroup group = entry.getKey().getValue();
        long limit = entry.getValue();

        Map<Integer, Long> backlogs = calculateBacklog(topic, group);

        long totalBacklog = 0;
        for (Long backlog : backlogs.values()) {
          totalBacklog += backlog;
        }

        if (totalBacklog >= limit) {
          m_result.addMonitorEvent(
              new ConsumeLargeBacklogEvent(topic.getName(), group.getName(), backlogs));
        }
      }
    } catch (Exception e) {
      m_exceptions.add(e);
    } finally {
      m_latch.countDown();
    }
  }
コード例 #2
0
 private Partition findPartition(String topicName, int partitionId) {
   Topic topic = findTopic(topicName);
   if (topic != null) {
     return topic.findPartition(partitionId);
   }
   throw new RuntimeException("Can not find topic: " + topicName);
 }
コード例 #3
0
  @Test
  public void testSendWithNoEndpoint() throws Exception {
    Meta meta = m_metaHolder.getMeta();
    reset(m_metaHolder);
    Topic topic = meta.findTopic(TEST_TOPIC);
    for (Partition p : topic.getPartitions()) {
      p.setEndpoint(null);
    }
    when(m_metaHolder.getMeta()).thenReturn(meta);

    brokerActionsWhenReceivedSendMessageCmd( //
        MessageSendAnswer.NoOp);

    List<Pair<String, String>> appProperties = Arrays.asList(new Pair<String, String>("a", "A"));
    Future<SendResult> future =
        sendAsync(TEST_TOPIC, "pKey", "body", "rKey", appProperties, false, null);

    try {
      future.get(
          lookup(ProducerConfig.class).getBrokerSenderSendTimeoutMillis() + 200L,
          TimeUnit.MILLISECONDS);
      fail();
    } catch (TimeoutException e) {
      // do nothing
    } catch (Exception e) {
      fail();
    }
    assertFalse(future.isDone());

    List<Command> brokerReceivedCmds = getBrokerReceivedCmds();
    assertEquals(0, brokerReceivedCmds.size());
  }
コード例 #4
0
  private Map<Integer, Long> calculateBacklog(Topic topic, ConsumerGroup group) {
    Map<Integer, Long> backlogs = new HashMap<Integer, Long>();

    for (Partition partition : topic.getPartitions()) {
      long pBacklog = doCalculateBacklog(topic.getName(), partition.getId(), 0, group.getId());
      long npBacklog = doCalculateBacklog(topic.getName(), partition.getId(), 1, group.getId());
      backlogs.put(partition.getId(), pBacklog + npBacklog);
    }

    return backlogs;
  }
コード例 #5
0
  @Override
  public SubscribeHandle start(List<Subscriber> subscribers) {
    CompositeSubscribeHandle handle = new CompositeSubscribeHandle();

    for (Subscriber s : subscribers) {
      List<Topic> topics = m_metaService.listTopicsByPattern(s.getTopicPattern());

      if (topics == null || topics.isEmpty()) {
        throw new IllegalArgumentException(
            String.format("Can not find any topics matching pattern %s", s.getTopicPattern()));
      }

      log.info(
          "Found topics({}) matching pattern({}), groupId={}.",
          CollectionUtil.collect(
              topics,
              new Transformer() {

                @Override
                public Object transform(Object topic) {
                  return ((Topic) topic).getName();
                }
              }),
          s.getTopicPattern(),
          s.getGroupId());

      for (Topic topic : topics) {
        ConsumerContext context =
            new ConsumerContext(
                topic, s.getGroupId(), s.getConsumer(), s.getMessageClass(), s.getConsumerType());

        // FIXME validate all, if fail in any validator, exit
        if (validate(topic, context)) {
          try {
            String endpointType = m_metaService.findEndpointTypeByTopic(topic.getName());
            ConsumerBootstrap consumerBootstrap =
                m_consumerManager.findConsumerBootStrap(endpointType);
            handle.addSubscribeHandle(consumerBootstrap.start(context));

          } catch (RuntimeException e) {
            log.error(
                "Failed to start consumer for topic {}(consumer: groupId={}, sessionId={})",
                topic.getName(),
                context.getGroupId(),
                context.getSessionId(),
                e);
            throw e;
          }
        }
      }
    }

    return handle;
  }
コード例 #6
0
  private boolean validate(Topic topic, ConsumerContext context) {
    if (Endpoint.BROKER.equals(topic.getEndpointType())) {
      if (!m_metaService.containsConsumerGroup(topic.getName(), context.getGroupId())) {
        log.error(
            "Consumer group {} not found for topic {}, please add consumer group in Hermes-Portal first.",
            context.getGroupId(),
            topic.getName());
        return false;
      }
    }

    return true;
  }