Beispiel #1
0
  public void verifySequences(long fromOffset, long toOffset) throws Exception {
    populateTopicPartitionToOffsetToFiles();
    filterOffsets(fromOffset, toOffset);

    Iterator iterator = mTopicPartitionToOffsetToFiles.entrySet().iterator();
    while (iterator.hasNext()) {
      TreeSet<Long> offsets = new TreeSet<Long>();
      Map.Entry entry = (Map.Entry) iterator.next();
      TopicPartition topicPartition = (TopicPartition) entry.getKey();
      SortedMap<Long, HashSet<LogFilePath>> offsetToFiles =
          (SortedMap<Long, HashSet<LogFilePath>>) entry.getValue();
      for (HashSet<LogFilePath> logFilePaths : offsetToFiles.values()) {
        for (LogFilePath logFilePath : logFilePaths) {
          getOffsets(logFilePath, offsets);
        }
      }
      long lastOffset = -2;
      for (Long offset : offsets) {
        if (lastOffset != -2) {
          assert lastOffset + 1 == offset
              : Long.toString(offset)
                  + " + 1 == "
                  + offset
                  + " for topic "
                  + topicPartition.getTopic()
                  + " partition "
                  + topicPartition.getPartition();
        }
        lastOffset = offset;
      }
    }
  }
Beispiel #2
0
 private long getCommittedTimestampMillis(TopicPartition topicPartition) throws Exception {
   Message message = mKafkaClient.getCommittedMessage(topicPartition);
   if (message == null) {
     LOG.error(
         "No message found for topic "
             + topicPartition.getTopic()
             + " partition "
             + topicPartition.getPartition());
     return -1;
   }
   return mMessageParser.extractTimestampMillis(message);
 }
Beispiel #3
0
 private long getLastTimestampMillis(TopicPartition topicPartition) throws Exception {
   Message message = mKafkaClient.getLastMessage(topicPartition);
   if (message == null) {
     // This will happen if no messages have been posted to the given topic partition.
     LOG.error(
         "No message found for topic "
             + topicPartition.getTopic()
             + " partition "
             + topicPartition.getPartition());
     return -1;
   }
   return mMessageParser.extractTimestampMillis(message);
 }
Beispiel #4
0
 public void verifyCounts(long fromOffset, long toOffset, int numMessages) throws Exception {
   populateTopicPartitionToOffsetToFiles();
   filterOffsets(fromOffset, toOffset);
   Iterator iterator = mTopicPartitionToOffsetToFiles.entrySet().iterator();
   int aggregateMessageCount = 0;
   while (iterator.hasNext()) {
     long previousOffset = -2L;
     long previousMessageCount = -2L;
     Map.Entry entry = (Map.Entry) iterator.next();
     SortedMap<Long, HashSet<LogFilePath>> offsetToFiles =
         (SortedMap<Long, HashSet<LogFilePath>>) entry.getValue();
     for (HashSet<LogFilePath> logFilePaths : offsetToFiles.values()) {
       int messageCount = 0;
       long offset = -2;
       for (LogFilePath logFilePath : logFilePaths) {
         assert offset == -2 || offset == logFilePath.getOffset()
             : Long.toString(offset) + " || " + offset + " == " + logFilePath.getOffset();
         messageCount += getMessageCount(logFilePath);
         offset = logFilePath.getOffset();
       }
       if (previousOffset != -2 && offset - previousOffset != previousMessageCount) {
         TopicPartition topicPartition = (TopicPartition) entry.getKey();
         throw new RuntimeException(
             "Message count of "
                 + previousMessageCount
                 + " in topic "
                 + topicPartition.getTopic()
                 + " partition "
                 + topicPartition.getPartition()
                 + " does not agree with adjacent offsets "
                 + previousOffset
                 + " and "
                 + offset);
       }
       previousOffset = offset;
       previousMessageCount = messageCount;
       aggregateMessageCount += messageCount;
     }
   }
   if (numMessages != -1 && aggregateMessageCount != numMessages) {
     throw new RuntimeException(
         "Message count "
             + aggregateMessageCount
             + " does not agree with the expected count "
             + numMessages);
   }
 }