Exemplo n.º 1
0
  @Override
  public KafkaTridentSpoutBatchMetadata<K, V> emitPartitionBatch(
      TransactionAttempt tx,
      TridentCollector collector,
      KafkaTridentSpoutTopicPartition partitionTs,
      KafkaTridentSpoutBatchMetadata<K, V> lastBatch) {
    LOG.debug(
        "Emitting batch: [transaction = {}], [partition = {}], [collector = {}], [lastBatchMetadata = {}]",
        tx,
        partitionTs,
        collector,
        lastBatch);

    final TopicPartition topicPartition = partitionTs.getTopicPartition();
    KafkaTridentSpoutBatchMetadata<K, V> currentBatch = lastBatch;
    Collection<TopicPartition> pausedTopicPartitions = Collections.emptySet();

    try {
      // pause other topic partitions to only poll from current topic partition
      pausedTopicPartitions = pauseTopicPartitions(topicPartition);

      seek(topicPartition, lastBatch);

      // poll
      final ConsumerRecords<K, V> records = kafkaConsumer.poll(pollTimeoutMs);
      LOG.debug("Polled [{}] records from Kafka.", records.count());

      if (!records.isEmpty()) {
        emitTuples(collector, records);
        // build new metadata
        currentBatch = new KafkaTridentSpoutBatchMetadata<>(topicPartition, records, lastBatch);
      }
    } finally {
      kafkaConsumer.resume(pausedTopicPartitions);
      LOG.trace("Resumed topic partitions [{}]", pausedTopicPartitions);
    }
    LOG.debug("Current batch metadata {}", currentBatch);
    return currentBatch;
  }
  private static void processRecords(KafkaConsumer<String, String> consumer)
      throws InterruptedException {

    while (true) {

      ConsumerRecords<String, String> records = consumer.poll(100);
      long lastOffset = 0;

      for (ConsumerRecord<String, String> record : records) {
        System.out.printf(
            "\n\roffset = %d, key = %s, value = %s", record.offset(), record.key(), record.value());
        lastOffset = record.offset();
      }

      System.out.println("lastOffset read: " + lastOffset);

      process();

      // Below call is important to control the offset commit. Do this call after you
      // finish processing the business process to get the at least once guarantee.

      consumer.commitSync();
    }
  }
Exemplo n.º 3
0
  public static void main(String[] args) throws UnknownHostException {
    Properties props = new Properties();
    props.put("bootstrap.servers", "kafka01:9092,kafka02:9092,kafka03:9092");
    props.put("group.id", "test");
    props.put("enable.auto.commit", "true");
    props.put("auto.commit.interval.ms", "1000");
    props.put("session.timeout.ms", "30000");
    props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList("apt-receive1"));
    List<TopicPartition> partitions = new ArrayList<>();
    //		partitions.add(new TopicPartition("apt-receive1", 2));
    //		partitions.add(new TopicPartition("apt-receive1", 13));
    //		consumer.assign(partitions);
    for (int i = 0; i < 10000; i++) {
      ConsumerRecords<String, byte[]> records = consumer.poll(100);
      System.out.println(i + ": " + records.count());
      for (ConsumerRecord<String, byte[]> record : records) {
        //				System.out.println(record.key());
        bloom[Integer.parseInt(record.key())] = 1;
      }
      //			if (sum == 10000) {
      //				System.out.println("sum=" + sum);
      //				break;
      //			}

    }
    for (int j = 0; j < 10_000_000; j++) {
      if (bloom[j] == 0) {
        System.err.println("" + j);
      }
    }
    consumer.close();
    System.err.println("Finish!");
  }
Exemplo n.º 4
0
  @Override
  public void drive() {
    // A Consumer is not thread-safe
    // {@see
    // http://kafka.apache.org/090/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html}
    // {@see
    // http://kafka.apache.org/090/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html#multithreaded}
    try (KafkaConsumer<byte[], byte[]> consumer =
        new KafkaConsumer<>(consumerDefinition.getKafkaConfig())) {

      String topic = consumerDefinition.getTopic();
      log.info("Subscribing to {}", topic);
      if (consumerRebalanceListener == null) {
        consumer.subscribe(Collections.singletonList(topic));
      } else {
        consumer.subscribe(Collections.singletonList(topic), consumerRebalanceListener);
      }

      long messagesToReceive = consumerDefinition.getMessagesToReceive();
      log.info("Expecting {} messages", messagesToReceive);

      StopWatch stopWatch = new StopWatch();
      stopWatch.start();

      do {
        ConsumerRecords<byte[], byte[]> records =
            consumer.poll(consumerDefinition.getPollTimeout());
        if (records == null) {
          throw new IllegalStateException("null ConsumerRecords polled");
        } else {
          if (records.count() == 0) {
            try {
              log.info("No records fetched, pausing");
              Thread.sleep(1000);
            } catch (InterruptedException e) {
              throw new RuntimeException(e);
            }
          } else {
            if (log.isTraceEnabled()) {
              log.trace("Fetched {} records", records.count());
            }
            for (ConsumerRecord<byte[], byte[]> record : records) {
              recordsFetched += 1;
              applyReceiveDelay();
              if (recordsFetched % consumerDefinition.getReportReceivedEvery() == 0) {
                log.info("Received {} messages", recordsFetched);
              }
            }
          }
        }

        if (isShutdownRequested()) {
          break;
        }
        stopWatch.split();
      } while ((recordsFetched < messagesToReceive)
          && (stopWatch.getSplitTime() < consumerDefinition.getTestRunTimeout()));

      stopWatch.stop();
      if (isShutdownRequested()) {
        log.info("Shutting down");
      } else {
        long runTime = stopWatch.getTime();
        log.info("Done. Consumer received {} msgs in {} ms", messagesToReceive, runTime);

        double averageThroughput = (1000d / runTime) * messagesToReceive;
        log.info("Average throughput: {} msg/s", averageThroughput);
      }

    } finally {
      log.debug("Consumer closed");
      if (completionLatch != null) {
        completionLatch.countDown();
      }
    }
  }