private static void execute() throws InterruptedException {

    KafkaConsumer<String, String> consumer = createConsumer();

    // Subscribe to all partition in that topic. 'assign' could be used here
    // instead of 'subscribe' to subscribe to specific partition.
    consumer.subscribe(Arrays.asList("normal-topic"));

    processRecords(consumer);
  }
Beispiel #2
0
  public static void main(String[] args) throws UnknownHostException {
    Properties props = new Properties();
    props.put("bootstrap.servers", "kafka01:9092,kafka02:9092,kafka03:9092");
    props.put("group.id", "test");
    props.put("enable.auto.commit", "true");
    props.put("auto.commit.interval.ms", "1000");
    props.put("session.timeout.ms", "30000");
    props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList("apt-receive1"));
    List<TopicPartition> partitions = new ArrayList<>();
    //		partitions.add(new TopicPartition("apt-receive1", 2));
    //		partitions.add(new TopicPartition("apt-receive1", 13));
    //		consumer.assign(partitions);
    for (int i = 0; i < 10000; i++) {
      ConsumerRecords<String, byte[]> records = consumer.poll(100);
      System.out.println(i + ": " + records.count());
      for (ConsumerRecord<String, byte[]> record : records) {
        //				System.out.println(record.key());
        bloom[Integer.parseInt(record.key())] = 1;
      }
      //			if (sum == 10000) {
      //				System.out.println("sum=" + sum);
      //				break;
      //			}

    }
    for (int j = 0; j < 10_000_000; j++) {
      if (bloom[j] == 0) {
        System.err.println("" + j);
      }
    }
    consumer.close();
    System.err.println("Finish!");
  }
  /** @see stream.io.AbstractStream#init() */
  @Override
  public void init() throws Exception {
    super.init();

    if (group == null) {
      group = UUID.randomUUID().toString();
    }

    Properties props = new Properties();
    props.put("zookeeper.connect", zookeeper);
    props.put("metadata.broker.list", broker);
    props.put("bootstrap.servers", broker);
    props.put("group.id", group);
    props.put("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
    props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    props.put("enable.auto.commit", "true");
    props.put("auto.commit.interval.ms", "10000");

    log.debug("Creating kafka consumer...");
    final KafkaConsumer<byte[], byte[]> kc = new KafkaConsumer<byte[], byte[]>(props);
    consumer = kc;

    log.info("Subscribing to topic '{}'", topic);
    consumer.subscribe(Arrays.asList(topic));
    log.info("Using codec {}", valueCodec);

    List<PartitionInfo> partitionInfo = consumer.partitionsFor(topic);
    log.debug("topic '{}' has {} partitions", topic, partitionInfo.size());

    pollThread =
        new Thread() {
          public void run() {
            try {
              running.set(true);
              while (running.get()) {
                ConsumerRecords<byte[], byte[]> messages = consumer.poll(100L);
                log.debug("Polled new messages: {}", messages);
                while (messages == null || messages.count() < 1) {
                  messages = consumer.poll(1000L);
                  log.debug("Polled new messages: {}", messages);
                }

                if (messages != null) {
                  Iterator<ConsumerRecord<byte[], byte[]>> it = messages.iterator();
                  while (it.hasNext()) {
                    ConsumerRecord<byte[], byte[]> record = it.next();

                    Object key = record.key();
                    byte[] data = record.value();

                    Data item = null;
                    if (valueCodec != null) {
                      log.debug("de-serializing item using {}", valueCodec);
                      item = valueCodec.decode(data);
                    } else {
                      item = DataFactory.create();
                      if (key != null) {
                        item.put("@kafka:key", (Serializable) key);
                      }
                      item.put("@kafka:value", data);
                    }
                    item.put("@kafka:topic", record.topic());
                    item.put("@kafka:partition", record.partition());
                    queue.add(item);
                  }
                }
              }
            } catch (Exception e) {
              e.printStackTrace();
            } finally {
              log.info("Closing topic-stream reader...");
              queue.add(endOfStream);
            }
          }
        };

    pollThread.setDaemon(true);
    pollThread.start();

    Signals.register(
        new Hook() {
          @Override
          public void signal(int flags) {
            log.info("Received shutdown-signal!");
            running.set(false);
            pollThread.interrupt();
            log.info("Adding EOF to queue...");
            queue.add(endOfStream);
          }
        });
  }
  @Override
  public void drive() {
    // A Consumer is not thread-safe
    // {@see
    // http://kafka.apache.org/090/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html}
    // {@see
    // http://kafka.apache.org/090/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html#multithreaded}
    try (KafkaConsumer<byte[], byte[]> consumer =
        new KafkaConsumer<>(consumerDefinition.getKafkaConfig())) {

      String topic = consumerDefinition.getTopic();
      log.info("Subscribing to {}", topic);
      if (consumerRebalanceListener == null) {
        consumer.subscribe(Collections.singletonList(topic));
      } else {
        consumer.subscribe(Collections.singletonList(topic), consumerRebalanceListener);
      }

      long messagesToReceive = consumerDefinition.getMessagesToReceive();
      log.info("Expecting {} messages", messagesToReceive);

      StopWatch stopWatch = new StopWatch();
      stopWatch.start();

      do {
        ConsumerRecords<byte[], byte[]> records =
            consumer.poll(consumerDefinition.getPollTimeout());
        if (records == null) {
          throw new IllegalStateException("null ConsumerRecords polled");
        } else {
          if (records.count() == 0) {
            try {
              log.info("No records fetched, pausing");
              Thread.sleep(1000);
            } catch (InterruptedException e) {
              throw new RuntimeException(e);
            }
          } else {
            if (log.isTraceEnabled()) {
              log.trace("Fetched {} records", records.count());
            }
            for (ConsumerRecord<byte[], byte[]> record : records) {
              recordsFetched += 1;
              applyReceiveDelay();
              if (recordsFetched % consumerDefinition.getReportReceivedEvery() == 0) {
                log.info("Received {} messages", recordsFetched);
              }
            }
          }
        }

        if (isShutdownRequested()) {
          break;
        }
        stopWatch.split();
      } while ((recordsFetched < messagesToReceive)
          && (stopWatch.getSplitTime() < consumerDefinition.getTestRunTimeout()));

      stopWatch.stop();
      if (isShutdownRequested()) {
        log.info("Shutting down");
      } else {
        long runTime = stopWatch.getTime();
        log.info("Done. Consumer received {} msgs in {} ms", messagesToReceive, runTime);

        double averageThroughput = (1000d / runTime) * messagesToReceive;
        log.info("Average throughput: {} msg/s", averageThroughput);
      }

    } finally {
      log.debug("Consumer closed");
      if (completionLatch != null) {
        completionLatch.countDown();
      }
    }
  }