示例#1
0
 protected void setOffset(int partition, long offset) {
   if (commitedOffsets[partition] < offset) {
     setOffset(zkClient, consumerConfig.groupId(), topicName, partition, offset);
     commitedOffsets[partition] = offset;
   } else {
     LOG.debug(
         "Ignoring offset {} for partition {} because it is already committed", offset, partition);
   }
 }
  @Test
  public void javaHandleKafkaStorage() {

    final ConsumerProperties consumerProperties =
        new PropertiesBuilder.Consumer(
                brokerList, zooKeepHost, topic, groupId, new StringDecoder(null))
            .build()
            .readFromEndOfStream()
            .consumerTimeoutMs(1234)
            .kafkaOffsetsStorage(true);

    final ConsumerConfig consumerConfig = consumerProperties.toConsumerConfig();

    assertEquals(consumerProperties.zookeeperConnect(), zooKeepHost);
    assertEquals(consumerProperties.topic(), topic);
    assertEquals(consumerProperties.groupId(), groupId);
    assertEquals(
        consumerProperties.decoder().getClass().getSimpleName(),
        StringDecoder.class.getSimpleName());
    assertEquals(consumerConfig.clientId(), groupId);
    assertEquals(consumerConfig.autoOffsetReset(), "largest");
    assertEquals(consumerConfig.offsetsStorage(), "kafka");
    assertEquals(consumerConfig.consumerTimeoutMs(), 1234);
    assertEquals(consumerConfig.dualCommitEnabled(), true);
  }
  @Test
  public void javaHandleBaseCase() {

    final PropertiesBuilder.Consumer propsBuilder =
        new PropertiesBuilder.Consumer(
            brokerList, zooKeepHost, topic, groupId, new StringDecoder(null));
    assertEquals(propsBuilder.getBrokerList(), brokerList);
    assertEquals(propsBuilder.getZooKeeperHost(), zooKeepHost);

    final ConsumerProperties consumerProperties = propsBuilder.build();

    final ConsumerConfig consumerConfig = consumerProperties.toConsumerConfig();

    assertEquals(consumerProperties.zookeeperConnect(), zooKeepHost);
    assertEquals(consumerProperties.topic(), topic);
    assertEquals(consumerProperties.groupId(), groupId);
    assertEquals(
        consumerProperties.decoder().getClass().getSimpleName(),
        StringDecoder.class.getSimpleName());
    assertEquals(consumerConfig.clientId(), groupId);
    assertEquals(consumerConfig.autoOffsetReset(), "smallest");
    assertEquals(consumerConfig.offsetsStorage(), "zookeeper");
    assertEquals(consumerConfig.consumerTimeoutMs(), 1500);
    assertEquals(consumerConfig.dualCommitEnabled(), false);
  }
示例#4
0
  /**
   * For the @param consumerConfig, specify at least the "groupid" and "zookeeper.connect" string.
   * The config will be passed into the Kafka High Level Consumer. For a full list of possible
   * values, check this out: https://kafka.apache.org/documentation.html#consumerconfigs
   */
  public PersistentKafkaSource(
      String topicName,
      DeserializationSchema<OUT> deserializationSchema,
      ConsumerConfig consumerConfig) {
    Preconditions.checkNotNull(topicName);
    Preconditions.checkNotNull(deserializationSchema);
    Preconditions.checkNotNull(consumerConfig);

    this.topicName = topicName;
    this.deserializationSchema = deserializationSchema;
    this.consumerConfig = consumerConfig;
    if (consumerConfig.autoCommitEnable()) {
      throw new IllegalArgumentException(
          "'auto.commit.enable' is set to 'true'. "
              + "This source can only be used with auto commit disabled because the "
              + "source is committing to zookeeper by itself (not using the KafkaConsumer).");
    }
    if (!consumerConfig.offsetsStorage().equals("zookeeper")) {
      // we can currently only commit to ZK.
      throw new IllegalArgumentException(
          "The 'offsets.storage' has to be set to 'zookeeper' for this Source to work reliably");
    }
  }
示例#5
0
 private void writeObject(ObjectOutputStream out) throws IOException, ClassNotFoundException {
   out.defaultWriteObject();
   out.writeObject(consumerConfig.props().props());
 }
示例#6
0
  @Override
  public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    ConsumerConnector consumer = Consumer.createJavaConsumerConnector(this.consumerConfig);
    // we request only one stream per consumer instance. Kafka will make sure that each consumer
    // group
    // will see each message only once.
    Map<String, Integer> topicCountMap = Collections.singletonMap(topicName, 1);
    Map<String, List<KafkaStream<byte[], byte[]>>> streams =
        consumer.createMessageStreams(topicCountMap);
    if (streams.size() != 1) {
      throw new RuntimeException("Expected only one message stream but got " + streams.size());
    }
    List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
    if (kafkaStreams == null) {
      throw new RuntimeException(
          "Requested stream not available. Available streams: " + streams.toString());
    }
    if (kafkaStreams.size() != 1) {
      throw new RuntimeException(
          "Requested 1 stream from Kafka, bot got " + kafkaStreams.size() + " streams");
    }
    LOG.info(
        "Opening Consumer instance for topic '{}' on group '{}'",
        topicName,
        consumerConfig.groupId());
    this.iteratorToRead = kafkaStreams.get(0).iterator();
    this.consumer = consumer;

    zkClient =
        new ZkClient(
            consumerConfig.zkConnect(),
            consumerConfig.zkSessionTimeoutMs(),
            consumerConfig.zkConnectionTimeoutMs(),
            new KafkaZKStringSerializer());

    // most likely the number of offsets we're going to store here will be lower than the number of
    // partitions.
    int numPartitions = getNumberOfPartitions();
    LOG.debug("The topic {} has {} partitions", topicName, numPartitions);
    this.lastOffsets =
        getRuntimeContext().getOperatorState("offset", new long[numPartitions], false);
    this.commitedOffsets = new long[numPartitions];
    // check if there are offsets to restore
    if (!Arrays.equals(lastOffsets.getState(), new long[numPartitions])) {
      if (lastOffsets.getState().length != numPartitions) {
        throw new IllegalStateException(
            "There are "
                + lastOffsets.getState().length
                + " offsets to restore for topic "
                + topicName
                + " but "
                + "there are only "
                + numPartitions
                + " in the topic");
      }

      LOG.info("Setting restored offsets {} in ZooKeeper", Arrays.toString(lastOffsets.getState()));
      setOffsetsInZooKeeper(lastOffsets.getState());
    } else {
      // initialize empty offsets
      Arrays.fill(this.lastOffsets.getState(), -1);
    }
    Arrays.fill(this.commitedOffsets, 0); // just to make it clear

    running = true;
  }
 public void setConsumerProperty(String key, Object value) {
   consumerConfig.props().props().setProperty(checkNotNull(key), checkNotNull(value).toString());
 }