@Test
  public void javaHandleKafkaStorage() {

    final ConsumerProperties consumerProperties =
        new PropertiesBuilder.Consumer(
                brokerList, zooKeepHost, topic, groupId, new StringDecoder(null))
            .build()
            .readFromEndOfStream()
            .consumerTimeoutMs(1234)
            .kafkaOffsetsStorage(true);

    final ConsumerConfig consumerConfig = consumerProperties.toConsumerConfig();

    assertEquals(consumerProperties.zookeeperConnect(), zooKeepHost);
    assertEquals(consumerProperties.topic(), topic);
    assertEquals(consumerProperties.groupId(), groupId);
    assertEquals(
        consumerProperties.decoder().getClass().getSimpleName(),
        StringDecoder.class.getSimpleName());
    assertEquals(consumerConfig.clientId(), groupId);
    assertEquals(consumerConfig.autoOffsetReset(), "largest");
    assertEquals(consumerConfig.offsetsStorage(), "kafka");
    assertEquals(consumerConfig.consumerTimeoutMs(), 1234);
    assertEquals(consumerConfig.dualCommitEnabled(), true);
  }
  @Test
  public void javaHandleBaseCase() {

    final PropertiesBuilder.Consumer propsBuilder =
        new PropertiesBuilder.Consumer(
            brokerList, zooKeepHost, topic, groupId, new StringDecoder(null));
    assertEquals(propsBuilder.getBrokerList(), brokerList);
    assertEquals(propsBuilder.getZooKeeperHost(), zooKeepHost);

    final ConsumerProperties consumerProperties = propsBuilder.build();

    final ConsumerConfig consumerConfig = consumerProperties.toConsumerConfig();

    assertEquals(consumerProperties.zookeeperConnect(), zooKeepHost);
    assertEquals(consumerProperties.topic(), topic);
    assertEquals(consumerProperties.groupId(), groupId);
    assertEquals(
        consumerProperties.decoder().getClass().getSimpleName(),
        StringDecoder.class.getSimpleName());
    assertEquals(consumerConfig.clientId(), groupId);
    assertEquals(consumerConfig.autoOffsetReset(), "smallest");
    assertEquals(consumerConfig.offsetsStorage(), "zookeeper");
    assertEquals(consumerConfig.consumerTimeoutMs(), 1500);
    assertEquals(consumerConfig.dualCommitEnabled(), false);
  }
Example #3
0
  /**
   * For the @param consumerConfig, specify at least the "groupid" and "zookeeper.connect" string.
   * The config will be passed into the Kafka High Level Consumer. For a full list of possible
   * values, check this out: https://kafka.apache.org/documentation.html#consumerconfigs
   */
  public PersistentKafkaSource(
      String topicName,
      DeserializationSchema<OUT> deserializationSchema,
      ConsumerConfig consumerConfig) {
    Preconditions.checkNotNull(topicName);
    Preconditions.checkNotNull(deserializationSchema);
    Preconditions.checkNotNull(consumerConfig);

    this.topicName = topicName;
    this.deserializationSchema = deserializationSchema;
    this.consumerConfig = consumerConfig;
    if (consumerConfig.autoCommitEnable()) {
      throw new IllegalArgumentException(
          "'auto.commit.enable' is set to 'true'. "
              + "This source can only be used with auto commit disabled because the "
              + "source is committing to zookeeper by itself (not using the KafkaConsumer).");
    }
    if (!consumerConfig.offsetsStorage().equals("zookeeper")) {
      // we can currently only commit to ZK.
      throw new IllegalArgumentException(
          "The 'offsets.storage' has to be set to 'zookeeper' for this Source to work reliably");
    }
  }