Exemplo n.º 1
0
  /**
   * The main constructor for creating a FlinkKafkaProducer.
   *
   * @param defaultTopicId The default topic to write data to
   * @param serializationSchema A serializable serialization schema for turning user objects into a
   *     kafka-consumable byte[] supporting key/value messages
   * @param producerConfig Configuration properties for the KafkaProducer. 'bootstrap.servers.' is
   *     the only required argument.
   * @param customPartitioner A serializable partitioner for assigning messages to Kafka partitions.
   *     Passing null will use Kafka's partitioner
   */
  public FlinkKafkaProducerBase(
      String defaultTopicId,
      KeyedSerializationSchema<IN> serializationSchema,
      Properties producerConfig,
      KafkaPartitioner<IN> customPartitioner) {
    requireNonNull(defaultTopicId, "TopicID not set");
    requireNonNull(serializationSchema, "serializationSchema not set");
    requireNonNull(producerConfig, "producerConfig not set");
    ClosureCleaner.ensureSerializable(customPartitioner);
    ClosureCleaner.ensureSerializable(serializationSchema);

    this.defaultTopicId = defaultTopicId;
    this.schema = serializationSchema;
    this.producerConfig = producerConfig;

    // set the producer configuration properties.

    if (!producerConfig.contains(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)) {
      this.producerConfig.put(
          ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getCanonicalName());
    } else {
      LOG.warn(
          "Overwriting the '{}' is not recommended", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
    }

    if (!producerConfig.contains(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)) {
      this.producerConfig.put(
          ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
          ByteArraySerializer.class.getCanonicalName());
    } else {
      LOG.warn(
          "Overwriting the '{}' is not recommended", ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
    }

    // create a local KafkaProducer to get the list of partitions.
    // this will also ensure locally that all required ProducerConfig values are set.
    try (Producer<Void, IN> getPartitionsProd = getKafkaProducer(this.producerConfig)) {
      List<PartitionInfo> partitionsList = getPartitionsProd.partitionsFor(defaultTopicId);

      this.partitions = new int[partitionsList.size()];
      for (int i = 0; i < partitions.length; i++) {
        partitions[i] = partitionsList.get(i).partition();
      }
      getPartitionsProd.close();
    }

    this.partitioner = customPartitioner;
    this.producerId = UUID.randomUUID().toString();
  }
Exemplo n.º 2
0
 public void publish(byte[] event, String Id) throws ExecutionException, InterruptedException {
   logger.debug("Send message");
   RecordMetadata m =
       kafkaProducer.send(new ProducerRecord<String, byte[]>("events", Id, event)).get();
   System.out.println("Message produced, offset: " + m.offset());
   System.out.println("Message produced, partition : " + m.partition());
   System.out.println("Message produced, topic: " + m.topic());
 }
Exemplo n.º 3
0
 private static void test(Producer<String, String> producer) {
   for (int i = testNumber; i > 0; i--) {
     // partion 随机
     //            ProducerRecord record = new ProducerRecord<String, String>(TopicOne.TOPIC, i%2,
     // i+"", i+"");
     ProducerRecord record = new ProducerRecord<String, String>(TopicOne.TOPIC, 0, i + "", i + "");
     producer.send(record);
   }
 }
 public void onStatus(Status status) {
   Tweet tweet = TwitterUtils.parseTweet(status);
   ProducerRecord<String, Tweet> data = new ProducerRecord<String, Tweet>(topic, tweet);
   producer.send(data);
 }