Esempio n. 1
0
  @Override
  public void run(SourceContext<Long> ctx) throws Exception {
    final Object checkpointLock = ctx.getCheckpointLock();

    RuntimeContext context = getRuntimeContext();

    final long stepSize = context.getNumberOfParallelSubtasks();
    final long congruence = start + context.getIndexOfThisSubtask();

    final long toCollect =
        ((end - start + 1) % stepSize > (congruence - start))
            ? ((end - start + 1) / stepSize + 1)
            : ((end - start + 1) / stepSize);

    while (isRunning && collected < toCollect) {
      synchronized (checkpointLock) {
        ctx.collect(collected * stepSize + congruence);
        collected++;
      }
    }
  }
Esempio n. 2
0
  /** Initializes the connection to Kafka. */
  @Override
  public void open(Configuration configuration) {
    producer = getKafkaProducer(this.producerConfig);

    RuntimeContext ctx = getRuntimeContext();
    if (partitioner != null) {
      partitioner.open(ctx.getIndexOfThisSubtask(), ctx.getNumberOfParallelSubtasks(), partitions);
    }

    LOG.info(
        "Starting FlinkKafkaProducer ({}/{}) to produce into topic {}",
        ctx.getIndexOfThisSubtask(),
        ctx.getNumberOfParallelSubtasks(),
        defaultTopicId);

    // register Kafka metrics to Flink accumulators
    if (!Boolean.valueOf(producerConfig.getProperty(KEY_DISABLE_METRICS, "false"))) {
      Map<MetricName, ? extends Metric> metrics = this.producer.metrics();

      if (metrics == null) {
        // MapR's Kafka implementation returns null here.
        LOG.info("Producer implementation does not support metrics");
      } else {
        for (Map.Entry<MetricName, ? extends Metric> metric : metrics.entrySet()) {
          String name = producerId + "-producer-" + metric.getKey().name();
          DefaultKafkaMetricAccumulator kafkaAccumulator =
              DefaultKafkaMetricAccumulator.createFor(metric.getValue());
          // best effort: we only add the accumulator if available.
          if (kafkaAccumulator != null) {
            getRuntimeContext().addAccumulator(name, kafkaAccumulator);
          }
        }
      }
    }

    if (flushOnCheckpoint
        && !((StreamingRuntimeContext) this.getRuntimeContext()).isCheckpointingEnabled()) {
      LOG.warn(
          "Flushing on checkpoint is enabled, but checkpointing is not enabled. Disabling flushing.");
      flushOnCheckpoint = false;
    }
    if (flushOnCheckpoint) {
      pendingRecordsLock = new Object();
    }

    if (logFailuresOnly) {
      callback =
          new Callback() {
            @Override
            public void onCompletion(RecordMetadata metadata, Exception e) {
              if (e != null) {
                LOG.error("Error while sending record to Kafka: " + e.getMessage(), e);
              }
              acknowledgeMessage();
            }
          };
    } else {
      callback =
          new Callback() {
            @Override
            public void onCompletion(RecordMetadata metadata, Exception exception) {
              if (exception != null && asyncException == null) {
                asyncException = exception;
              }
              acknowledgeMessage();
            }
          };
    }
  }