Пример #1
0
  @Override
  public void invoke(OUT value) throws Exception {
    if (this.producer == null) {
      throw new RuntimeException("Kinesis producer has been closed");
    }
    if (thrownException != null) {
      String errorMessages = "";
      if (thrownException instanceof UserRecordFailedException) {
        List<Attempt> attempts =
            ((UserRecordFailedException) thrownException).getResult().getAttempts();
        for (Attempt attempt : attempts) {
          if (attempt.getErrorMessage() != null) {
            errorMessages += attempt.getErrorMessage() + "\n";
          }
        }
      }
      if (failOnError) {
        throw new RuntimeException(
            "An exception was thrown while processing a record: " + errorMessages, thrownException);
      } else {
        LOG.warn(
            "An exception was thrown while processing a record: {}",
            thrownException,
            errorMessages);
        thrownException = null; // reset
      }
    }

    String stream = defaultStream;
    String partition = defaultPartition;

    ByteBuffer serialized = schema.serialize(value);

    // maybe set custom stream
    String customStream = schema.getTargetStream(value);
    if (customStream != null) {
      stream = customStream;
    }

    String explicitHashkey = null;
    // maybe set custom partition
    if (customPartitioner != null) {
      partition = customPartitioner.getPartitionId(value);
      explicitHashkey = customPartitioner.getExplicitHashKey(value);
    }

    if (stream == null) {
      if (failOnError) {
        throw new RuntimeException("No target stream set");
      } else {
        LOG.warn("No target stream set. Skipping record");
        return;
      }
    }

    ListenableFuture<UserRecordResult> cb =
        producer.addUserRecord(stream, partition, explicitHashkey, serialized);
    Futures.addCallback(cb, callback);
  }
  public static void main(String[] args) throws InterruptedException {
    STREAM_NAME = args[0];

    final KinesisProducer producer = getKinesisProducer();
    for (int i = 100; i < 200; i++) {
      String json2 =
          "\n{\"id\": \""
              + i
              + "\",\"epochseconds\":123456792,\"state\":1,\"geoAttributes\": {\"location\": \"12.9715987,77.5945627\"}}\n";
      ByteBuffer data = ByteBuffer.wrap(json2.getBytes());
      producer.addUserRecord(STREAM_NAME, TIMESTAMP, Utils.randomExplicitHashKey(), data);
      producer.addUserRecord(STREAM_NAME, TIMESTAMP, data);

      Thread.sleep(100);
    }
    System.out.println("Put Result complete");
  }
Пример #3
0
 @Override
 public void close() throws Exception {
   LOG.info("Closing producer");
   super.close();
   KinesisProducer kp = this.producer;
   this.producer = null;
   if (kp != null) {
     LOG.info("Flushing outstanding {} records", kp.getOutstandingRecordsCount());
     // try to flush all outstanding records
     while (kp.getOutstandingRecordsCount() > 0) {
       kp.flush();
       try {
         Thread.sleep(500);
       } catch (InterruptedException e) {
         LOG.warn("Flushing was interrupted.");
         // stop the blocking flushing and destroy producer immediately
         break;
       }
     }
     LOG.info("Flushing done. Destroying producer instance.");
     kp.destroy();
   }
 }