public static KinesisProducer getKinesisProducer() {
// There are many configurable parameters in the KPL. See the javadocs
// on each each set method for details.
KinesisProducerConfiguration config = new KinesisProducerConfiguration();
// You can also load config from file. A sample properties file is
// included in the project folder.
// KinesisProducerConfiguration config =
// KinesisProducerConfiguration.fromPropertiesFile("default_config.properties");
// If you're running in EC2 and want to use the same Kinesis region as
// the one your instance is in, you can simply leave out the region
// configuration; the KPL will retrieve it from EC2 metadata.
config.setRegion(REGION);
// You can pass credentials programmatically through the configuration,
// similar to the AWS SDK. DefaultAWSCredentialsProviderChain is used
// by default, so this configuration can be omitted if that is all
// that is needed.
// config.setCredentialsProvider(new DefaultAWSCredentialsProviderChain());
// The maxConnections parameter can be used to control the degree of
// parallelism when making HTTP requests. We're going to use only 1 here
// since our throughput is fairly low. Using a high number will cause a
// bunch of broken pipe errors to show up in the logs. This is due to
// idle connections being closed by the server. Setting this value too
// large may also cause request timeouts if you do not have enough
// bandwidth.
config.setMaxConnections(1);
// Set a more generous timeout in case we're on a slow connection.
config.setRequestTimeout(60000);
// RecordMaxBufferedTime controls how long records are allowed to wait
// in the KPL's buffers before being sent. Larger values increase
// aggregation and reduces the number of Kinesis records put, which can
// be helpful if you're getting throttled because of the records per
// second limit on a shard. The default value is set very low to
// minimize propagation delay, so we'll increase it here to get more
// aggregation.
config.setRecordMaxBufferedTime(1500);
// If you have built the native binary yourself, you can point the Java
// wrapper to it with the NativeExecutable option. If you want to pass
// environment variables to the executable, you can either use a wrapper
// shell script, or set them for the Java process, which will then pass
// them on to the child process.
// config.setNativeExecutable("my_directory/kinesis_producer");
// If you end up using the default configuration (a Configuration instance
// without any calls to set*), you can just leave the config argument
// out.
//
// Note that if you do pass a Configuration instance, mutating that
// instance after initializing KinesisProducer has no effect. We do not
// support dynamic re-configuration at the moment.
KinesisProducer producer = new KinesisProducer(config);
return producer;
}
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
KinesisProducerConfiguration producerConfig = new KinesisProducerConfiguration();
producerConfig.setRegion(configProps.getProperty(ProducerConfigConstants.AWS_REGION));
producerConfig.setCredentialsProvider(AWSUtil.getCredentialsProvider(configProps));
if (configProps.containsKey(ProducerConfigConstants.COLLECTION_MAX_COUNT)) {
producerConfig.setCollectionMaxCount(
PropertiesUtil.getLong(
configProps,
ProducerConfigConstants.COLLECTION_MAX_COUNT,
producerConfig.getCollectionMaxCount(),
LOG));
}
if (configProps.containsKey(ProducerConfigConstants.AGGREGATION_MAX_COUNT)) {
producerConfig.setAggregationMaxCount(
PropertiesUtil.getLong(
configProps,
ProducerConfigConstants.AGGREGATION_MAX_COUNT,
producerConfig.getAggregationMaxCount(),
LOG));
}
producer = new KinesisProducer(producerConfig);
callback =
new FutureCallback<UserRecordResult>() {
@Override
public void onSuccess(UserRecordResult result) {
if (!result.isSuccessful()) {
if (failOnError) {
thrownException = new RuntimeException("Record was not sent successful");
} else {
LOG.warn("Record was not sent successful");
}
}
}
@Override
public void onFailure(Throwable t) {
if (failOnError) {
thrownException = t;
} else {
LOG.warn("An exception occurred while processing a record", t);
}
}
};
if (this.customPartitioner != null) {
this.customPartitioner.initialize(
getRuntimeContext().getIndexOfThisSubtask(),
getRuntimeContext().getNumberOfParallelSubtasks());
}
LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}