/**
* Creates a new StreamSource.
*
* @param config Configuration to determine which stream to put records to and get {@link
* AWSCredentialsProvider}
* @param inputFile File containing record data to emit on each line
* @param loopOverStreamSource Loop over the stream source to continually put records
*/
public StreamSource(
KinesisConnectorConfiguration config, String inputFile, boolean loopOverStreamSource) {
this.config = config;
this.inputFile = inputFile;
this.loopOverInputFile = loopOverStreamSource;
this.objectMapper = new ObjectMapper();
kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
if (config.KINESIS_ENDPOINT != null) {
kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
}
KinesisUtils.createInputStream(config);
}
@Override
public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException {
// Store the contents of buffer.getRecords because superclass will
// clear the buffer on success
List<byte[]> failed = super.emit(buffer);
// calls S3Emitter to write objects to S3
if (!failed.isEmpty()) {
return buffer.getRecords();
}
String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber());
// wrap the name of the S3 file as the record data
ByteBuffer data = ByteBuffer.wrap(s3File.getBytes());
// Put the list of file names to the manifest Kinesis stream
PutRecordRequest putRecordRequest = new PutRecordRequest();
putRecordRequest.setData(data);
putRecordRequest.setStreamName(manifestStream);
// Use constant partition key to ensure file order
putRecordRequest.setPartitionKey(manifestStream);
try {
kinesisClient.putRecord(putRecordRequest);
LOG.info("S3ManifestEmitter emitted record downstream: " + s3File);
return Collections.emptyList();
} catch (AmazonServiceException e) {
LOG.error(e);
return buffer.getRecords();
}
}
/**
* Get the available shards from the kinesis
*
* @param streamName Name of the stream from where the shards to be accessed
* @return the list of shards from the given stream
*/
public List<Shard> getShardList(String streamName) {
assert client != null : "Illegal client";
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.setStreamName(streamName);
DescribeStreamResult describeResponse = client.describeStream(describeRequest);
return describeResponse.getStreamDescription().getShards();
}
/**
* Get the records from the particular shard
*
* @param streamName Name of the stream from where the records to be accessed
* @param recordsLimit Number of records to return from shard
* @param shId Shard Id of the shard
* @param iteratorType Shard iterator type
* @param seqNo Record sequence number
* @return the list of records from the given shard
* @throws AmazonClientException
*/
public List<Record> getRecords(
String streamName,
Integer recordsLimit,
String shId,
ShardIteratorType iteratorType,
String seqNo)
throws AmazonClientException {
assert client != null : "Illegal client";
try {
// Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType
// to it
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.setStreamName(streamName);
iteratorRequest.setShardId(shId);
iteratorRequest.setShardIteratorType(iteratorType);
// If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest
if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType)
|| ShardIteratorType.AT_SEQUENCE_NUMBER.equals(iteratorType))
iteratorRequest.setStartingSequenceNumber(seqNo);
// Get the Response from the getShardIterator service method & get the shardIterator from that
// response
GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);
// getShardIterator() specifies the position in the shard
String iterator = iteratorResponse.getShardIterator();
// Create the GetRecordsRequest instance and set the recordsLimit and iterator
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.setLimit(recordsLimit);
getRequest.setShardIterator(iterator);
// Get the Response from the getRecords service method and get the data records from that
// response.
GetRecordsResult getResponse = client.getRecords(getRequest);
return getResponse.getRecords();
} catch (AmazonClientException e) {
throw new RuntimeException(e);
}
}
/**
* Create the AmazonKinesisClient with the given credentials
*
* @param accessKey AWS accessKeyId
* @param secretKey AWS secretAccessKey
* @throws Exception
*/
public void createKinesisClient(String accessKey, String secretKey, String endPoint)
throws Exception {
if (client == null) {
try {
client = new AmazonKinesisClient(new BasicAWSCredentials(accessKey, secretKey));
if (endPoint != null) {
client.setEndpoint(endPoint);
}
} catch (Exception e) {
throw new AmazonClientException("Unable to load credentials", e);
}
}
}
/**
* Process the input file and send PutRecordRequests to Amazon Kinesis.
*
* <p>This function serves to Isolate StreamSource logic so subclasses can process input files
* differently.
*
* @param inputStream the input stream to process
* @param iteration the iteration if looping over file
* @throws IOException throw exception if error processing inputStream.
*/
protected void processInputStream(InputStream inputStream, int iteration) throws IOException {
try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) {
String line;
int lines = 0;
while ((line = br.readLine()) != null) {
KinesisMessageModel kinesisMessageModel =
objectMapper.readValue(line, KinesisMessageModel.class);
PutRecordRequest putRecordRequest = new PutRecordRequest();
putRecordRequest.setStreamName(config.KINESIS_INPUT_STREAM);
putRecordRequest.setData(ByteBuffer.wrap(line.getBytes()));
putRecordRequest.setPartitionKey(Integer.toString(kinesisMessageModel.getUserid()));
kinesisClient.putRecord(putRecordRequest);
lines++;
}
LOG.info("Added " + lines + " records to stream source.");
}
}
@Override
public void shutdown() {
super.shutdown();
kinesisClient.shutdown();
}
public S3ManifestEmitter(KinesisConnectorConfiguration configuration) {
super(configuration);
manifestStream = configuration.KINESIS_OUTPUT_STREAM;
kinesisClient = new AmazonKinesisClient(configuration.AWS_CREDENTIALS_PROVIDER);
kinesisClient.setEndpoint(configuration.KINESIS_ENDPOINT);
}