@Override
public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException {
// Store the contents of buffer.getRecords because superclass will
// clear the buffer on success
List<byte[]> failed = super.emit(buffer);
// calls S3Emitter to write objects to S3
if (!failed.isEmpty()) {
return buffer.getRecords();
}
String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber());
// wrap the name of the S3 file as the record data
ByteBuffer data = ByteBuffer.wrap(s3File.getBytes());
// Put the list of file names to the manifest Kinesis stream
PutRecordRequest putRecordRequest = new PutRecordRequest();
putRecordRequest.setData(data);
putRecordRequest.setStreamName(manifestStream);
// Use constant partition key to ensure file order
putRecordRequest.setPartitionKey(manifestStream);
try {
kinesisClient.putRecord(putRecordRequest);
LOG.info("S3ManifestEmitter emitted record downstream: " + s3File);
return Collections.emptyList();
} catch (AmazonServiceException e) {
LOG.error(e);
return buffer.getRecords();
}
}
/**
* Process the input file and send PutRecordRequests to Amazon Kinesis.
*
* <p>This function serves to Isolate StreamSource logic so subclasses can process input files
* differently.
*
* @param inputStream the input stream to process
* @param iteration the iteration if looping over file
* @throws IOException throw exception if error processing inputStream.
*/
protected void processInputStream(InputStream inputStream, int iteration) throws IOException {
try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) {
String line;
int lines = 0;
while ((line = br.readLine()) != null) {
KinesisMessageModel kinesisMessageModel =
objectMapper.readValue(line, KinesisMessageModel.class);
PutRecordRequest putRecordRequest = new PutRecordRequest();
putRecordRequest.setStreamName(config.KINESIS_INPUT_STREAM);
putRecordRequest.setData(ByteBuffer.wrap(line.getBytes()));
putRecordRequest.setPartitionKey(Integer.toString(kinesisMessageModel.getUserid()));
kinesisClient.putRecord(putRecordRequest);
lines++;
}
LOG.info("Added " + lines + " records to stream source.");
}
}
