@Override public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException { // Store the contents of buffer.getRecords because superclass will // clear the buffer on success List<byte[]> failed = super.emit(buffer); // calls S3Emitter to write objects to S3 if (!failed.isEmpty()) { return buffer.getRecords(); } String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber()); // wrap the name of the S3 file as the record data ByteBuffer data = ByteBuffer.wrap(s3File.getBytes()); // Put the list of file names to the manifest Kinesis stream PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setData(data); putRecordRequest.setStreamName(manifestStream); // Use constant partition key to ensure file order putRecordRequest.setPartitionKey(manifestStream); try { kinesisClient.putRecord(putRecordRequest); LOG.info("S3ManifestEmitter emitted record downstream: " + s3File); return Collections.emptyList(); } catch (AmazonServiceException e) { LOG.error(e); return buffer.getRecords(); } }
/** * Process the input file and send PutRecordRequests to Amazon Kinesis. * * <p>This function serves to Isolate StreamSource logic so subclasses can process input files * differently. * * @param inputStream the input stream to process * @param iteration the iteration if looping over file * @throws IOException throw exception if error processing inputStream. */ protected void processInputStream(InputStream inputStream, int iteration) throws IOException { try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { String line; int lines = 0; while ((line = br.readLine()) != null) { KinesisMessageModel kinesisMessageModel = objectMapper.readValue(line, KinesisMessageModel.class); PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setStreamName(config.KINESIS_INPUT_STREAM); putRecordRequest.setData(ByteBuffer.wrap(line.getBytes())); putRecordRequest.setPartitionKey(Integer.toString(kinesisMessageModel.getUserid())); kinesisClient.putRecord(putRecordRequest); lines++; } LOG.info("Added " + lines + " records to stream source."); } }