@Override
 public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException {
   // Store the contents of buffer.getRecords because superclass will
   // clear the buffer on success
   List<byte[]> failed = super.emit(buffer);
   // calls S3Emitter to write objects to S3
   if (!failed.isEmpty()) {
     return buffer.getRecords();
   }
   String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber());
   // wrap the name of the S3 file as the record data
   ByteBuffer data = ByteBuffer.wrap(s3File.getBytes());
   // Put the list of file names to the manifest Kinesis stream
   PutRecordRequest putRecordRequest = new PutRecordRequest();
   putRecordRequest.setData(data);
   putRecordRequest.setStreamName(manifestStream);
   // Use constant partition key to ensure file order
   putRecordRequest.setPartitionKey(manifestStream);
   try {
     kinesisClient.putRecord(putRecordRequest);
     LOG.info("S3ManifestEmitter emitted record downstream: " + s3File);
     return Collections.emptyList();
   } catch (AmazonServiceException e) {
     LOG.error(e);
     return buffer.getRecords();
   }
 }
示例#2
0
  @Override
  public List<KinesisMessageModel> emit(final UnmodifiableBuffer<KinesisMessageModel> buffer) {
    List<KinesisMessageModel> items = buffer.getRecords();
    List<KinesisMessageModel> failedItems = new ArrayList<KinesisMessageModel>();
    // failedItems.

    DynamoDBMapper mapper = new DynamoDBMapper(dynamoDBClient);

    KinesisMessageModel message = null;
    try {
      mapper.batchSave(items);
      LOG.info("Successfully emitted: " + items.size() + " items");

    } catch (AmazonClientException e) {
      e.printStackTrace();

      //
    }

    // LOG.info("Successfully emitted " + (items.size() - failedItems.size()) + " records into
    // DynamoDB.");
    // failedItems.
    // return failedItems;

    return failedItems;
  }