/**
  * Creates a new StreamSource.
  *
  * @param config Configuration to determine which stream to put records to and get {@link
  *     AWSCredentialsProvider}
  * @param inputFile File containing record data to emit on each line
  * @param loopOverStreamSource Loop over the stream source to continually put records
  */
 public StreamSource(
     KinesisConnectorConfiguration config, String inputFile, boolean loopOverStreamSource) {
   this.config = config;
   this.inputFile = inputFile;
   this.loopOverInputFile = loopOverStreamSource;
   this.objectMapper = new ObjectMapper();
   kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
   kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
   if (config.KINESIS_ENDPOINT != null) {
     kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
   }
   KinesisUtils.createInputStream(config);
 }
 @Override
 public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException {
   // Store the contents of buffer.getRecords because superclass will
   // clear the buffer on success
   List<byte[]> failed = super.emit(buffer);
   // calls S3Emitter to write objects to S3
   if (!failed.isEmpty()) {
     return buffer.getRecords();
   }
   String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber());
   // wrap the name of the S3 file as the record data
   ByteBuffer data = ByteBuffer.wrap(s3File.getBytes());
   // Put the list of file names to the manifest Kinesis stream
   PutRecordRequest putRecordRequest = new PutRecordRequest();
   putRecordRequest.setData(data);
   putRecordRequest.setStreamName(manifestStream);
   // Use constant partition key to ensure file order
   putRecordRequest.setPartitionKey(manifestStream);
   try {
     kinesisClient.putRecord(putRecordRequest);
     LOG.info("S3ManifestEmitter emitted record downstream: " + s3File);
     return Collections.emptyList();
   } catch (AmazonServiceException e) {
     LOG.error(e);
     return buffer.getRecords();
   }
 }
Exemple #3
0
  /**
   * Get the available shards from the kinesis
   *
   * @param streamName Name of the stream from where the shards to be accessed
   * @return the list of shards from the given stream
   */
  public List<Shard> getShardList(String streamName) {
    assert client != null : "Illegal client";
    DescribeStreamRequest describeRequest = new DescribeStreamRequest();
    describeRequest.setStreamName(streamName);

    DescribeStreamResult describeResponse = client.describeStream(describeRequest);
    return describeResponse.getStreamDescription().getShards();
  }
Exemple #4
0
  /**
   * Get the records from the particular shard
   *
   * @param streamName Name of the stream from where the records to be accessed
   * @param recordsLimit Number of records to return from shard
   * @param shId Shard Id of the shard
   * @param iteratorType Shard iterator type
   * @param seqNo Record sequence number
   * @return the list of records from the given shard
   * @throws AmazonClientException
   */
  public List<Record> getRecords(
      String streamName,
      Integer recordsLimit,
      String shId,
      ShardIteratorType iteratorType,
      String seqNo)
      throws AmazonClientException {
    assert client != null : "Illegal client";
    try {
      // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType
      // to it
      GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
      iteratorRequest.setStreamName(streamName);
      iteratorRequest.setShardId(shId);
      iteratorRequest.setShardIteratorType(iteratorType);

      // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest
      if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType)
          || ShardIteratorType.AT_SEQUENCE_NUMBER.equals(iteratorType))
        iteratorRequest.setStartingSequenceNumber(seqNo);

      // Get the Response from the getShardIterator service method & get the shardIterator from that
      // response
      GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);
      // getShardIterator() specifies the position in the shard
      String iterator = iteratorResponse.getShardIterator();

      // Create the GetRecordsRequest instance and set the recordsLimit and iterator
      GetRecordsRequest getRequest = new GetRecordsRequest();
      getRequest.setLimit(recordsLimit);
      getRequest.setShardIterator(iterator);

      // Get the Response from the getRecords service method and get the data records from that
      // response.
      GetRecordsResult getResponse = client.getRecords(getRequest);
      return getResponse.getRecords();
    } catch (AmazonClientException e) {
      throw new RuntimeException(e);
    }
  }
Exemple #5
0
 /**
  * Create the AmazonKinesisClient with the given credentials
  *
  * @param accessKey AWS accessKeyId
  * @param secretKey AWS secretAccessKey
  * @throws Exception
  */
 public void createKinesisClient(String accessKey, String secretKey, String endPoint)
     throws Exception {
   if (client == null) {
     try {
       client = new AmazonKinesisClient(new BasicAWSCredentials(accessKey, secretKey));
       if (endPoint != null) {
         client.setEndpoint(endPoint);
       }
     } catch (Exception e) {
       throw new AmazonClientException("Unable to load credentials", e);
     }
   }
 }
  /**
   * Process the input file and send PutRecordRequests to Amazon Kinesis.
   *
   * <p>This function serves to Isolate StreamSource logic so subclasses can process input files
   * differently.
   *
   * @param inputStream the input stream to process
   * @param iteration the iteration if looping over file
   * @throws IOException throw exception if error processing inputStream.
   */
  protected void processInputStream(InputStream inputStream, int iteration) throws IOException {
    try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) {
      String line;
      int lines = 0;
      while ((line = br.readLine()) != null) {
        KinesisMessageModel kinesisMessageModel =
            objectMapper.readValue(line, KinesisMessageModel.class);

        PutRecordRequest putRecordRequest = new PutRecordRequest();
        putRecordRequest.setStreamName(config.KINESIS_INPUT_STREAM);
        putRecordRequest.setData(ByteBuffer.wrap(line.getBytes()));
        putRecordRequest.setPartitionKey(Integer.toString(kinesisMessageModel.getUserid()));
        kinesisClient.putRecord(putRecordRequest);
        lines++;
      }
      LOG.info("Added " + lines + " records to stream source.");
    }
  }
 @Override
 public void shutdown() {
   super.shutdown();
   kinesisClient.shutdown();
 }
 public S3ManifestEmitter(KinesisConnectorConfiguration configuration) {
   super(configuration);
   manifestStream = configuration.KINESIS_OUTPUT_STREAM;
   kinesisClient = new AmazonKinesisClient(configuration.AWS_CREDENTIALS_PROVIDER);
   kinesisClient.setEndpoint(configuration.KINESIS_ENDPOINT);
 }