@Override
 public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException {
   // Store the contents of buffer.getRecords because superclass will
   // clear the buffer on success
   List<byte[]> failed = super.emit(buffer);
   // calls S3Emitter to write objects to S3
   if (!failed.isEmpty()) {
     return buffer.getRecords();
   }
   String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber());
   // wrap the name of the S3 file as the record data
   ByteBuffer data = ByteBuffer.wrap(s3File.getBytes());
   // Put the list of file names to the manifest Kinesis stream
   PutRecordRequest putRecordRequest = new PutRecordRequest();
   putRecordRequest.setData(data);
   putRecordRequest.setStreamName(manifestStream);
   // Use constant partition key to ensure file order
   putRecordRequest.setPartitionKey(manifestStream);
   try {
     kinesisClient.putRecord(putRecordRequest);
     LOG.info("S3ManifestEmitter emitted record downstream: " + s3File);
     return Collections.emptyList();
   } catch (AmazonServiceException e) {
     LOG.error(e);
     return buffer.getRecords();
   }
 }
Ejemplo n.º 2
0
  private static void put_record(AmazonKinesis client, int current_index) {
    PutRecordRequest putRecord = new PutRecordRequest();
    putRecord.setStreamName(Constant.STREAM_NAME);
    putRecord.setPartitionKey(get_partition_key(current_index));
    putRecord.setExplicitHashKey(get_explicit_key(current_index));
    putRecord.setData(ByteBuffer.wrap((current_index + "").getBytes()));

    try {
      PutRecordResult pr = client.putRecord(putRecord);
      System.out.println("put record : " + current_index + "  shard id: " + pr.getShardId());
    } catch (AmazonClientException ex) {
      LOG.warn("Error sending record to Amazon Kinesis.", ex);
    }
  }
Ejemplo n.º 3
0
  public static void sendObject(Object objectToSend, StreamName streamName) throws Exception {

    byte[] bytes = Piping.serialize(objectToSend).getBytes();

    PutRecordRequest putRecord = new PutRecordRequest();
    putRecord.setStreamName(streamName.toString());

    // We use the resource as the partition key so we can accurately calculate totals for a given
    // resource
    putRecord.setPartitionKey(objectToSend.toString());
    putRecord.setData(ByteBuffer.wrap(bytes));

    // Order is not important for this application so we do not send a SequenceNumberForOrdering
    putRecord.setSequenceNumberForOrdering(null);

    kinesis.putRecord(putRecord);
  }
Ejemplo n.º 4
0
  /**
   * Process the input file and send PutRecordRequests to Amazon Kinesis.
   *
   * <p>This function serves to Isolate StreamSource logic so subclasses can process input files
   * differently.
   *
   * @param inputStream the input stream to process
   * @param iteration the iteration if looping over file
   * @throws IOException throw exception if error processing inputStream.
   */
  protected void processInputStream(InputStream inputStream, int iteration) throws IOException {
    try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) {
      String line;
      int lines = 0;
      while ((line = br.readLine()) != null) {
        KinesisMessageModel kinesisMessageModel =
            objectMapper.readValue(line, KinesisMessageModel.class);

        PutRecordRequest putRecordRequest = new PutRecordRequest();
        putRecordRequest.setStreamName(config.KINESIS_INPUT_STREAM);
        putRecordRequest.setData(ByteBuffer.wrap(line.getBytes()));
        putRecordRequest.setPartitionKey(Integer.toString(kinesisMessageModel.getUserid()));
        kinesisClient.putRecord(putRecordRequest);
        lines++;
      }
      LOG.info("Added " + lines + " records to stream source.");
    }
  }
 /**
  * The name of the stream to put the data record into.
  *
  * @param streamName The name of the stream to put the data record into.
  * @return Returns a reference to this object so that method calls can be chained together.
  */
 public PutRecordRequest withStreamName(String streamName) {
   setStreamName(streamName);
   return this;
 }