@Override
public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException {
// Store the contents of buffer.getRecords because superclass will
// clear the buffer on success
List<byte[]> failed = super.emit(buffer);
// calls S3Emitter to write objects to S3
if (!failed.isEmpty()) {
return buffer.getRecords();
}
String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber());
// wrap the name of the S3 file as the record data
ByteBuffer data = ByteBuffer.wrap(s3File.getBytes());
// Put the list of file names to the manifest Kinesis stream
PutRecordRequest putRecordRequest = new PutRecordRequest();
putRecordRequest.setData(data);
putRecordRequest.setStreamName(manifestStream);
// Use constant partition key to ensure file order
putRecordRequest.setPartitionKey(manifestStream);
try {
kinesisClient.putRecord(putRecordRequest);
LOG.info("S3ManifestEmitter emitted record downstream: " + s3File);
return Collections.emptyList();
} catch (AmazonServiceException e) {
LOG.error(e);
return buffer.getRecords();
}
}
@Test
public void convertRequestToJSONAllElementsIntact() {
PutRecordRequest putRequest = new PutRecordRequest();
putRequest
.withStreamName(stream)
.withPartitionKey(partitionKey)
.withExplicitHashKey(explicitHashKey)
.withData(data)
.withSequenceNumberForOrdering(sequenceNumberForOrdering);
JSONRecordAdapter adapter = new JSONRecordAdapter();
JSONObject json = adapter.translateFromRecord(putRequest);
try {
assertTrue(json.getString(JSONRecordAdapter.STREAM_NAME_FIELD).equalsIgnoreCase(stream));
assertTrue(
json.getString(JSONRecordAdapter.DATA_FIELD_KEY)
.equalsIgnoreCase(
Base64.encodeToString("TestData".getBytes(StringUtils.UTF8), Base64.DEFAULT)));
assertTrue(
json.getString(JSONRecordAdapter.PARTITION_KEY_FIELD).equalsIgnoreCase(partitionKey));
assertTrue(
json.getString(JSONRecordAdapter.SEQUENCE_NUMBER_FIELD)
.equalsIgnoreCase(sequenceNumberForOrdering));
assertTrue(
json.getString(JSONRecordAdapter.EXPLICIT_HASH_FIELD).equalsIgnoreCase(explicitHashKey));
} catch (JSONException e) {
e.printStackTrace();
fail(e.getMessage());
}
}
@Test(expected = AmazonClientException.class)
public void convertRequestWhenStreamNull() {
PutRecordRequest putRequest = new PutRecordRequest();
putRequest.withPartitionKey(partitionKey).withData(data);
JSONRecordAdapter adapter = new JSONRecordAdapter();
adapter.translateFromRecord(putRequest);
}
@Test(expected = AmazonClientException.class)
public void convertRequestWhenPartitionKeyEmpty() {
PutRecordRequest putRequest = new PutRecordRequest();
String partitionKey = "";
putRequest.withStreamName(stream).withData(data).withPartitionKey(partitionKey);
JSONRecordAdapter adapter = new JSONRecordAdapter();
adapter.translateFromRecord(putRequest);
}
@Test
public void testStaticGetters() throws JSONException {
PutRecordRequest putRequest = new PutRecordRequest();
putRequest.withStreamName(stream).withPartitionKey(partitionKey).withData(data);
JSONRecordAdapter adapter = new JSONRecordAdapter();
JSONObject json = adapter.translateFromRecord(putRequest);
assertArrayEquals(data.array(), JSONRecordAdapter.getData(json).array());
assertTrue(partitionKey.equalsIgnoreCase(JSONRecordAdapter.getPartitionKey(json)));
assertTrue(stream.equalsIgnoreCase(JSONRecordAdapter.getStreamName(json)));
}
private static void put_record(AmazonKinesis client, int current_index) {
PutRecordRequest putRecord = new PutRecordRequest();
putRecord.setStreamName(Constant.STREAM_NAME);
putRecord.setPartitionKey(get_partition_key(current_index));
putRecord.setExplicitHashKey(get_explicit_key(current_index));
putRecord.setData(ByteBuffer.wrap((current_index + "").getBytes()));
try {
PutRecordResult pr = client.putRecord(putRecord);
System.out.println("put record : " + current_index + " shard id: " + pr.getShardId());
} catch (AmazonClientException ex) {
LOG.warn("Error sending record to Amazon Kinesis.", ex);
}
}
public static void sendObject(Object objectToSend, StreamName streamName) throws Exception {
byte[] bytes = Piping.serialize(objectToSend).getBytes();
PutRecordRequest putRecord = new PutRecordRequest();
putRecord.setStreamName(streamName.toString());
// We use the resource as the partition key so we can accurately calculate totals for a given
// resource
putRecord.setPartitionKey(objectToSend.toString());
putRecord.setData(ByteBuffer.wrap(bytes));
// Order is not important for this application so we do not send a SequenceNumberForOrdering
putRecord.setSequenceNumberForOrdering(null);
kinesis.putRecord(putRecord);
}
/**
* Process the input file and send PutRecordRequests to Amazon Kinesis.
*
* <p>This function serves to Isolate StreamSource logic so subclasses can process input files
* differently.
*
* @param inputStream the input stream to process
* @param iteration the iteration if looping over file
* @throws IOException throw exception if error processing inputStream.
*/
protected void processInputStream(InputStream inputStream, int iteration) throws IOException {
try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) {
String line;
int lines = 0;
while ((line = br.readLine()) != null) {
KinesisMessageModel kinesisMessageModel =
objectMapper.readValue(line, KinesisMessageModel.class);
PutRecordRequest putRecordRequest = new PutRecordRequest();
putRecordRequest.setStreamName(config.KINESIS_INPUT_STREAM);
putRecordRequest.setData(ByteBuffer.wrap(line.getBytes()));
putRecordRequest.setPartitionKey(Integer.toString(kinesisMessageModel.getUserid()));
kinesisClient.putRecord(putRecordRequest);
lines++;
}
LOG.info("Added " + lines + " records to stream source.");
}
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (obj instanceof PutRecordRequest == false) return false;
PutRecordRequest other = (PutRecordRequest) obj;
if (other.getStreamName() == null ^ this.getStreamName() == null) return false;
if (other.getStreamName() != null
&& other.getStreamName().equals(this.getStreamName()) == false) return false;
if (other.getData() == null ^ this.getData() == null) return false;
if (other.getData() != null && other.getData().equals(this.getData()) == false) return false;
if (other.getPartitionKey() == null ^ this.getPartitionKey() == null) return false;
if (other.getPartitionKey() != null
&& other.getPartitionKey().equals(this.getPartitionKey()) == false) return false;
if (other.getExplicitHashKey() == null ^ this.getExplicitHashKey() == null) return false;
if (other.getExplicitHashKey() != null
&& other.getExplicitHashKey().equals(this.getExplicitHashKey()) == false) return false;
if (other.getSequenceNumberForOrdering() == null ^ this.getSequenceNumberForOrdering() == null)
return false;
if (other.getSequenceNumberForOrdering() != null
&& other.getSequenceNumberForOrdering().equals(this.getSequenceNumberForOrdering())
== false) return false;
return true;
}