@Override public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException { // Store the contents of buffer.getRecords because superclass will // clear the buffer on success List<byte[]> failed = super.emit(buffer); // calls S3Emitter to write objects to S3 if (!failed.isEmpty()) { return buffer.getRecords(); } String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber()); // wrap the name of the S3 file as the record data ByteBuffer data = ByteBuffer.wrap(s3File.getBytes()); // Put the list of file names to the manifest Kinesis stream PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setData(data); putRecordRequest.setStreamName(manifestStream); // Use constant partition key to ensure file order putRecordRequest.setPartitionKey(manifestStream); try { kinesisClient.putRecord(putRecordRequest); LOG.info("S3ManifestEmitter emitted record downstream: " + s3File); return Collections.emptyList(); } catch (AmazonServiceException e) { LOG.error(e); return buffer.getRecords(); } }
@Test public void convertRequestToJSONAllElementsIntact() { PutRecordRequest putRequest = new PutRecordRequest(); putRequest .withStreamName(stream) .withPartitionKey(partitionKey) .withExplicitHashKey(explicitHashKey) .withData(data) .withSequenceNumberForOrdering(sequenceNumberForOrdering); JSONRecordAdapter adapter = new JSONRecordAdapter(); JSONObject json = adapter.translateFromRecord(putRequest); try { assertTrue(json.getString(JSONRecordAdapter.STREAM_NAME_FIELD).equalsIgnoreCase(stream)); assertTrue( json.getString(JSONRecordAdapter.DATA_FIELD_KEY) .equalsIgnoreCase( Base64.encodeToString("TestData".getBytes(StringUtils.UTF8), Base64.DEFAULT))); assertTrue( json.getString(JSONRecordAdapter.PARTITION_KEY_FIELD).equalsIgnoreCase(partitionKey)); assertTrue( json.getString(JSONRecordAdapter.SEQUENCE_NUMBER_FIELD) .equalsIgnoreCase(sequenceNumberForOrdering)); assertTrue( json.getString(JSONRecordAdapter.EXPLICIT_HASH_FIELD).equalsIgnoreCase(explicitHashKey)); } catch (JSONException e) { e.printStackTrace(); fail(e.getMessage()); } }
@Test(expected = AmazonClientException.class) public void convertRequestWhenStreamNull() { PutRecordRequest putRequest = new PutRecordRequest(); putRequest.withPartitionKey(partitionKey).withData(data); JSONRecordAdapter adapter = new JSONRecordAdapter(); adapter.translateFromRecord(putRequest); }
@Test(expected = AmazonClientException.class) public void convertRequestWhenPartitionKeyEmpty() { PutRecordRequest putRequest = new PutRecordRequest(); String partitionKey = ""; putRequest.withStreamName(stream).withData(data).withPartitionKey(partitionKey); JSONRecordAdapter adapter = new JSONRecordAdapter(); adapter.translateFromRecord(putRequest); }
@Test public void testStaticGetters() throws JSONException { PutRecordRequest putRequest = new PutRecordRequest(); putRequest.withStreamName(stream).withPartitionKey(partitionKey).withData(data); JSONRecordAdapter adapter = new JSONRecordAdapter(); JSONObject json = adapter.translateFromRecord(putRequest); assertArrayEquals(data.array(), JSONRecordAdapter.getData(json).array()); assertTrue(partitionKey.equalsIgnoreCase(JSONRecordAdapter.getPartitionKey(json))); assertTrue(stream.equalsIgnoreCase(JSONRecordAdapter.getStreamName(json))); }
private static void put_record(AmazonKinesis client, int current_index) { PutRecordRequest putRecord = new PutRecordRequest(); putRecord.setStreamName(Constant.STREAM_NAME); putRecord.setPartitionKey(get_partition_key(current_index)); putRecord.setExplicitHashKey(get_explicit_key(current_index)); putRecord.setData(ByteBuffer.wrap((current_index + "").getBytes())); try { PutRecordResult pr = client.putRecord(putRecord); System.out.println("put record : " + current_index + " shard id: " + pr.getShardId()); } catch (AmazonClientException ex) { LOG.warn("Error sending record to Amazon Kinesis.", ex); } }
public static void sendObject(Object objectToSend, StreamName streamName) throws Exception { byte[] bytes = Piping.serialize(objectToSend).getBytes(); PutRecordRequest putRecord = new PutRecordRequest(); putRecord.setStreamName(streamName.toString()); // We use the resource as the partition key so we can accurately calculate totals for a given // resource putRecord.setPartitionKey(objectToSend.toString()); putRecord.setData(ByteBuffer.wrap(bytes)); // Order is not important for this application so we do not send a SequenceNumberForOrdering putRecord.setSequenceNumberForOrdering(null); kinesis.putRecord(putRecord); }
/** * Process the input file and send PutRecordRequests to Amazon Kinesis. * * <p>This function serves to Isolate StreamSource logic so subclasses can process input files * differently. * * @param inputStream the input stream to process * @param iteration the iteration if looping over file * @throws IOException throw exception if error processing inputStream. */ protected void processInputStream(InputStream inputStream, int iteration) throws IOException { try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { String line; int lines = 0; while ((line = br.readLine()) != null) { KinesisMessageModel kinesisMessageModel = objectMapper.readValue(line, KinesisMessageModel.class); PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setStreamName(config.KINESIS_INPUT_STREAM); putRecordRequest.setData(ByteBuffer.wrap(line.getBytes())); putRecordRequest.setPartitionKey(Integer.toString(kinesisMessageModel.getUserid())); kinesisClient.putRecord(putRecordRequest); lines++; } LOG.info("Added " + lines + " records to stream source."); } }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof PutRecordRequest == false) return false; PutRecordRequest other = (PutRecordRequest) obj; if (other.getStreamName() == null ^ this.getStreamName() == null) return false; if (other.getStreamName() != null && other.getStreamName().equals(this.getStreamName()) == false) return false; if (other.getData() == null ^ this.getData() == null) return false; if (other.getData() != null && other.getData().equals(this.getData()) == false) return false; if (other.getPartitionKey() == null ^ this.getPartitionKey() == null) return false; if (other.getPartitionKey() != null && other.getPartitionKey().equals(this.getPartitionKey()) == false) return false; if (other.getExplicitHashKey() == null ^ this.getExplicitHashKey() == null) return false; if (other.getExplicitHashKey() != null && other.getExplicitHashKey().equals(this.getExplicitHashKey()) == false) return false; if (other.getSequenceNumberForOrdering() == null ^ this.getSequenceNumberForOrdering() == null) return false; if (other.getSequenceNumberForOrdering() != null && other.getSequenceNumberForOrdering().equals(this.getSequenceNumberForOrdering()) == false) return false; return true; }