@Override
 public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException {
   // Store the contents of buffer.getRecords because superclass will
   // clear the buffer on success
   List<byte[]> failed = super.emit(buffer);
   // calls S3Emitter to write objects to S3
   if (!failed.isEmpty()) {
     return buffer.getRecords();
   }
   String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber());
   // wrap the name of the S3 file as the record data
   ByteBuffer data = ByteBuffer.wrap(s3File.getBytes());
   // Put the list of file names to the manifest Kinesis stream
   PutRecordRequest putRecordRequest = new PutRecordRequest();
   putRecordRequest.setData(data);
   putRecordRequest.setStreamName(manifestStream);
   // Use constant partition key to ensure file order
   putRecordRequest.setPartitionKey(manifestStream);
   try {
     kinesisClient.putRecord(putRecordRequest);
     LOG.info("S3ManifestEmitter emitted record downstream: " + s3File);
     return Collections.emptyList();
   } catch (AmazonServiceException e) {
     LOG.error(e);
     return buffer.getRecords();
   }
 }
  @Test
  public void convertRequestToJSONAllElementsIntact() {
    PutRecordRequest putRequest = new PutRecordRequest();

    putRequest
        .withStreamName(stream)
        .withPartitionKey(partitionKey)
        .withExplicitHashKey(explicitHashKey)
        .withData(data)
        .withSequenceNumberForOrdering(sequenceNumberForOrdering);

    JSONRecordAdapter adapter = new JSONRecordAdapter();
    JSONObject json = adapter.translateFromRecord(putRequest);

    try {
      assertTrue(json.getString(JSONRecordAdapter.STREAM_NAME_FIELD).equalsIgnoreCase(stream));
      assertTrue(
          json.getString(JSONRecordAdapter.DATA_FIELD_KEY)
              .equalsIgnoreCase(
                  Base64.encodeToString("TestData".getBytes(StringUtils.UTF8), Base64.DEFAULT)));
      assertTrue(
          json.getString(JSONRecordAdapter.PARTITION_KEY_FIELD).equalsIgnoreCase(partitionKey));
      assertTrue(
          json.getString(JSONRecordAdapter.SEQUENCE_NUMBER_FIELD)
              .equalsIgnoreCase(sequenceNumberForOrdering));
      assertTrue(
          json.getString(JSONRecordAdapter.EXPLICIT_HASH_FIELD).equalsIgnoreCase(explicitHashKey));

    } catch (JSONException e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }
  @Test(expected = AmazonClientException.class)
  public void convertRequestWhenStreamNull() {
    PutRecordRequest putRequest = new PutRecordRequest();
    putRequest.withPartitionKey(partitionKey).withData(data);

    JSONRecordAdapter adapter = new JSONRecordAdapter();
    adapter.translateFromRecord(putRequest);
  }
  @Test(expected = AmazonClientException.class)
  public void convertRequestWhenPartitionKeyEmpty() {
    PutRecordRequest putRequest = new PutRecordRequest();
    String partitionKey = "";
    putRequest.withStreamName(stream).withData(data).withPartitionKey(partitionKey);

    JSONRecordAdapter adapter = new JSONRecordAdapter();
    adapter.translateFromRecord(putRequest);
  }
  @Test
  public void testStaticGetters() throws JSONException {

    PutRecordRequest putRequest = new PutRecordRequest();
    putRequest.withStreamName(stream).withPartitionKey(partitionKey).withData(data);

    JSONRecordAdapter adapter = new JSONRecordAdapter();
    JSONObject json = adapter.translateFromRecord(putRequest);

    assertArrayEquals(data.array(), JSONRecordAdapter.getData(json).array());
    assertTrue(partitionKey.equalsIgnoreCase(JSONRecordAdapter.getPartitionKey(json)));
    assertTrue(stream.equalsIgnoreCase(JSONRecordAdapter.getStreamName(json)));
  }
Exemplo n.º 6
0
  private static void put_record(AmazonKinesis client, int current_index) {
    PutRecordRequest putRecord = new PutRecordRequest();
    putRecord.setStreamName(Constant.STREAM_NAME);
    putRecord.setPartitionKey(get_partition_key(current_index));
    putRecord.setExplicitHashKey(get_explicit_key(current_index));
    putRecord.setData(ByteBuffer.wrap((current_index + "").getBytes()));

    try {
      PutRecordResult pr = client.putRecord(putRecord);
      System.out.println("put record : " + current_index + "  shard id: " + pr.getShardId());
    } catch (AmazonClientException ex) {
      LOG.warn("Error sending record to Amazon Kinesis.", ex);
    }
  }
Exemplo n.º 7
0
  public static void sendObject(Object objectToSend, StreamName streamName) throws Exception {

    byte[] bytes = Piping.serialize(objectToSend).getBytes();

    PutRecordRequest putRecord = new PutRecordRequest();
    putRecord.setStreamName(streamName.toString());

    // We use the resource as the partition key so we can accurately calculate totals for a given
    // resource
    putRecord.setPartitionKey(objectToSend.toString());
    putRecord.setData(ByteBuffer.wrap(bytes));

    // Order is not important for this application so we do not send a SequenceNumberForOrdering
    putRecord.setSequenceNumberForOrdering(null);

    kinesis.putRecord(putRecord);
  }
Exemplo n.º 8
0
  /**
   * Process the input file and send PutRecordRequests to Amazon Kinesis.
   *
   * <p>This function serves to Isolate StreamSource logic so subclasses can process input files
   * differently.
   *
   * @param inputStream the input stream to process
   * @param iteration the iteration if looping over file
   * @throws IOException throw exception if error processing inputStream.
   */
  protected void processInputStream(InputStream inputStream, int iteration) throws IOException {
    try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) {
      String line;
      int lines = 0;
      while ((line = br.readLine()) != null) {
        KinesisMessageModel kinesisMessageModel =
            objectMapper.readValue(line, KinesisMessageModel.class);

        PutRecordRequest putRecordRequest = new PutRecordRequest();
        putRecordRequest.setStreamName(config.KINESIS_INPUT_STREAM);
        putRecordRequest.setData(ByteBuffer.wrap(line.getBytes()));
        putRecordRequest.setPartitionKey(Integer.toString(kinesisMessageModel.getUserid()));
        kinesisClient.putRecord(putRecordRequest);
        lines++;
      }
      LOG.info("Added " + lines + " records to stream source.");
    }
  }
  @Override
  public boolean equals(Object obj) {
    if (this == obj) return true;
    if (obj == null) return false;

    if (obj instanceof PutRecordRequest == false) return false;
    PutRecordRequest other = (PutRecordRequest) obj;
    if (other.getStreamName() == null ^ this.getStreamName() == null) return false;
    if (other.getStreamName() != null
        && other.getStreamName().equals(this.getStreamName()) == false) return false;
    if (other.getData() == null ^ this.getData() == null) return false;
    if (other.getData() != null && other.getData().equals(this.getData()) == false) return false;
    if (other.getPartitionKey() == null ^ this.getPartitionKey() == null) return false;
    if (other.getPartitionKey() != null
        && other.getPartitionKey().equals(this.getPartitionKey()) == false) return false;
    if (other.getExplicitHashKey() == null ^ this.getExplicitHashKey() == null) return false;
    if (other.getExplicitHashKey() != null
        && other.getExplicitHashKey().equals(this.getExplicitHashKey()) == false) return false;
    if (other.getSequenceNumberForOrdering() == null ^ this.getSequenceNumberForOrdering() == null)
      return false;
    if (other.getSequenceNumberForOrdering() != null
        && other.getSequenceNumberForOrdering().equals(this.getSequenceNumberForOrdering())
            == false) return false;
    return true;
  }