@Override
public void run() {
long offset = 0;
while (isAlive) {
// create a fetch request for topic “topic1”, partition 0, current offset, and fetch size of
// 1MB
FetchRequest fetchRequest =
new FetchRequestBuilder()
.clientId("default_client")
.addFetch("topic1", 1, offset, 1000000)
.build();
// FetchRequest fetchRequest = new FetchRequest("topic1", 0, offset, 1000000);
// get the message set from the consumer and print them out
ByteBufferMessageSet messages = consumer.fetch(fetchRequest).messageSet("topic1", 1);
Iterator<MessageAndOffset> itr = messages.iterator();
while (itr.hasNext() && isAlive) {
MessageAndOffset msg = itr.next();
// advance the offset after consuming each message
offset = msg.offset();
logger.debug(
"consumed: {} offset: {}",
byteBufferToString(msg.message().payload()).toString(),
offset);
receiveCount++;
}
}
}
protected boolean get(KafkaETLKey key, BytesWritable value) throws IOException {
if (_messageIt != null && _messageIt.hasNext()) {
MessageAndOffset messageAndOffset = _messageIt.next();
ByteBuffer buf = messageAndOffset.message().buffer();
int origSize = buf.remaining();
byte[] bytes = new byte[origSize];
buf.get(bytes, buf.position(), origSize);
value.set(bytes, 0, origSize);
key.set(_index, _offset, messageAndOffset.message().checksum());
_offset = messageAndOffset.nextOffset(); // increase offset
_count++; // increase count
return true;
} else return false;
}
/** consume data from kafka to list */
private void fill() {
SimpleConsumer consumer = _partitions.getConsumer(_partition);
int hostPartition = _partitions.getHostPartition(_partition);
ByteBufferMessageSet msgs =
consumer.fetch(
new FetchRequest(
_spoutConfig.topic,
hostPartition,
_emittedToOffset,
_spoutConfig.fetchSizeBytes));
// LOG.info("partition fill msgs " + msgs);
for (MessageAndOffset msg : msgs) {
_pending.add(_emittedToOffset);
_waitingToEmit.add(new MessageAndRealOffset(msg.message(), _emittedToOffset));
_emittedToOffset = msg.offset();
// LOG.info("parition fill msg " + msg + " offset" + _emittedToOffset);
}
}
// @Ignore
@Test
public void test() throws EventDeliveryException, UnsupportedEncodingException {
Transaction tx = channel.getTransaction();
tx.begin();
ObjectNode jsonBody = new ObjectNode(JsonNodeFactory.instance);
jsonBody.put("myString", "foo");
jsonBody.put("myInt32", 32);
Map<String, String> headers = new HashMap<String, String>();
headers.put("myString", "bar");
headers.put("myInt64", "64");
headers.put("myBoolean", "true");
headers.put("myDouble", "1.0");
headers.put("myNull", "foobar");
Event event = EventBuilder.withBody(jsonBody.toString().getBytes(Charsets.UTF_8), headers);
channel.put(event);
tx.commit();
tx.close();
kafkaSink.process();
kafka.api.FetchRequest req =
new FetchRequestBuilder().clientId(CLIENT_ID).addFetch("test", 0, 0L, 100).build();
FetchResponse fetchResponse = simpleConsumer.fetch(req);
ByteBufferMessageSet messageSet = fetchResponse.messageSet("test", 0);
// Assert.assertTrue(messageSet.sizeInBytes() > 0);
for (MessageAndOffset messageAndOffset : messageSet) {
ByteBuffer payload = messageAndOffset.message().payload();
byte[] bytes = new byte[payload.limit()];
payload.get(bytes);
String message = new String(bytes, "UTF-8");
Assert.assertNotNull(message);
Assert.assertEquals(message, "{\"myString\":\"foo\",\"myInt32\":32}");
}
}
@Override
protected GenericRecord decodeRecord(MessageAndOffset messageAndOffset) throws IOException {
byte[] payload = getBytes(messageAndOffset.message().payload());
Schema recordSchema = getRecordSchema(payload);
Decoder decoder = getDecoder(payload);
this.reader.get().setSchema(recordSchema);
try {
GenericRecord record = this.reader.get().read(null, decoder);
record = AvroUtils.convertRecordSchema(record, this.schema.get());
return record;
} catch (IOException e) {
log.error(
String.format(
"Error during decoding record for partition %s: ", this.getCurrentPartition()));
throw e;
}
}
public void run(
long a_maxReads, String a_topic, int a_partition, List<String> a_seedBrokers, int a_port)
throws Exception {
// find the meta data about the topic and partition we are interested in
//
PartitionMetadata metadata = findLeader(a_seedBrokers, a_port, a_topic, a_partition);
if (metadata == null) {
System.out.println("Can't find metadata for Topic and Partition. Exiting");
return;
}
if (metadata.leader() == null) {
System.out.println("Can't find Leader for Topic and Partition. Exiting");
return;
}
String leadBroker = metadata.leader().host();
String clientName = "Client_" + a_topic + "_" + a_partition;
SimpleConsumer consumer = new SimpleConsumer(leadBroker, a_port, 100000, 64 * 1024, clientName);
long readOffset =
getLastOffset(
consumer, a_topic, a_partition, kafka.api.OffsetRequest.EarliestTime(), clientName);
int numErrors = 0;
while (a_maxReads > 0) {
if (consumer == null) {
consumer = new SimpleConsumer(leadBroker, a_port, 100000, 64 * 1024, clientName);
}
FetchRequest req =
new FetchRequestBuilder()
.clientId(clientName)
.addFetch(
a_topic,
a_partition,
readOffset,
100000) // Note: this fetchSize of 100000 might need to
// be increased if large batches are written to
// Kafka
.build();
FetchResponse fetchResponse = consumer.fetch(req);
if (fetchResponse.hasError()) {
numErrors++;
// Something went wrong!
short code = fetchResponse.errorCode(a_topic, a_partition);
System.out.println(
"Error fetching data from the Broker:" + leadBroker + " Reason: " + code);
if (numErrors > 5) {
break;
}
if (code == ErrorMapping.OffsetOutOfRangeCode()) {
// We asked for an invalid offset. For simple case ask for the last element to reset
readOffset =
getLastOffset(
consumer, a_topic, a_partition, kafka.api.OffsetRequest.LatestTime(), clientName);
continue;
}
consumer.close();
consumer = null;
leadBroker = findNewLeader(leadBroker, a_topic, a_partition, a_port);
continue;
}
numErrors = 0;
long numRead = 0;
for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(a_topic, a_partition)) {
long currentOffset = messageAndOffset.offset();
if (currentOffset < readOffset) {
System.out.println("Found an old offset: " + currentOffset + " Expecting: " + readOffset);
continue;
}
readOffset = messageAndOffset.nextOffset();
ByteBuffer payload = messageAndOffset.message().payload();
byte[] bytes = new byte[payload.limit()];
payload.get(bytes);
System.out.println(
String.valueOf(messageAndOffset.offset()) + ": " + new String(bytes, "UTF-8"));
numRead++;
a_maxReads--;
}
if (numRead == 0) {
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
}
}
}
if (consumer != null) {
consumer.close();
}
}