@Override
public void receiveSpan(Span span) {
if (running.get()) {
try {
this.queue.add(span);
} catch (IllegalStateException e) {
// todo: supress repeating error logs.
LOG.error(
"Error trying to append span ("
+ span.getDescription()
+ ") to the queue. Blocking Queue was full.");
}
}
}
/** This runnable sends a HTrace span to the HBase. */
@Override
public void run() {
SpanProtos.Span.Builder sbuilder = SpanProtos.Span.newBuilder();
SpanProtos.TimelineAnnotation.Builder tlbuilder = SpanProtos.TimelineAnnotation.newBuilder();
List<Span> dequeuedSpans = new ArrayList<Span>(maxSpanBatchSize);
long errorCount = 0;
while (running.get() || queue.size() > 0) {
Span firstSpan = null;
try {
// Block for up to a second. to try and get a span.
// We only block for a little bit in order to notice
// if the running value has changed
firstSpan = queue.poll(1, TimeUnit.SECONDS);
// If the poll was successful then it's possible that there
// will be other spans to get. Try and get them.
if (firstSpan != null) {
// Add the first one that we got
dequeuedSpans.add(firstSpan);
// Try and get up to 100 queues
queue.drainTo(dequeuedSpans, maxSpanBatchSize - 1);
}
} catch (InterruptedException ie) {
// Ignored.
}
startClient();
if (dequeuedSpans.isEmpty()) {
try {
this.htable.flushCommits();
} catch (IOException e) {
LOG.error("failed to flush writes to HBase.");
closeClient();
}
continue;
}
try {
for (Span span : dequeuedSpans) {
sbuilder
.clear()
.setTraceId(span.getTraceId())
.setParentId(span.getParentId())
.setStart(span.getStartTimeMillis())
.setStop(span.getStopTimeMillis())
.setSpanId(span.getSpanId())
.setProcessId(span.getProcessId())
.setDescription(span.getDescription());
for (TimelineAnnotation ta : span.getTimelineAnnotations()) {
sbuilder.addTimeline(
tlbuilder.clear().setTime(ta.getTime()).setMessage(ta.getMessage()).build());
}
Put put = new Put(Bytes.toBytes(span.getTraceId()));
put.add(HBaseSpanReceiver.this.cf, sbuilder.build().toByteArray(), null);
this.htable.put(put);
}
// clear the list for the next time through.
dequeuedSpans.clear();
// reset the error counter.
errorCount = 0;
} catch (Exception e) {
errorCount += 1;
// If there have been ten errors in a row start dropping things.
if (errorCount < MAX_ERRORS) {
try {
queue.addAll(dequeuedSpans);
} catch (IllegalStateException ex) {
LOG.error(
"Drop " + dequeuedSpans.size() + " span(s) because writing to HBase failed.");
}
}
closeClient();
try {
// Since there was an error sleep just a little bit to try and allow the
// HBase some time to recover.
Thread.sleep(500);
} catch (InterruptedException e1) {
// Ignored
}
}
}
closeClient();
}