private String format(Event e) {
Date d = new Date(e.getTimestamp());
String data =
String.format(
"%s %s %s: %s\n",
DateUtils.asISO8601(d),
e.getPriority(),
"log4j",
StringEscapeUtils.escapeJava(new String(e.getBody())));
return data;
}
@Test
public void testAvroNativeJson() throws IOException {
AvroNativeFileOutputFormat format = new AvroNativeFileOutputFormat();
ByteArrayOutputStream sos = new ByteArrayOutputStream();
format.format(sos, e);
format.close();
byte[] bytes = sos.toByteArray();
ReflectData reflectData = ReflectData.get();
Schema schema = reflectData.getSchema(EventImpl.class);
ReflectDatumReader<EventImpl> dr = new ReflectDatumReader<EventImpl>(schema);
ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
DataFileStream<EventImpl> dec = new DataFileStream<EventImpl>(bais, dr);
Event er = dec.next();
assertEquals(e.getHost(), er.getHost());
assertEquals(e.getNanos(), er.getNanos());
assertEquals(e.getPriority(), er.getPriority());
assertTrue(Arrays.equals(e.getBody(), er.getBody()));
}
/** Calculate the crc based on the body of the message and xor it into the checksum. */
public void append(Event e) throws IOException, InterruptedException {
chk.reset();
chk.update(e.getBody());
long curchk = chk.getValue();
checksum ^= curchk; // update but do not send.
e.set(ATTR_ACK_TYPE, CHECKSUM_MSG);
e.set(ATTR_ACK_TAG, tag);
e.set(ATTR_ACK_HASH, ByteBuffer.allocate(8).putLong(curchk).array());
super.append(e);
}
@Override
public void append(Event e) throws IOException, InterruptedException {
String helloWorldBody = "Hello World! -- " + new String(e.getBody());
// make a copy of the event, but with the new body string.
EventImpl e2 =
new EventImpl(
helloWorldBody.getBytes(),
e.getTimestamp(),
e.getPriority(),
e.getNanos(),
e.getHost(),
e.getAttrs());
super.append(e2);
}
private void assertCorrectResponse(int count, Event event, SearchResponse response) {
SearchHits hits = response.getHits();
assertEquals(count, hits.getTotalHits());
SearchHit hit = hits.getAt(0);
Map<String, Object> source = hit.getSource();
assertEquals(event.getHost(), source.get("host"));
assertEquals("1970-01-01T00:00:00.000Z", source.get("timestamp"));
assertEquals(event.getPriority().name(), source.get("priority"));
@SuppressWarnings("unchecked")
Map<String, Object> message = (Map<String, Object>) source.get("message");
assertEquals(new String(event.getBody()), message.get("text"));
@SuppressWarnings("unchecked")
Map<String, Object> fields = (Map<String, Object>) source.get("fields");
assertEquals(new String(event.getAttrs().get("attr1")), fields.get("attr1"));
assertEquals(new String(event.getAttrs().get("attr2")), fields.get("attr2"));
}
public void run() {
EventSink sink = null;
EventSource source = null;
synchronized (DirectDriver.this) {
sink = DirectDriver.this.sink;
source = DirectDriver.this.source;
}
try {
synchronized (stateSignal) {
state = DriverState.OPENING;
stateSignal.notifyAll();
}
source.open();
sink.open();
} catch (Exception e) {
// if open is interrupted or has an exception there was a problem.
LOG.error("Closing down due to exception on open calls");
errorCleanup(PumperThread.this.getName(), e);
return;
}
synchronized (stateSignal) {
lastExn = null;
state = DriverState.ACTIVE;
stateSignal.notifyAll();
}
LOG.debug("Starting driver " + DirectDriver.this);
try {
Event e = null;
while (!stopped) {
try {
e = source.next();
} catch (InterruptedException eIn) {
// If we are interrupted then its time to go down. re-throw the exception.
// Details are logged by the outer catch block
throw eIn;
} catch (Exception eI) {
// If this is a chained or converted Interrupt then throw it back
if (eI.getCause() instanceof InterruptedException) throw eI;
// If there's an exception, try to reopen the source
// if the open or close still raises an exception, then bail out
LOG.warn("Exception in source: " + source.getName(), eI);
LOG.warn("Retrying after Error in source: " + source.getName());
source.close();
source.open();
LOG.info(" Source Retry successful");
e = source.next(); // try to get the next event again
}
if (e == null) {
LOG.warn("{}: Event is null or empty()", source.getName());
if ("NullSource".equals(source.getName())) {
break;
} else {
continue;
}
}
if (e.getBody().length == 0) {
LOG.warn("Event is empty; continue");
continue;
}
nextCount++;
try {
sink.append(e);
} catch (InterruptedException eIn) {
// If we are interrupted then its time to go down. re-throw the exception.
// Details are logged by the outer catch block
throw eIn;
} catch (Exception eI) {
// If this is a chained or converted Interrupt then throw it back
if (eI.getCause() instanceof InterruptedException) throw eI;
// If there's an exception, try to reopen the source
// if the open or close still raises an exception, then bail out
LOG.warn("Exception in sink: " + sink.getName(), eI);
LOG.warn("Retrying after Error in source: " + sink.getName());
sink.close();
sink.open();
LOG.info("Sink Retry successful");
sink.append(e); // try to sink the event again
}
appendCount++;
appendBytes += e.getBody().length;
}
} catch (Exception e1) {
// Catches all exceptions or throwables. This is a separate thread
LOG.error("Closing down due to exception during append calls");
errorCleanup(PumperThread.this.getName(), e1);
return;
}
try {
synchronized (stateSignal) {
state = DriverState.CLOSING;
stateSignal.notifyAll();
}
source.close();
sink.close();
} catch (Exception e) {
LOG.error("Closing down due to exception during close calls");
errorCleanup(PumperThread.this.getName(), e);
return;
}
synchronized (stateSignal) {
LOG.debug("Driver completed: " + DirectDriver.this);
stopped = true;
state = DriverState.IDLE;
stateSignal.notifyAll();
}
}