/**
* Test metrics
*
* @throws InterruptedException
* @throws IOException
*/
@Test
public void testGetRollMetrics()
throws JSONException, FlumeSpecException, IOException, InterruptedException {
ReportTestUtils.setupSinkFactory();
EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), "roll(100) { one } ");
ReportEvent rpt = ReportUtil.getFlattenedReport(snk);
LOG.info(ReportUtil.toJSONObject(rpt).toString());
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLFAILS));
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLS));
assertEquals("one", rpt.getStringMetric(RollSink.A_ROLLSPEC));
assertNull(rpt.getStringMetric("One.name"));
// need to open to have sub sink show up
snk.open();
ReportEvent all = ReportUtil.getFlattenedReport(snk);
LOG.info(ReportUtil.toJSONObject(all).toString());
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLFAILS));
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLS));
assertEquals("one", rpt.getStringMetric(RollSink.A_ROLLSPEC));
assertEquals("One", all.getStringMetric("One.name"));
snk.close();
}
public void testSyslogOutputFormat() throws IOException {
// set the output format.
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
// build a sink that outputs to that format.
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
// check the output to make sure it is what we expected.
File fo = new File(f.getPath() + "/sub-foo");
FileReader fr = new FileReader(fo);
BufferedReader br = new BufferedReader(fr);
String read = br.readLine() + "\n";
assertEquals(expected, read);
}
/**
* Tests to make sure the report sink receives data.
*
* @throws InterruptedException
*/
@Test
public void testReportSink() throws FlumeSpecException, IOException, InterruptedException {
String spec =
"{benchinject(\"foo\") => {benchreport(\"report\", \"[ console , counter(\\\"test\\\") ]\") => null } }";
EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), spec);
snk.open();
snk.append(new EventImpl(new byte[0]));
snk.append(new EventImpl(new byte[0]));
snk.close();
CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
Assert.assertEquals(1, ctr.getCount());
}
/*
* This tests to see if the seqfile event sink releases file handles.
*
* Default handle limit per process is around 1000 so this should be ample to
* cause problems.
*/
@Test
public void testSeqfileEventSinkHandleExhaust() throws IOException {
BenchmarkHarness.setupLocalWriteDir();
File tmp = BenchmarkHarness.tmpdir;
for (int i = 0; i < 3000; i++) {
File path = new File(tmp, "" + i);
EventSink snk = new SeqfileEventSink(path);
snk.open();
Event e = new EventImpl(("foo " + i).getBytes());
snk.append(e);
snk.close();
}
}
@Test
public void testSimple() throws IOException, InterruptedException {
EventSource src = new NoNlASCIISynthSource(25, 100, 1);
EventSink snk = new ConsoleEventSink();
EventSink snk2 = new BenchmarkReportDecorator<EventSink>("report", snk);
EventSink snk3 = new BenchmarkInjectDecorator<EventSink>(snk2);
DirectDriver connect = new DirectDriver(src, snk3);
src.open();
snk3.open();
connect.start();
connect.join(Long.MAX_VALUE);
src.close();
snk3.close();
snk2.getReport().toText(new OutputStreamWriter(System.err));
}
/**
* Checks for Benchmark Tags. If there are not tags events are passed through. If there are, there
* are three kinds - 'start' which instantiates a benchmark; 'first' which starts a benchmark; and
* 'stop' which ends a benchmark. These are consumed by this decorator.
*/
@Override
public void append(Event e) throws IOException, InterruptedException {
byte[] bench = e.get(BenchmarkInjectDecorator.ATTR_BENCHMARK);
if (bench == null) {
// This is the normal case -- a regular message
getSink().append(e);
return;
}
// All of these messages are silently consumed and not forwarded
byte[] tagbytes = e.get(BenchmarkInjectDecorator.ATTR_BENCHMARK_TAG);
Preconditions.checkNotNull(tagbytes);
String tag = new String(tagbytes, CharEncUtils.RAW);
if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_START)) {
StringWriter out = new StringWriter();
PrintWriter pw = new PrintWriter(out);
Benchmark b = new Benchmark(tag, pw, pw);
b.mark("benchmarkStart");
benchmarks.put(tag, new Pair<Benchmark, StringWriter>(b, out));
} else if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_FIRST)) {
Benchmark b = benchmarks.get(tag).getLeft();
b.mark("benchmarkFirst");
} else if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_STOP)) {
Benchmark b = benchmarks.get(tag).getLeft();
b.mark("benchmarkDone");
b.done();
ReportEvent rpt = getMetrics();
LOG.info(rpt.toText());
reportSink.append(rpt);
} else if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_ERROR)) {
Benchmark b = benchmarks.get(tag).getLeft();
b.mark("benchmarkError");
b.done();
LOG.info(getMetrics().toText());
ReportEvent rpt = getMetrics();
LOG.info(rpt.toText());
reportSink.append(rpt);
} else {
String msg = "Unexpected Benchmark event type: " + tag;
LOG.error(msg);
throw new IllegalArgumentException(msg);
}
}
/**
* This test builds a disk failover and then attempts to roll the output of it. The diskFailover
* is set to retry every 1s (1000ms). We then check to see if the number of elements has gone up
* for at most 3s.
*/
@Test
public void testAgentDFOCollector() throws IOException, FlumeSpecException, InterruptedException {
String agentCollector = "{diskFailover(1000) => roll (100000) { null } }";
Event e = new EventImpl("foo".getBytes());
EventSink agent = FlumeBuilder.buildSink(new Context(), agentCollector);
agent.open();
agent.append(e);
for (int i = 0; i < 30; i++) {
Clock.sleep(100);
ReportEvent r = mem.getReport();
LOG.info(r);
if (r.getLongMetric("number of events") > 0) {
return;
}
}
fail("Test timed out, event didn't make it");
}
@Override
public Map<String, Reportable> getSubMetrics() {
Map<String, Reportable> map = new HashMap<String, Reportable>();
if (src != null) {
map.put("source." + src.getName(), src);
}
if (snk != null) {
map.put("sink." + snk.getName(), snk);
}
return map;
}
@Test
public void testSizeTriggerFunctional()
throws FlumeSpecException, IOException, InterruptedException {
// a ridiculous amount of time trigger time to forces size trigger
EventSink snk =
FlumeBuilder.buildSink(
LogicalNodeContext.testingContext(), "roll(1000000, trigger=size(10)) { console }");
snk.open();
// Events from this loop:
// 0, 1, trigger, 2, 3, trigger, 4, 5, trigger, 6, 7, trigger, 8 ,9,
for (int i = 0; i < 10; i++) {
snk.append(new EventImpl("6chars".getBytes()));
}
snk.close();
ReportEvent rpt = snk.getMetrics();
// See above for why there are 4 triggers:
assertEquals(4, (long) rpt.getLongMetric(RollSink.A_ROLLS));
}
@Deprecated
public synchronized void getReports(Map<String, ReportEvent> reports) {
String phyName = FlumeNode.getInstance().getPhysicalNodeName();
String rprefix = phyName + "." + getName() + ".";
if (snk != null) {
snk.getReports(rprefix, reports);
}
if (src != null) {
src.getReports(rprefix, reports);
}
}
/**
* Test to write few log lines, compress using gzip, write to disk, read back the compressed file
* and verify the written lines.
*
* @throws IOException
*/
public void testGzipOutputFormat() throws IOException {
// set the output format.
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true");
// build a sink that outputs to that format.
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
// check the output to make sure it is what we expected.
// read the gzip file and verify the contents
GZIPInputStream gzin = new GZIPInputStream(new FileInputStream(f.getPath() + "/sub-foo.gz"));
byte[] buf = new byte[1];
StringBuilder output = new StringBuilder();
while ((gzin.read(buf)) > 0) {
output.append(new String(buf));
}
assertEquals(expected, output.toString());
assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
}
@Deprecated
public synchronized ReportEvent getReport() {
ReportEvent rpt = new ReportEvent(nodeName);
rpt.setStringMetric("nodename", nodeName);
rpt.setStringMetric("version", new Date(lastGoodCfg.timestamp).toString());
rpt.setStringMetric("state", state.state.toString());
rpt.setStringMetric("hostname", state.host);
rpt.setStringMetric(
"sourceConfig", (lastGoodCfg.sourceConfig == null) ? "" : lastGoodCfg.sourceConfig);
rpt.setStringMetric(
"sinkConfig", (lastGoodCfg.sinkConfig == null) ? "" : lastGoodCfg.sinkConfig);
rpt.setStringMetric("message", nodeMsg);
rpt.setLongMetric(A_RECONFIGURES, reconfigures.get());
rpt.setStringMetric("physicalnode", state.physicalNode);
if (snk != null) {
rpt.hierarchicalMerge("sink", snk.getReport());
}
return rpt;
}
@Override
public void close() throws IOException, InterruptedException {
super.close();
reportSink.close();
LOG.info(new String(getMetrics().getBody()));
}
@Override
public void open() throws IOException, InterruptedException {
super.open();
reportSink.open();
}
@Override
public String toString() {
return source.getClass().getSimpleName() + " | " + sink.getName();
}
public void run() {
EventSink sink = null;
EventSource source = null;
synchronized (DirectDriver.this) {
sink = DirectDriver.this.sink;
source = DirectDriver.this.source;
}
try {
synchronized (stateSignal) {
state = DriverState.OPENING;
stateSignal.notifyAll();
}
source.open();
sink.open();
} catch (Exception e) {
// if open is interrupted or has an exception there was a problem.
LOG.error("Closing down due to exception on open calls");
errorCleanup(PumperThread.this.getName(), e);
return;
}
synchronized (stateSignal) {
lastExn = null;
state = DriverState.ACTIVE;
stateSignal.notifyAll();
}
LOG.debug("Starting driver " + DirectDriver.this);
try {
Event e = null;
while (!stopped) {
try {
e = source.next();
} catch (InterruptedException eIn) {
// If we are interrupted then its time to go down. re-throw the exception.
// Details are logged by the outer catch block
throw eIn;
} catch (Exception eI) {
// If this is a chained or converted Interrupt then throw it back
if (eI.getCause() instanceof InterruptedException) throw eI;
// If there's an exception, try to reopen the source
// if the open or close still raises an exception, then bail out
LOG.warn("Exception in source: " + source.getName(), eI);
LOG.warn("Retrying after Error in source: " + source.getName());
source.close();
source.open();
LOG.info(" Source Retry successful");
e = source.next(); // try to get the next event again
}
if (e == null) {
LOG.warn("{}: Event is null or empty()", source.getName());
if ("NullSource".equals(source.getName())) {
break;
} else {
continue;
}
}
if (e.getBody().length == 0) {
LOG.warn("Event is empty; continue");
continue;
}
nextCount++;
try {
sink.append(e);
} catch (InterruptedException eIn) {
// If we are interrupted then its time to go down. re-throw the exception.
// Details are logged by the outer catch block
throw eIn;
} catch (Exception eI) {
// If this is a chained or converted Interrupt then throw it back
if (eI.getCause() instanceof InterruptedException) throw eI;
// If there's an exception, try to reopen the source
// if the open or close still raises an exception, then bail out
LOG.warn("Exception in sink: " + sink.getName(), eI);
LOG.warn("Retrying after Error in source: " + sink.getName());
sink.close();
sink.open();
LOG.info("Sink Retry successful");
sink.append(e); // try to sink the event again
}
appendCount++;
appendBytes += e.getBody().length;
}
} catch (Exception e1) {
// Catches all exceptions or throwables. This is a separate thread
LOG.error("Closing down due to exception during append calls");
errorCleanup(PumperThread.this.getName(), e1);
return;
}
try {
synchronized (stateSignal) {
state = DriverState.CLOSING;
stateSignal.notifyAll();
}
source.close();
sink.close();
} catch (Exception e) {
LOG.error("Closing down due to exception during close calls");
errorCleanup(PumperThread.this.getName(), e);
return;
}
synchronized (stateSignal) {
LOG.debug("Driver completed: " + DirectDriver.this);
stopped = true;
state = DriverState.IDLE;
stateSignal.notifyAll();
}
}