public void testSyslogOutputFormat() throws IOException {
// set the output format.
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
// build a sink that outputs to that format.
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
// check the output to make sure it is what we expected.
File fo = new File(f.getPath() + "/sub-foo");
FileReader fr = new FileReader(fo);
BufferedReader br = new BufferedReader(fr);
String read = br.readLine() + "\n";
assertEquals(expected, read);
}
/**
* Tests that the rolling event sink correctly tags the output filename.
*
* @throws InterruptedException
*/
@Test
public void testEscapedFilenameCloseFlushes() throws IOException, InterruptedException {
Tagger tagger =
new ProcessTagger() {
@Override
public String getTag() {
return "-testtag";
}
@Override
public String newTag() {
return "-testtag";
}
};
final File f = FileUtil.mktempdir();
RollSink snk =
new RollSink(new Context(), "test", new TimeTrigger(tagger, 10000), 250) {
@Override
protected EventSink newSink(Context ctx) throws IOException {
return new EscapedCustomDfsSink("file:///" + f.getPath(), "sub-%{service}%{rolltag}");
}
};
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
File fo = new File(f.getPath() + "/sub-foo-testtag");
assertTrue(fo.exists());
FileUtil.rmr(f);
}
@Override
public ReportEvent getMetrics() {
ReportEvent rpt = new ReportEvent(getName());
StringWriter sw = new StringWriter();
PrintWriter out = new PrintWriter(sw);
StringWriter csvsw = new StringWriter();
PrintWriter csvpw = new PrintWriter(csvsw);
for (Pair<Benchmark, StringWriter> e : benchmarks.values()) {
out.print(e.getRight().getBuffer().toString());
e.getLeft().printCsvLog(csvpw);
}
out.close();
csvpw.close();
Attributes.setString(rpt, A_BENCHMARK_RPT, sw.toString());
Attributes.setString(rpt, A_BENCHMARK_CSV, csvsw.toString());
return rpt;
}
@Override
public synchronized ReportEvent getReport() {
ReportEvent rpt = new ReportEvent(getName());
Attributes.setLong(rpt, A_RETRANSMIT_TIMEOUT, retransmitTime);
StringBuilder pendingAcks = new StringBuilder();
for (Map.Entry<String, Long> e : pending.entrySet()) {
pendingAcks.append(e.getKey());
pendingAcks.append(":");
pendingAcks.append(new Date(e.getValue()).toString());
pendingAcks.append(", ");
}
Attributes.setString(rpt, A_PENDING_ACK_INFO, pendingAcks.toString());
return rpt;
}
/**
* Test to write few log lines, compress using gzip, write to disk, read back the compressed file
* and verify the written lines.
*
* @throws IOException
*/
public void testGzipOutputFormat() throws IOException {
// set the output format.
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true");
// build a sink that outputs to that format.
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
// check the output to make sure it is what we expected.
// read the gzip file and verify the contents
GZIPInputStream gzin = new GZIPInputStream(new FileInputStream(f.getPath() + "/sub-foo.gz"));
byte[] buf = new byte[1];
StringBuilder output = new StringBuilder();
while ((gzin.read(buf)) > 0) {
output.append(new String(buf));
}
assertEquals(expected, output.toString());
assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
}