public void testSyslogOutputFormat() throws IOException { // set the output format. FlumeConfiguration conf = FlumeConfiguration.get(); conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog"); // build a sink that outputs to that format. File f = FileUtil.mktempdir(); SinkBuilder builder = EscapedCustomDfsSink.builder(); EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}"); Event e = new EventImpl("this is a test message".getBytes()); Attributes.setString(e, "service", "foo"); snk.open(); snk.append(e); snk.close(); ByteArrayOutputStream exWriter = new ByteArrayOutputStream(); SyslogEntryFormat fmt = new SyslogEntryFormat(); fmt.format(exWriter, e); exWriter.close(); String expected = new String(exWriter.toByteArray()); // check the output to make sure it is what we expected. File fo = new File(f.getPath() + "/sub-foo"); FileReader fr = new FileReader(fo); BufferedReader br = new BufferedReader(fr); String read = br.readLine() + "\n"; assertEquals(expected, read); }
/** * Tests that the rolling event sink correctly tags the output filename. * * @throws InterruptedException */ @Test public void testEscapedFilenameCloseFlushes() throws IOException, InterruptedException { Tagger tagger = new ProcessTagger() { @Override public String getTag() { return "-testtag"; } @Override public String newTag() { return "-testtag"; } }; final File f = FileUtil.mktempdir(); RollSink snk = new RollSink(new Context(), "test", new TimeTrigger(tagger, 10000), 250) { @Override protected EventSink newSink(Context ctx) throws IOException { return new EscapedCustomDfsSink("file:///" + f.getPath(), "sub-%{service}%{rolltag}"); } }; Event e = new EventImpl("this is a test message".getBytes()); Attributes.setString(e, "service", "foo"); snk.open(); snk.append(e); snk.close(); File fo = new File(f.getPath() + "/sub-foo-testtag"); assertTrue(fo.exists()); FileUtil.rmr(f); }
/** * This creates an environment where we have configurations set and then serving starts. This * simulates a zk configstore load and then the serve call being run. * * <p>Ideally we'd create a SetupTranslatingZKMasterTestEnv, but there is an issue when trying to * start/shutdown and start a new master in the same process/jvm. */ @Before public void setCfgAndStartMaster() throws TTransportException, IOException, FlumeSpecException { // Give ZK a temporary directory, otherwise it's possible we'll reload some // old configs tmpdir = FileUtil.mktempdir(); FlumeConfiguration.createTestableConfiguration(); FlumeConfiguration.get().set(FlumeConfiguration.MASTER_STORE, "memory"); buildMaster(); // Instead of loading from a ZK Store, we just see the config in the "deep" // config manager. Any translations will not occur. ConfigurationManager loaded = cfgMan; loaded.setConfig("node1", "flow", "autoCollectorSource", "null"); loaded.setConfig("node2", "flow", "autoCollectorSource", "null"); loaded.setConfig("node3", "flow", "autoCollectorSource", "null"); loaded.setConfig("node4", "flow", "autoCollectorSource", "null"); loaded.setConfig("agent", "flow", "null", "autoBEChain"); // this is the outer configman, should have no translation. ConfigurationManager cfgman1 = flumeMaster.getSpecMan(); Map<String, FlumeConfigData> cfgs1 = cfgman1.getTranslatedConfigs(); assertEquals(0, cfgs1.size()); // no translations happened // start the master (which should trigger an update and translation flumeMaster.serve(); }
/** * Test to write few log lines, compress using gzip, write to disk, read back the compressed file * and verify the written lines. * * @throws IOException */ public void testGzipOutputFormat() throws IOException { // set the output format. FlumeConfiguration conf = FlumeConfiguration.get(); conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog"); conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true"); // build a sink that outputs to that format. File f = FileUtil.mktempdir(); SinkBuilder builder = EscapedCustomDfsSink.builder(); EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}"); Event e = new EventImpl("this is a test message".getBytes()); Attributes.setString(e, "service", "foo"); snk.open(); snk.append(e); snk.close(); ByteArrayOutputStream exWriter = new ByteArrayOutputStream(); SyslogEntryFormat fmt = new SyslogEntryFormat(); fmt.format(exWriter, e); exWriter.close(); String expected = new String(exWriter.toByteArray()); // check the output to make sure it is what we expected. // read the gzip file and verify the contents GZIPInputStream gzin = new GZIPInputStream(new FileInputStream(f.getPath() + "/sub-foo.gz")); byte[] buf = new byte[1]; StringBuilder output = new StringBuilder(); while ((gzin.read(buf)) > 0) { output.append(new String(buf)); } assertEquals(expected, output.toString()); assertTrue("temp folder successfully deleted", FileUtil.rmr(f)); }