/**
* Tests that the rolling event sink correctly tags the output filename.
*
* @throws InterruptedException
*/
@Test
public void testEscapedFilenameCloseFlushes() throws IOException, InterruptedException {
Tagger tagger =
new ProcessTagger() {
@Override
public String getTag() {
return "-testtag";
}
@Override
public String newTag() {
return "-testtag";
}
};
final File f = FileUtil.mktempdir();
RollSink snk =
new RollSink(new Context(), "test", new TimeTrigger(tagger, 10000), 250) {
@Override
protected EventSink newSink(Context ctx) throws IOException {
return new EscapedCustomDfsSink("file:///" + f.getPath(), "sub-%{service}%{rolltag}");
}
};
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
File fo = new File(f.getPath() + "/sub-foo-testtag");
assertTrue(fo.exists());
FileUtil.rmr(f);
}
public synchronized void open() throws IOException {
// TODO (jon) be less strict. ?? need to return on and figure out why thisis
// wrong, add
// latches.
// Preconditions.checkState(state == ManagerState.CLOSED,
// "Must be in CLOSED state to open, currently " + state);
// make the dirs if they do not exist
if (!FileUtil.makeDirs(importDir)) {
throw new IOException("Unable to create import dir: " + importDir);
}
if (!FileUtil.makeDirs(writingDir)) {
throw new IOException("Unable to create writing dir: " + writingDir);
}
if (!FileUtil.makeDirs(loggedDir)) {
throw new IOException("Unable to create logged dir: " + loggedDir);
}
if (!FileUtil.makeDirs(sendingDir)) {
throw new IOException("Unable to create sending dir: " + sendingDir);
}
if (!FileUtil.makeDirs(errorDir)) {
throw new IOException("Unable to create error dir: " + sendingDir);
}
state = ManagerState.OPEN;
}
public void testSyslogOutputFormat() throws IOException {
// set the output format.
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
// build a sink that outputs to that format.
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
// check the output to make sure it is what we expected.
File fo = new File(f.getPath() + "/sub-foo");
FileReader fr = new FileReader(fo);
BufferedReader br = new BufferedReader(fr);
String read = br.readLine() + "\n";
assertEquals(expected, read);
}
/**
* This creates an environment where we have configurations set and then serving starts. This
* simulates a zk configstore load and then the serve call being run.
*
* <p>Ideally we'd create a SetupTranslatingZKMasterTestEnv, but there is an issue when trying to
* start/shutdown and start a new master in the same process/jvm.
*/
@Before
public void setCfgAndStartMaster() throws TTransportException, IOException, FlumeSpecException {
// Give ZK a temporary directory, otherwise it's possible we'll reload some
// old configs
tmpdir = FileUtil.mktempdir();
FlumeConfiguration.createTestableConfiguration();
FlumeConfiguration.get().set(FlumeConfiguration.MASTER_STORE, "memory");
buildMaster();
// Instead of loading from a ZK Store, we just see the config in the "deep"
// config manager. Any translations will not occur.
ConfigurationManager loaded = cfgMan;
loaded.setConfig("node1", "flow", "autoCollectorSource", "null");
loaded.setConfig("node2", "flow", "autoCollectorSource", "null");
loaded.setConfig("node3", "flow", "autoCollectorSource", "null");
loaded.setConfig("node4", "flow", "autoCollectorSource", "null");
loaded.setConfig("agent", "flow", "null", "autoBEChain");
// this is the outer configman, should have no translation.
ConfigurationManager cfgman1 = flumeMaster.getSpecMan();
Map<String, FlumeConfigData> cfgs1 = cfgman1.getTranslatedConfigs();
assertEquals(0, cfgs1.size()); // no translations happened
// start the master (which should trigger an update and translation
flumeMaster.serve();
}
@After
public void stopMaster() throws IOException {
if (flumeMaster != null) {
flumeMaster.shutdown();
flumeMaster = null;
}
if (tmpdir != null) {
FileUtil.rmr(tmpdir);
tmpdir = null;
}
}
/**
* Test to write few log lines, compress using gzip, write to disk, read back the compressed file
* and verify the written lines.
*
* @throws IOException
*/
public void testGzipOutputFormat() throws IOException {
// set the output format.
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true");
// build a sink that outputs to that format.
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
// check the output to make sure it is what we expected.
// read the gzip file and verify the contents
GZIPInputStream gzin = new GZIPInputStream(new FileInputStream(f.getPath() + "/sub-foo.gz"));
byte[] buf = new byte[1];
StringBuilder output = new StringBuilder();
while ((gzin.read(buf)) > 0) {
output.append(new String(buf));
}
assertEquals(expected, output.toString());
assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
}
/**
* This function checks the agent logs dir to make sure that the process has the ability to the
* directory if necessary, that the path if it does exist is a directory, and that it can in fact
* create files inside of the directory. If it fails any of these, it throws an exception.
*
* <p>Finally, it checks to see if the path is in /tmp and warns the user that this may not be the
* best idea.
*/
public static void nodeConfigChecksOk() throws IOException {
// TODO (jon) if we add more checks in here, make the different managers
// responsible for throwing an Exception on construction instead.
FlumeConfiguration conf = FlumeConfiguration.get();
String s = conf.getAgentLogsDir();
File f = new File(s);
if (!FileUtil.makeDirs(f)) {
throw new IOException("Path to Log dir cannot be created: '" + s + "'. Check permissions?");
}
if (!f.isDirectory()) {
throw new IOException("Log dir '" + s + "' already exists as a file. Check log dir path.");
}
File f2 = null;
try {
f2 = File.createTempFile("initcheck", ".test", f);
} catch (IOException e) {
throw new IOException("Failure to write in log directory: '" + s + "'. Check permissions?");
}
if (!f2.delete()) {
throw new IOException(
"Unable to delete "
+ f2
+ " from log directory "
+ "(but writing succeeded) - something is strange here");
}
File tmp = new File("/tmp");
File cur = f;
while (cur != null) {
if (cur.equals(tmp)) {
LOG.warn("Log directory is writing inside of /tmp. This data may not survive reboot!");
break;
}
cur = cur.getParentFile();
}
}