@Test
public void testTriggerKWArg() throws FlumeSpecException {
FlumeBuilder.buildSink(
LogicalNodeContext.testingContext(), "roll(1000, trigger=time) { null }");
FlumeBuilder.buildSink(
LogicalNodeContext.testingContext(), "roll(1000, trigger=size(1000)) { null }");
}
/**
* Test metrics
*
* @throws InterruptedException
* @throws IOException
*/
@Test
public void testGetRollMetrics()
throws JSONException, FlumeSpecException, IOException, InterruptedException {
ReportTestUtils.setupSinkFactory();
EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), "roll(100) { one } ");
ReportEvent rpt = ReportUtil.getFlattenedReport(snk);
LOG.info(ReportUtil.toJSONObject(rpt).toString());
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLFAILS));
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLS));
assertEquals("one", rpt.getStringMetric(RollSink.A_ROLLSPEC));
assertNull(rpt.getStringMetric("One.name"));
// need to open to have sub sink show up
snk.open();
ReportEvent all = ReportUtil.getFlattenedReport(snk);
LOG.info(ReportUtil.toJSONObject(all).toString());
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLFAILS));
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLS));
assertEquals("one", rpt.getStringMetric(RollSink.A_ROLLSPEC));
assertEquals("One", all.getStringMetric("One.name"));
snk.close();
}
/**
* Tests to make sure the report sink receives data.
*
* @throws InterruptedException
*/
@Test
public void testReportSink() throws FlumeSpecException, IOException, InterruptedException {
String spec =
"{benchinject(\"foo\") => {benchreport(\"report\", \"[ console , counter(\\\"test\\\") ]\") => null } }";
EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), spec);
snk.open();
snk.append(new EventImpl(new byte[0]));
snk.append(new EventImpl(new byte[0]));
snk.close();
CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
Assert.assertEquals(1, ctr.getCount());
}
@Before
public void replaceEscapedCustomDfsSink() {
// Replace the null with a memory buffer that we have a reference to
SinkFactoryImpl sf = new SinkFactoryImpl();
sf.setSink(
"null",
new SinkBuilder() {
@Override
public EventSink build(Context context, String... argv) {
LOG.info("excapedCustomDfs replaced with MemorySinkSource");
mem.reset();
return mem;
}
});
FlumeBuilder.setSinkFactory(sf);
}
/**
* This test builds a disk failover and then attempts to roll the output of it. The diskFailover
* is set to retry every 1s (1000ms). We then check to see if the number of elements has gone up
* for at most 3s.
*/
@Test
public void testAgentDFOCollector() throws IOException, FlumeSpecException, InterruptedException {
String agentCollector = "{diskFailover(1000) => roll (100000) { null } }";
Event e = new EventImpl("foo".getBytes());
EventSink agent = FlumeBuilder.buildSink(new Context(), agentCollector);
agent.open();
agent.append(e);
for (int i = 0; i < 30; i++) {
Clock.sleep(100);
ReportEvent r = mem.getReport();
LOG.info(r);
if (r.getLongMetric("number of events") > 0) {
return;
}
}
fail("Test timed out, event didn't make it");
}
@Test
public void testSizeTriggerFunctional()
throws FlumeSpecException, IOException, InterruptedException {
// a ridiculous amount of time trigger time to forces size trigger
EventSink snk =
FlumeBuilder.buildSink(
LogicalNodeContext.testingContext(), "roll(1000000, trigger=size(10)) { console }");
snk.open();
// Events from this loop:
// 0, 1, trigger, 2, 3, trigger, 4, 5, trigger, 6, 7, trigger, 8 ,9,
for (int i = 0; i < 10; i++) {
snk.append(new EventImpl("6chars".getBytes()));
}
snk.close();
ReportEvent rpt = snk.getMetrics();
// See above for why there are 4 triggers:
assertEquals(4, (long) rpt.getLongMetric(RollSink.A_ROLLS));
}
public void _jspService(HttpServletRequest request, HttpServletResponse response)
throws java.io.IOException, ServletException {
JspFactory _jspxFactory = null;
PageContext pageContext = null;
HttpSession session = null;
ServletContext application = null;
ServletConfig config = null;
JspWriter out = null;
Object page = this;
JspWriter _jspx_out = null;
PageContext _jspx_page_context = null;
try {
_jspxFactory = JspFactory.getDefaultFactory();
response.setContentType("text/html; charset=UTF-8");
pageContext = _jspxFactory.getPageContext(this, request, response, null, true, 8192, true);
_jspx_page_context = pageContext;
application = pageContext.getServletContext();
config = pageContext.getServletConfig();
session = pageContext.getSession();
out = pageContext.getOut();
_jspx_out = out;
out.write(
"<!--\n Licensed to Cloudera, Inc. under one\n or more contributor license agreements. See the NOTICE file\n distributed with this work for additional information\n regarding copyright ownership. Cloudera, Inc. licenses this file\n to you under the Apache License, Version 2.0 (the\n \"License\"); you may not use this file except in compliance\n with the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n<html><head>\n<!-- Copyright (c) 2010 Cloudera, Inc. All rights reserved. -->\n<!-- Retro web 1.0 flume Agent configuration display -->\n\n<title>Flume Node: ");
out.print(FlumeNode.getInstance().getPhysicalNodeName());
out.write(
" :: Sources and Sinks</title>\n\n<link rel=\"stylesheet\" type=\"text/css\" href=\"/flume.css\" />\n\n</head>\n<body>\n");
org.apache.jasper.runtime.JspRuntimeLibrary.include(
request, response, "menu_agent.jsp", out, false);
out.write("\n\n<h1>Flume Node: ");
out.print(FlumeNode.getInstance().getPhysicalNodeName());
out.write(" :: Extensions - Sink/Source/Decorator</h1>\n\n");
org.apache.jasper.runtime.JspRuntimeLibrary.include(
request, response, "version.jsp", out, false);
out.write("\n<hr>\n\n\n<div id=\"sinks\">\n<h2>Sinks</h2>\n\n<table>\n\n");
// treeset so that the list is sorted
for (String n : new TreeSet<String>(FlumeBuilder.getSinkNames())) {
out.write("\n <tr><td>");
out.print(n);
out.write("</td></tr>\n");
}
out.write("\n\n</table>\n</div>\n\n<div id=\"sources\">\n<h2>Sources</h2>\n<table>\n\n");
// treeset so that the list is sorted
for (String n : new TreeSet<String>(FlumeBuilder.getSourceNames())) {
out.write("\n <tr><td>");
out.print(n);
out.write("</td></tr>\n");
}
out.write("\n\n</table>\n</div>\n\n<div id=\"decos\">\n<h2>Decorators</h2>\n<table>\n\n");
// treeset so that the list is sorted
for (String n : new TreeSet<String>(FlumeBuilder.getDecoratorNames())) {
out.write("\n <tr><td>");
out.print(n);
out.write("</td></tr>\n");
}
out.write("\n\n</table>\n</div>\n</body></html>\n");
} catch (Throwable t) {
if (!(t instanceof SkipPageException)) {
out = _jspx_out;
if (out != null && out.getBufferSize() != 0) out.clearBuffer();
if (_jspx_page_context != null) _jspx_page_context.handlePageException(t);
}
} finally {
if (_jspxFactory != null) _jspxFactory.releasePageContext(_jspx_page_context);
}
}
/**
* Returns a Flume Node with settings from specified command line parameters. (See usage for
* instructions)
*
* @param argv
* @return
* @throws IOException
*/
public static FlumeNode setup(String[] argv) throws IOException {
logVersion(LOG);
logEnvironment(LOG);
// Make sure the Java version is not older than 1.6
if (!CheckJavaVersion.isVersionOk()) {
LOG.error("Exiting because of an old Java version or Java version in bad format");
System.exit(-1);
}
LOG.info("Starting flume agent on: " + NetUtils.localhost());
LOG.info(" Working directory is: " + new File(".").getAbsolutePath());
FlumeConfiguration.hardExitLoadConfig(); // will exit if conf file is bad.
CommandLine cmd = null;
Options options = new Options();
options.addOption("c", true, "Load initial config from cmdline arg");
options.addOption("n", true, "Set node name");
options.addOption("s", false, "Do not start local flume status server on node");
options.addOption("1", false, "Make flume node one shot (if closes or errors, exits)");
options.addOption("m", false, "Have flume hard exit if in likely GC thrash situation");
options.addOption("h", false, "Print help information");
options.addOption("v", false, "Print version information");
try {
CommandLineParser parser = new PosixParser();
cmd = parser.parse(options, argv);
} catch (ParseException e) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
return null;
}
// dump version info only
if (cmd != null && cmd.hasOption("v")) {
return null;
}
// dump help info.
if (cmd != null && cmd.hasOption("h")) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
return null;
}
// Check FlumeConfiguration file for settings that may cause node to fail.
nodeConfigChecksOk();
String nodename = NetUtils.localhost(); // default to local host name.
if (cmd != null && cmd.hasOption("n")) {
// select a different name, allow for multiple processes configured
// differently on same node.
nodename = cmd.getOptionValue("n");
}
boolean startHttp = false;
if (cmd != null && !cmd.hasOption("s")) {
// no -s option, start the local status server
startHttp = true;
}
boolean oneshot = false;
if (cmd != null && cmd.hasOption("1")) {
oneshot = true;
}
FormatFactory.loadOutputFormatPlugins();
// Instantiate the flume node.
FlumeConfiguration conf = FlumeConfiguration.get();
FlumeNode flume = new FlumeNode(nodename, conf, startHttp, oneshot);
flume.start();
// load an initial configuration from command line
if (cmd != null && cmd.hasOption("c")) {
String spec = cmd.getOptionValue("c");
LOG.info("Loading spec from command line: '" + spec + "'");
try {
// node name is the default logical and physical name.
Context ctx = new LogicalNodeContext(nodename, nodename);
Map<String, Pair<String, String>> cfgs = FlumeBuilder.parseConf(ctx, spec);
Pair<String, String> node = cfgs.get(nodename);
FlumeConfigData fcd = new FlumeConfigData(0, node.getLeft(), node.getRight(), 0, 0, null);
flume.nodesMan.spawn(ctx, nodename, fcd);
} catch (Exception e) {
LOG.warn("Caught exception loading node:" + e.getMessage());
LOG.debug("Exception: ", e);
if (oneshot) {
System.exit(0); // exit cleanly
}
}
} else {
try {
// default to null configurations.
Context ctx = new LogicalNodeContext(nodename, nodename);
FlumeConfigData fcd = new FlumeConfigData(0, "null", "null", 0, 0, null);
flume.nodesMan.spawn(ctx, nodename, fcd);
} catch (Exception e) {
LOG.error("Caught exception loading node", e);
}
}
if (cmd != null && cmd.hasOption("m")) {
// setup memory use monitor
LOG.info("Setup hard exit on memory exhaustion");
MemoryMonitor.setupHardExitMemMonitor(FlumeConfiguration.get().getAgentMemoryThreshold());
}
try {
tryKerberosLogin();
} catch (IOException ioe) {
LOG.error("Failed to kerberos login.", ioe);
}
// hangout, waiting for other agent thread to exit.
return flume;
}
@Test
public void testBuildInjectDeco() throws FlumeSpecException {
String spec = "{benchinject(\"report\") => null}";
FlumeBuilder.buildSink(new Context(), spec);
}
@Test
public void testBuildReportSink2() throws FlumeSpecException {
String spec = "{benchreport(\"report\", \"text(\\\"test\\\")\") => null } ";
FlumeBuilder.buildSink(new Context(), spec);
}
@Test
public void testSimpleBuilder() throws FlumeSpecException {
String spec = "{benchinject => null}";
FlumeBuilder.buildSink(new Context(), spec);
}
public void loadConfig(FlumeConfigData cfg)
throws IOException, RuntimeException, FlumeSpecException {
// got a newer configuration
LOG.debug("Attempt to load config " + cfg);
EventSink newSnk = null;
EventSource newSrc = null;
try {
String errMsg = null;
if (cfg.sinkConfig == null || cfg.sinkConfig.length() == 0) {
errMsg = this.getName() + " - empty sink";
}
if (cfg.sourceConfig == null || cfg.sourceConfig.length() == 0) {
errMsg = this.getName() + " - empty source";
}
if (errMsg != null) {
LOG.info(errMsg);
return; // Do nothing.
}
newSnk = FlumeBuilder.buildSink(ctx, cfg.sinkConfig);
newSrc = FlumeBuilder.buildSource(ctx, cfg.sourceConfig);
// TODO (jon) ERROR isn't quite right here -- the connection is in ERROR
// but the previous connection is ok. Need to just add states to the
// connections, and have each node maintain a list of connections.
// This error conditions should not occur. One way we could have this
// error is if the node does not have all plugins that the master has
// installed. The master could then accept a particular source/sink but
// the node would fail when attempting to instantiate it.
if (newSnk == null) {
LOG.error("failed to create sink config: " + cfg.sinkConfig);
state.state = NodeState.ERROR;
return;
}
if (newSrc == null) {
LOG.error("failed to create sink config: " + cfg.sourceConfig);
state.state = NodeState.ERROR;
return;
}
} catch (RuntimeException e) {
LOG.error("Runtime ex: " + new File(".").getAbsolutePath() + " " + cfg, e);
state.state = NodeState.ERROR;
throw e;
} catch (FlumeSpecException e) {
LOG.error("FlumeSpecExn : " + new File(".").getAbsolutePath() + " " + cfg, e);
state.state = NodeState.ERROR;
throw e;
}
try {
loadNodeDriver(newSrc, newSnk);
// We have successfully opened the source and sinks for the config. We can
// mark this as the last good / successful config. It does not mean that
// this configuration will open without errors!
this.lastGoodCfg = cfg;
LOG.info("Node config successfully set to " + cfg);
} catch (InterruptedException e) {
// TODO figure out what to do on interruption
LOG.error("Load Config interrupted", e);
}
}
@Test(expected = FlumeSpecException.class)
public void testBadTriggerKWArgArg() throws FlumeSpecException {
FlumeBuilder.buildSink(
LogicalNodeContext.testingContext(), "roll(1000, trigger=size(\"badarg\")) { null }");
}
@Test(expected = FlumeSpecException.class)
public void testBadMissingTimeTriggerKWArg() throws FlumeSpecException {
FlumeBuilder.buildSink(LogicalNodeContext.testingContext(), "roll(trigger=size(100)) { null }");
}
/**
* This test has an append that is blocked because a sink inside of a roll always throws an
* exception. (ex: a down network connection). This test attempts to guarantee that the rotate
* succeeds in a reasoanble amoutn of time.
*
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testRollAppendBlockedDeadlock() throws IOException, InterruptedException {
final EventSink mock = mock(EventSink.Base.class);
doNothing().doThrow(new IOException()).when(mock).close();
final Event e1 = new EventImpl("foo1".getBytes());
final Event e2 = new EventImpl("foo2".getBytes());
doThrow(new IOException()).when(mock).append(e1);
doThrow(new IOException()).when(mock).append(e2);
SinkFactoryImpl sfi = new SinkFactoryImpl();
sfi.setSink(
"rollLock",
new SinkBuilder() {
@Override
public EventSink build(Context context, String... argv) {
return mock;
}
});
FlumeBuilder.setSinkFactory(sfi);
final RollSink roll =
new RollSink(
LogicalNodeContext.testingContext(),
"insistentOpen stubbornAppend insistentAppend rollLock",
1000000,
1000000);
final CountDownLatch latch = new CountDownLatch(1);
// excessively long roll and check times which allow test to force checks.
Thread t =
new Thread("blocked append thread") {
public void run() {
try {
roll.open();
roll.append(e1); // append blocks.
} catch (InterruptedException e) {
latch.countDown();
LOG.error("Exited with expected Exception");
return;
} catch (IOException e) {
LOG.info("Got the unexpected IOException exit", e);
e.printStackTrace();
}
LOG.info("Got the unexpected clean exit");
}
};
t.start();
// have another thread timeout.
final CountDownLatch latch2 = new CountDownLatch(1);
Thread t2 =
new Thread("roll rotate thread") {
public void run() {
try {
Clock.sleep(1000); // this is imperfect, but wait for append to block.
roll.rotate();
roll.close();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
latch2.countDown();
}
}
};
t2.start();
boolean success = latch.await(5, TimeUnit.SECONDS);
assertTrue(success);
success = latch2.await(5, TimeUnit.SECONDS);
assertTrue(success);
}