@Test
public void testTriggerKWArg() throws FlumeSpecException {
FlumeBuilder.buildSink(
LogicalNodeContext.testingContext(), "roll(1000, trigger=time) { null }");
FlumeBuilder.buildSink(
LogicalNodeContext.testingContext(), "roll(1000, trigger=size(1000)) { null }");
}
/**
* Test metrics
*
* @throws InterruptedException
* @throws IOException
*/
@Test
public void testGetRollMetrics()
throws JSONException, FlumeSpecException, IOException, InterruptedException {
ReportTestUtils.setupSinkFactory();
EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), "roll(100) { one } ");
ReportEvent rpt = ReportUtil.getFlattenedReport(snk);
LOG.info(ReportUtil.toJSONObject(rpt).toString());
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLFAILS));
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLS));
assertEquals("one", rpt.getStringMetric(RollSink.A_ROLLSPEC));
assertNull(rpt.getStringMetric("One.name"));
// need to open to have sub sink show up
snk.open();
ReportEvent all = ReportUtil.getFlattenedReport(snk);
LOG.info(ReportUtil.toJSONObject(all).toString());
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLFAILS));
assertNotNull(rpt.getLongMetric(RollSink.A_ROLLS));
assertEquals("one", rpt.getStringMetric(RollSink.A_ROLLSPEC));
assertEquals("One", all.getStringMetric("One.name"));
snk.close();
}
/**
* Tests to make sure the report sink receives data.
*
* @throws InterruptedException
*/
@Test
public void testReportSink() throws FlumeSpecException, IOException, InterruptedException {
String spec =
"{benchinject(\"foo\") => {benchreport(\"report\", \"[ console , counter(\\\"test\\\") ]\") => null } }";
EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), spec);
snk.open();
snk.append(new EventImpl(new byte[0]));
snk.append(new EventImpl(new byte[0]));
snk.close();
CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
Assert.assertEquals(1, ctr.getCount());
}
/**
* This test builds a disk failover and then attempts to roll the output of it. The diskFailover
* is set to retry every 1s (1000ms). We then check to see if the number of elements has gone up
* for at most 3s.
*/
@Test
public void testAgentDFOCollector() throws IOException, FlumeSpecException, InterruptedException {
String agentCollector = "{diskFailover(1000) => roll (100000) { null } }";
Event e = new EventImpl("foo".getBytes());
EventSink agent = FlumeBuilder.buildSink(new Context(), agentCollector);
agent.open();
agent.append(e);
for (int i = 0; i < 30; i++) {
Clock.sleep(100);
ReportEvent r = mem.getReport();
LOG.info(r);
if (r.getLongMetric("number of events") > 0) {
return;
}
}
fail("Test timed out, event didn't make it");
}
@Test
public void testSizeTriggerFunctional()
throws FlumeSpecException, IOException, InterruptedException {
// a ridiculous amount of time trigger time to forces size trigger
EventSink snk =
FlumeBuilder.buildSink(
LogicalNodeContext.testingContext(), "roll(1000000, trigger=size(10)) { console }");
snk.open();
// Events from this loop:
// 0, 1, trigger, 2, 3, trigger, 4, 5, trigger, 6, 7, trigger, 8 ,9,
for (int i = 0; i < 10; i++) {
snk.append(new EventImpl("6chars".getBytes()));
}
snk.close();
ReportEvent rpt = snk.getMetrics();
// See above for why there are 4 triggers:
assertEquals(4, (long) rpt.getLongMetric(RollSink.A_ROLLS));
}
@Test
public void testBuildInjectDeco() throws FlumeSpecException {
String spec = "{benchinject(\"report\") => null}";
FlumeBuilder.buildSink(new Context(), spec);
}
@Test
public void testBuildReportSink2() throws FlumeSpecException {
String spec = "{benchreport(\"report\", \"text(\\\"test\\\")\") => null } ";
FlumeBuilder.buildSink(new Context(), spec);
}
@Test
public void testSimpleBuilder() throws FlumeSpecException {
String spec = "{benchinject => null}";
FlumeBuilder.buildSink(new Context(), spec);
}
public void loadConfig(FlumeConfigData cfg)
throws IOException, RuntimeException, FlumeSpecException {
// got a newer configuration
LOG.debug("Attempt to load config " + cfg);
EventSink newSnk = null;
EventSource newSrc = null;
try {
String errMsg = null;
if (cfg.sinkConfig == null || cfg.sinkConfig.length() == 0) {
errMsg = this.getName() + " - empty sink";
}
if (cfg.sourceConfig == null || cfg.sourceConfig.length() == 0) {
errMsg = this.getName() + " - empty source";
}
if (errMsg != null) {
LOG.info(errMsg);
return; // Do nothing.
}
newSnk = FlumeBuilder.buildSink(ctx, cfg.sinkConfig);
newSrc = FlumeBuilder.buildSource(ctx, cfg.sourceConfig);
// TODO (jon) ERROR isn't quite right here -- the connection is in ERROR
// but the previous connection is ok. Need to just add states to the
// connections, and have each node maintain a list of connections.
// This error conditions should not occur. One way we could have this
// error is if the node does not have all plugins that the master has
// installed. The master could then accept a particular source/sink but
// the node would fail when attempting to instantiate it.
if (newSnk == null) {
LOG.error("failed to create sink config: " + cfg.sinkConfig);
state.state = NodeState.ERROR;
return;
}
if (newSrc == null) {
LOG.error("failed to create sink config: " + cfg.sourceConfig);
state.state = NodeState.ERROR;
return;
}
} catch (RuntimeException e) {
LOG.error("Runtime ex: " + new File(".").getAbsolutePath() + " " + cfg, e);
state.state = NodeState.ERROR;
throw e;
} catch (FlumeSpecException e) {
LOG.error("FlumeSpecExn : " + new File(".").getAbsolutePath() + " " + cfg, e);
state.state = NodeState.ERROR;
throw e;
}
try {
loadNodeDriver(newSrc, newSnk);
// We have successfully opened the source and sinks for the config. We can
// mark this as the last good / successful config. It does not mean that
// this configuration will open without errors!
this.lastGoodCfg = cfg;
LOG.info("Node config successfully set to " + cfg);
} catch (InterruptedException e) {
// TODO figure out what to do on interruption
LOG.error("Load Config interrupted", e);
}
}
@Test(expected = FlumeSpecException.class)
public void testBadTriggerKWArgArg() throws FlumeSpecException {
FlumeBuilder.buildSink(
LogicalNodeContext.testingContext(), "roll(1000, trigger=size(\"badarg\")) { null }");
}
@Test(expected = FlumeSpecException.class)
public void testBadMissingTimeTriggerKWArg() throws FlumeSpecException {
FlumeBuilder.buildSink(LogicalNodeContext.testingContext(), "roll(trigger=size(100)) { null }");
}
