public void doTestLogicalNodesConcurrentDFOMans(final int threads, final int events, int timeout)
throws IOException, InterruptedException, FlumeSpecException {
FlumeTestHarness.setupLocalWriteDir();
FlumeMaster master = new FlumeMaster();
FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);
final Reportable[] dfos = new Reportable[threads];
for (int i = 0; i < threads; i++) {
String name = "test." + i;
String report = "report." + i;
int count = events + i;
String src = "asciisynth(" + count + ",100)";
String snk = "{ diskFailover => counter(\"" + report + "\") } ";
node.getLogicalNodeManager().testingSpawn(name, src, snk);
dfos[i] = node.getLogicalNodeManager().get(name);
}
// TODO (jon) using sleep is cheating to give all threads a chance to start.
// Test seems flakey without this due to a race condition.
Thread.sleep(500);
// wait for all to be done.
waitForEmptyDFOs(node, timeout);
// check to make sure everyone got the right number of events
boolean success = true;
for (int i = 0; i < threads; i++) {
LOG.info(dfos[i].getMetrics());
}
for (int i = 0; i < threads; i++) {
CounterSink cnt = (CounterSink) ReportManager.get().getReportable("report." + i);
LOG.info(i + " expected " + (events + i) + " and got " + cnt.getCount());
success &= ((events + i) == cnt.getCount());
assertEquals(events + i, cnt.getCount());
}
assertTrue("Counts did not line up", success);
FlumeTestHarness.cleanupLocalWriteDir();
}
@Test
public void testSimpleIntervalSamplerSink() throws IOException {
System.out.println("Simple interval sampler sink");
CounterSink count = new CounterSink("count");
IntervalSampler<CounterSink> sink = new IntervalSampler<CounterSink>(count, 10);
sink.open();
for (int i = 0; i < 30; i++) {
sink.append(new EventImpl(("test " + i).getBytes()));
}
Assert.assertEquals(3, count.getCount());
sink.close();
// do the boundary condition.
CounterSink count2 = new CounterSink("count");
IntervalSampler<CounterSink> sink2 = new IntervalSampler<CounterSink>(count2, 10);
sink2.open();
for (int i = 0; i < 31; i++) {
sink2.append(new EventImpl(("test " + i).getBytes()));
}
Assert.assertEquals(4, count2.getCount());
sink2.close();
}
/**
* Tests to make sure the report sink receives data.
*
* @throws InterruptedException
*/
@Test
public void testReportSink() throws FlumeSpecException, IOException, InterruptedException {
String spec =
"{benchinject(\"foo\") => {benchreport(\"report\", \"[ console , counter(\\\"test\\\") ]\") => null } }";
EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), spec);
snk.open();
snk.append(new EventImpl(new byte[0]));
snk.append(new EventImpl(new byte[0]));
snk.close();
CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
Assert.assertEquals(1, ctr.getCount());
}
@Test
public void testAutoRoll() throws IOException, InterruptedException {
RollSink snk = new RollSink(new ReportTestingContext(), "counter(\"foo\")", 2000, 10000); // two
// second sleeper, but check period is really long
Event e = new EventImpl("this is a test message".getBytes());
snk.open();
snk.append(e);
CounterSink cnt = (CounterSink) ReportManager.get().getReportable("foo");
Clock.sleep(3000); // sleep 3s
// the roller automatically flushed!
assertEquals(1, cnt.getCount());
snk.close();
}
/**
* This tests the new failover builder that uses specs strings as arguments and instantiates them!
*
* @throws InterruptedException
*/
@Test
public void testBackoffFailoverBuilder() throws IOException, InterruptedException {
SinkBuilder bld = BackOffFailOverSink.builder();
EventSink snk =
bld.build(
new ReportTestingContext(),
"{intervalFlakeyAppend(2) => counter(\"pri\") } ",
"counter(\"sec\")");
snk.open();
Event e = new EventImpl("foo".getBytes());
snk.append(e);
snk.append(e);
snk.append(e);
snk.append(e);
snk.append(e);
snk.close();
CounterSink priCnt = (CounterSink) ReportManager.get().getReportable("pri");
CounterSink secCnt = (CounterSink) ReportManager.get().getReportable("sec");
// these are timing based, may fail.
Assert.assertEquals(1, priCnt.getCount());
Assert.assertEquals(4, secCnt.getCount());
}
@Test
public void testCountSink() throws IOException, InterruptedException {
Benchmark b = new Benchmark("nullsink");
b.mark("begin");
TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
txt.open();
MemorySinkSource mem = new MemorySinkSource();
mem.open();
EventUtil.dumpAll(txt, mem);
b.mark("disk_loaded");
CounterSink snk = new CounterSink("counter");
EventUtil.dumpAll(mem, snk);
b.mark(snk.getName() + " done", snk.getCount());
b.done();
}
/**
* tests a series of messages being sent when append of the primary will fail succeed or fail
* based on its twiddle state.
*
* @throws InterruptedException
*/
@Test
public void testFailOverSink() throws IOException, InterruptedException {
MockClock mock = new MockClock(0);
Clock.setClock(mock);
CounterSink primary = new CounterSink("primary");
CounterSink secondary = new CounterSink("backup");
ExceptionTwiddleDecorator<CounterSink> twiddle =
new ExceptionTwiddleDecorator<CounterSink>(primary);
BackOffFailOverSink failsink =
new BackOffFailOverSink(twiddle, secondary, 100, 10000); // 100 ms
// initial
// backoff,
// 10000ms max
// backoff
failsink.open();
Event e = new EventImpl("event".getBytes());
// two successful appends to primary.
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(2, primary.getCount());
Assert.assertEquals(0, secondary.getCount());
mock.forward(100);
twiddle.setAppendOk(false); // go to fail over.
failsink.append(e); // primary fails and automatically go to 2ndary
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(1, failsink.getFails()); // one attempt on primary
// failed.
Assert.assertEquals(2, primary.getCount()); // same as before,
Assert.assertEquals(1, secondary.getCount()); // message went to the
// secondary
mock.forward(50);
failsink.append(e); // skip primary and just go to 2ndary
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(1, failsink.getFails()); // still only one attempt on
// primary
Assert.assertEquals(2, primary.getCount()); // same as before,
Assert.assertEquals(2, secondary.getCount()); // message went to the
// secondary
mock.forward(50);
failsink.append(e); // after this fails backoff is now 200
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(2, failsink.getFails()); // try primary
Assert.assertEquals(0, primary.getCount()); // resets because primary
// restarted
// (and still fails)
Assert.assertEquals(3, secondary.getCount()); // but failover to secondary
mock.forward(200);
failsink.append(e); // should go to 2ndary, after this fails backoff is now
// 400
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(3, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(4, secondary.getCount());
twiddle.setAppendOk(true);
failsink.append(e); // even through primary is ok, we are backing off
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(3, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(5, secondary.getCount());
mock.forward(400);
failsink.append(e); // now that the backoff has expired, we retry the
// primary and succeed
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(3, failsink.getFails());
Assert.assertEquals(1, primary.getCount());
Assert.assertEquals(5, secondary.getCount());
// this should succeed, with the counts being equal in primary and
// secondary.
failsink.close();
}
/**
* Purposely tests backoff timeout.
*
* @throws InterruptedException
*/
@Test
public void testFailTimeout() throws IOException, InterruptedException {
System.out.println("===========================");
MockClock mock = new MockClock(0);
Clock.setClock(mock);
CounterSink primary = new CounterSink("primary");
CounterSink secondary = new CounterSink("backup");
ExceptionTwiddleDecorator<CounterSink> twiddle =
new ExceptionTwiddleDecorator<CounterSink>(primary);
BackOffFailOverSink failsink = new BackOffFailOverSink(twiddle, secondary, 100, 1000); // 100 ms
// initial
// backoff,
// 10000ms max
// backoff
failsink.open();
Event e = new EventImpl("event".getBytes());
mock.forward(100);
twiddle.setAppendOk(false); // go to fail over.
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
mock.forward(100);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
mock.forward(200);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
mock.forward(400);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(4, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(8, secondary.getCount());
mock.forward(800);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(5, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(10, secondary.getCount());
// without capping there would be no new fail here bug still the
// twelve on the secondary count.
mock.forward(1000);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(6, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(12, secondary.getCount());
}