@Test
public void testCheckup() throws IOException {
MockClock clk = new MockClock(0);
Clock.setClock(clk);
StatusManager stats = new StatusManager();
// foo is in state HELLO and has configuration numbered 0
long prev = Clock.unixTime();
boolean needsRefresh =
stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.HELLO, 0);
LOG.info(stats.getNodeStatuses());
assertTrue(needsRefresh);
// move forward in time, but not far enough to trigger being lost
clk.forward(FlumeConfiguration.get().getConfigHeartbeatPeriod() * 5);
stats.checkup();
StatusManager.NodeStatus ns = stats.getNodeStatuses().get("foo");
assertEquals(0, ns.version);
assertTrue(prev <= ns.lastseen);
assertEquals(NodeState.HELLO, ns.state);
prev = ns.lastseen;
clk.forward(FlumeConfiguration.get().getConfigHeartbeatPeriod() * 20);
stats.checkup();
ns = stats.getNodeStatuses().get("foo");
assertEquals(0, ns.version);
assertTrue(prev <= ns.lastseen);
assertEquals(NodeState.LOST, ns.state);
prev = ns.lastseen;
LOG.info(ns.toString());
LOG.info(stats.getStatus("foo").toString());
}
/**
* tests a series of messages being sent when append of the primary will fail succeed or fail
* based on its twiddle state.
*
* @throws InterruptedException
*/
@Test
public void testFailOverSink() throws IOException, InterruptedException {
MockClock mock = new MockClock(0);
Clock.setClock(mock);
CounterSink primary = new CounterSink("primary");
CounterSink secondary = new CounterSink("backup");
ExceptionTwiddleDecorator<CounterSink> twiddle =
new ExceptionTwiddleDecorator<CounterSink>(primary);
BackOffFailOverSink failsink =
new BackOffFailOverSink(twiddle, secondary, 100, 10000); // 100 ms
// initial
// backoff,
// 10000ms max
// backoff
failsink.open();
Event e = new EventImpl("event".getBytes());
// two successful appends to primary.
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(2, primary.getCount());
Assert.assertEquals(0, secondary.getCount());
mock.forward(100);
twiddle.setAppendOk(false); // go to fail over.
failsink.append(e); // primary fails and automatically go to 2ndary
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(1, failsink.getFails()); // one attempt on primary
// failed.
Assert.assertEquals(2, primary.getCount()); // same as before,
Assert.assertEquals(1, secondary.getCount()); // message went to the
// secondary
mock.forward(50);
failsink.append(e); // skip primary and just go to 2ndary
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(1, failsink.getFails()); // still only one attempt on
// primary
Assert.assertEquals(2, primary.getCount()); // same as before,
Assert.assertEquals(2, secondary.getCount()); // message went to the
// secondary
mock.forward(50);
failsink.append(e); // after this fails backoff is now 200
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(2, failsink.getFails()); // try primary
Assert.assertEquals(0, primary.getCount()); // resets because primary
// restarted
// (and still fails)
Assert.assertEquals(3, secondary.getCount()); // but failover to secondary
mock.forward(200);
failsink.append(e); // should go to 2ndary, after this fails backoff is now
// 400
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(3, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(4, secondary.getCount());
twiddle.setAppendOk(true);
failsink.append(e); // even through primary is ok, we are backing off
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(3, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(5, secondary.getCount());
mock.forward(400);
failsink.append(e); // now that the backoff has expired, we retry the
// primary and succeed
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(3, failsink.getFails());
Assert.assertEquals(1, primary.getCount());
Assert.assertEquals(5, secondary.getCount());
// this should succeed, with the counts being equal in primary and
// secondary.
failsink.close();
}
/**
* Purposely tests backoff timeout.
*
* @throws InterruptedException
*/
@Test
public void testFailTimeout() throws IOException, InterruptedException {
System.out.println("===========================");
MockClock mock = new MockClock(0);
Clock.setClock(mock);
CounterSink primary = new CounterSink("primary");
CounterSink secondary = new CounterSink("backup");
ExceptionTwiddleDecorator<CounterSink> twiddle =
new ExceptionTwiddleDecorator<CounterSink>(primary);
BackOffFailOverSink failsink = new BackOffFailOverSink(twiddle, secondary, 100, 1000); // 100 ms
// initial
// backoff,
// 10000ms max
// backoff
failsink.open();
Event e = new EventImpl("event".getBytes());
mock.forward(100);
twiddle.setAppendOk(false); // go to fail over.
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
mock.forward(100);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
mock.forward(200);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
mock.forward(400);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(4, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(8, secondary.getCount());
mock.forward(800);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(5, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(10, secondary.getCount());
// without capping there would be no new fail here bug still the
// twelve on the secondary count.
mock.forward(1000);
failsink.append(e);
failsink.append(e);
System.out.println(mock);
System.out.printf(
"pri: %4d sec: %4d fail: %4d\n",
primary.getCount(), secondary.getCount(), failsink.getFails());
Assert.assertEquals(6, failsink.getFails());
Assert.assertEquals(0, primary.getCount());
Assert.assertEquals(12, secondary.getCount());
}