/**
* Unit test for quorum member add/remove.
*
* @throws InterruptedException
*/
public void test_memberAddRemove() throws InterruptedException {
final Quorum<?, ?> quorum = quorums[0];
final QuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
// client is not a member.
assertFalse(client.isMember());
assertEquals(new UUID[] {}, quorum.getMembers());
// instruct actor to add client as a member.
actor.memberAdd();
fixture.awaitDeque();
// client is a member.
assertTrue(client.isMember());
assertEquals(new UUID[] {serviceId}, quorum.getMembers());
// instruct actor to remove client as a member.
actor.memberRemove();
fixture.awaitDeque();
// client is not a member.
assertFalse(client.isMember());
assertEquals(new UUID[] {}, quorum.getMembers());
}
/**
* Unit test for write pipeline add/remove.
*
* @throws InterruptedException
*/
public void test_pipelineAddRemove() throws InterruptedException {
final Quorum<?, ?> quorum = quorums[0];
final MockQuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
assertFalse(client.isMember());
assertNull(client.downStreamId);
assertFalse(client.isPipelineMember());
assertEquals(new UUID[] {}, quorum.getPipeline());
actor.memberAdd();
fixture.awaitDeque();
/*
* add to the pipeline. since this is a singleton quorum, the downstream
* service will remain null.
*/
assertNull(client.downStreamId);
actor.pipelineAdd();
fixture.awaitDeque();
assertNull(client.downStreamId);
assertTrue(client.isPipelineMember());
assertEquals(new UUID[] {serviceId}, quorum.getPipeline());
/*
* remove from the pipeline. since this is a singleton quorum, the
* downstream service will remain null.
*/
assertNull(client.downStreamId);
actor.pipelineRemove();
fixture.awaitDeque();
assertNull(client.downStreamId);
assertFalse(client.isPipelineMember());
assertEquals(new UUID[] {}, quorum.getPipeline());
actor.memberRemove();
fixture.awaitDeque();
assertFalse(client.isMember());
}
/**
* Unit test for the voting protocol for a singleton quorum.
*
* <p>FIXME For some reason this unit test occasionally takes much longer to run than would
* otherwise be expected (up to a few seconds versus a small fraction of a second). You can see
* this in the timestamps of the logger.
*
* <p>The test terminates when the fixture tears down the AbstractQuorum's internal watcher action
* service, which has a hung action. If the WatcherActionService is also single-threaded, then
* this could clearly lead to a deadlock since there would be no thread available to handle new
* events.
*
* <p>It is awaiting the quorumBreak condition in AbstractQuorumActor.clearToken(). This issue may
* be that we have two distinct signals for quorumBreak versus quorumMeet which need to be
* combined and then the various methods modified to also test the condition variable. [I've made
* that change.]
*
* <p>It seems likely that either a concurrent watcherActionService -or- a finite timeout would
* get the unit tests to pass. However, only the former would work around a deadlock due to a
* stuck Condition.
*
* <p>Look again at what Condition is getting stuck and at the stress test in {@link
* #main(String[])} for causes. [nhang=17, nerr=0, nrun=1000]. There are several different causes,
* each of which clearly reflects a different ordering of the events.
*
* <p>It maybe that we see the problem with a singleton quorum because there are no other sources
* of events to kick the quorum into motion again once it fails to join under the initial impetus.
*
* <p>The problem appears to stem from withdrawing the cast vote before the quorum meets.
*/
public void test_voting()
throws InterruptedException, AsynchronousQuorumCloseException, TimeoutException {
/*
* When true, this test also exercises the awaitQuorum() and
* awaitBreak() methods that accept a timeout, but only for the case in
* which the condition should be true on entry. There is another unit
* test in this class that verifies that the TimeoutException is
* correctly thrown if the condition does not become true within the
* timeout.
*/
final boolean awaitMeetsAndBreaks = true;
final Quorum<?, ?> quorum = quorums[0];
final MockQuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
final long lastCommitTime1 = 0L;
final long lastCommitTime2 = 2L;
// Verify that no consensus has been achieved yet.
assertEquals(-1L, clients[0].lastConsensusValue);
// add as member service.
actor.memberAdd();
fixture.awaitDeque();
assertTrue(clients[0].isMember());
// join the pipeline.
actor.pipelineAdd();
fixture.awaitDeque();
// Verify that timestamps must be non-negative.
try {
actor.castVote(-1L);
fail("Expected " + IllegalArgumentException.class);
} catch (IllegalArgumentException ex) {
if (log.isInfoEnabled()) log.info("Ignoring expected exception: " + ex);
}
// Should not be any votes.
assertEquals(0, quorum.getVotes().size());
// Cast a vote.
actor.castVote(lastCommitTime1);
fixture.awaitDeque();
// Should be just one vote.
assertEquals(1, quorum.getVotes().size());
// Verify the consensus was updated
assertEquals(lastCommitTime1, client.lastConsensusValue);
if (awaitMeetsAndBreaks)
assertEquals(Quorum.NO_QUORUM + 1, quorum.awaitQuorum(100, TimeUnit.MILLISECONDS));
if (awaitMeetsAndBreaks) {
actor.withdrawVote();
quorum.awaitBreak();
/*
* FIXME I have modified castVote() to automatically add the service
* to the pipeline. Otherwise we must do this explicitly after a
* quorum break. Maybe the better approach to take is to have each
* service self-report its lastCommitTime and have Quorum#start(M)
* automatically strive towards a service join while
* Quorum#terminate() causes leaves and then halts processing for
* the service. Alternatively, each client could set its target
* state in MEMBER, PIPELINE, JOIN. That would allow clients to
* manage resynchronization, which they need to do.
*/
// actor.pipelineAdd();
}
// Cast another vote.
actor.castVote(lastCommitTime2);
fixture.awaitDeque();
// Should be just one vote since a service can only vote for one
// lastCommitTime at a time.
if (quorum.getVotes().size() != 1) {
assertEquals(quorum.getVotes().toString(), 1, quorum.getVotes().size());
}
// Verify the consensus was updated again.
assertEquals(lastCommitTime2, client.lastConsensusValue);
if (awaitMeetsAndBreaks)
assertEquals(Quorum.NO_QUORUM + 2, quorum.awaitQuorum(100, TimeUnit.MILLISECONDS));
// Remove as a member.
actor.memberRemove();
fixture.awaitDeque();
assertFalse(clients[0].isMember());
// The service vote was also removed.
assertEquals(0, quorum.getVotes().size());
if (awaitMeetsAndBreaks) quorum.awaitBreak(100, TimeUnit.MILLISECONDS);
}