/**
* Unit test for write pipeline add/remove.
*
* @throws InterruptedException
*/
public void test_pipelineAddRemove() throws InterruptedException {
final Quorum<?, ?> quorum = quorums[0];
final MockQuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
assertFalse(client.isMember());
assertNull(client.downStreamId);
assertFalse(client.isPipelineMember());
assertEquals(new UUID[] {}, quorum.getPipeline());
actor.memberAdd();
fixture.awaitDeque();
/*
* add to the pipeline. since this is a singleton quorum, the downstream
* service will remain null.
*/
assertNull(client.downStreamId);
actor.pipelineAdd();
fixture.awaitDeque();
assertNull(client.downStreamId);
assertTrue(client.isPipelineMember());
assertEquals(new UUID[] {serviceId}, quorum.getPipeline());
/*
* remove from the pipeline. since this is a singleton quorum, the
* downstream service will remain null.
*/
assertNull(client.downStreamId);
actor.pipelineRemove();
fixture.awaitDeque();
assertNull(client.downStreamId);
assertFalse(client.isPipelineMember());
assertEquals(new UUID[] {}, quorum.getPipeline());
actor.memberRemove();
fixture.awaitDeque();
assertFalse(client.isMember());
}
/**
* Unit test of timeout in {@link Quorum#awaitQuorum(long, TimeUnit)}. and {@link
* Quorum#awaitBreak(long, TimeUnit)}.
*
* @throws AsynchronousQuorumCloseException
* @throws InterruptedException
*/
public void test_awaitQuorum() throws AsynchronousQuorumCloseException, InterruptedException {
final AbstractQuorum<?, ?> quorum = quorums[0];
final MockQuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
final long lastCommitTime = 0L;
final long lastCommitTime2 = 2L;
// declare the service as a quorum member.
actor.memberAdd();
fixture.awaitDeque();
assertTrue(client.isMember());
assertEquals(new UUID[] {serviceId}, quorum.getMembers());
// add to the pipeline.
actor.pipelineAdd();
fixture.awaitDeque();
assertTrue(client.isPipelineMember());
assertEquals(new UUID[] {serviceId}, quorum.getPipeline());
final long timeout = 1500; // ms
final long slop = 100; // margin of error.
{
/*
* Verify that a we timeout when awaiting a quorum meet that does
* not occur.
*/
final AtomicLong didTimeout = new AtomicLong(-1L);
final Thread t =
new Thread() {
public void run() {
final long begin = System.currentTimeMillis();
try {
// wait for a quorum (but will not meet).
log.info("Waiting for quorum meet.");
quorum.awaitQuorum(timeout, TimeUnit.MILLISECONDS);
} catch (TimeoutException e) {
// This is what we are looking for.
final long elapsed = System.currentTimeMillis() - begin;
didTimeout.set(elapsed);
if (log.isInfoEnabled()) log.info("Timeout after " + elapsed + "ms");
} catch (Exception e) {
log.error(e, e);
}
}
};
t.run();
Thread.sleep(timeout + 250 /* ms */);
t.interrupt();
final long elapsed = didTimeout.get();
assertTrue("did not timeout", elapsed != -1);
assertTrue(
"Timeout occurred too soon: elapsed=" + elapsed + ",timeout=" + timeout,
elapsed >= timeout);
assertTrue(
"Timeout took too long: elapsed=" + elapsed + ",timeout=" + timeout,
elapsed < (timeout + slop));
}
// cast a vote for a lastCommitTime.
actor.castVote(lastCommitTime);
fixture.awaitDeque();
assertEquals(1, quorum.getVotes().size());
assertEquals(new UUID[] {serviceId}, quorum.getVotes().get(lastCommitTime));
// verify the consensus was updated.
assertEquals(lastCommitTime, client.lastConsensusValue);
// wait for quorum meet.
final long token1 = quorum.awaitQuorum();
// verify service was joined.
assertTrue(client.isJoinedMember(quorum.token()));
assertEquals(new UUID[] {serviceId}, quorum.getJoined());
// validate the token was assigned.
fixture.awaitDeque();
assertEquals(Quorum.NO_QUORUM + 1, quorum.lastValidToken());
assertEquals(Quorum.NO_QUORUM + 1, quorum.token());
assertTrue(quorum.isQuorumMet());
{
/*
* Verify that we timeout when awaiting a quorum break that does not
* occur.
*/
final AtomicLong didTimeout = new AtomicLong(-1L);
final Thread t =
new Thread() {
public void run() {
final long begin = System.currentTimeMillis();
try {
// wait for a quorum break (but will not break).
log.info("Waiting for quorum break.");
quorum.awaitBreak(timeout, TimeUnit.MILLISECONDS);
} catch (TimeoutException e) {
// This is what we are looking for.
final long elapsed = System.currentTimeMillis() - begin;
didTimeout.set(elapsed);
if (log.isInfoEnabled()) log.error("Timeout after " + elapsed + "ms");
} catch (Exception e) {
log.error(e, e);
}
}
};
t.run();
Thread.sleep(timeout + 250 /* ms */);
t.interrupt();
final long elapsed = didTimeout.get();
assertTrue("did not timeout", elapsed != -1);
assertTrue(
"Timeout occurred too soon: elapsed=" + elapsed + ",timeout=" + timeout,
elapsed >= timeout);
assertTrue(
"Timeout took too long: elapsed=" + elapsed + ",timeout=" + timeout,
elapsed < (timeout + slop));
}
try {
// Verify awaitBreak() does not return normally.
quorum.awaitBreak(1, TimeUnit.MILLISECONDS);
fail("Not expecting quorum break");
} catch (TimeoutException e) {
if (log.isInfoEnabled()) log.info("Ignoring expected excption: " + e);
}
/*
* Do service leave, quorum should break.
*/
actor.serviceLeave();
fixture.awaitDeque();
quorum.awaitBreak();
try {
// Verify awaitBreak() returns normally.
quorum.awaitBreak(1, TimeUnit.MILLISECONDS);
} catch (TimeoutException e) {
fail("Not expecting " + e, e);
}
}
/**
* Unit test verifying that we clear down the quorum's reflection of the distributed quorum state
* where we first have a quorum meet and then terminate the quorum client.
*
* @throws InterruptedException
*/
public void test_serviceJoin_terminateClient() throws InterruptedException {
final AbstractQuorum<?, ?> quorum = quorums[0];
final MockQuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
final long lastCommitTime = 0L;
final long lastCommitTime2 = 2L;
// declare the service as a quorum member.
actor.memberAdd();
fixture.awaitDeque();
assertTrue(client.isMember());
assertEquals(new UUID[] {serviceId}, quorum.getMembers());
// add to the pipeline.
actor.pipelineAdd();
fixture.awaitDeque();
assertTrue(client.isPipelineMember());
assertEquals(new UUID[] {serviceId}, quorum.getPipeline());
// cast a vote for a lastCommitTime.
actor.castVote(lastCommitTime);
fixture.awaitDeque();
assertEquals(1, quorum.getVotes().size());
assertEquals(new UUID[] {serviceId}, quorum.getVotes().get(lastCommitTime));
// verify the consensus was updated.
assertEquals(lastCommitTime, client.lastConsensusValue);
// wait for quorum meet.
final long token1 = quorum.awaitQuorum();
// verify service was joined.
assertTrue(client.isJoinedMember(quorum.token()));
assertEquals(new UUID[] {serviceId}, quorum.getJoined());
// validate the token was assigned.
fixture.awaitDeque();
assertEquals(Quorum.NO_QUORUM + 1, quorum.lastValidToken());
assertEquals(Quorum.NO_QUORUM + 1, quorum.token());
assertTrue(quorum.isQuorumMet());
/*
* Terminate the quorum. The state should be cleared down.
*/
// Verify termination of the quorum for that client.
assertEquals(client, quorum.getClient());
// terminate the quorum.
quorum.terminate();
try {
quorum.getClient();
} catch (IllegalStateException ex) {
log.info("Ignoring expected exception: " + ex);
}
// State was cleared.
assertEquals(Quorum.NO_QUORUM, quorum.token());
assertEquals(Quorum.NO_QUORUM, quorum.lastValidToken());
assertEquals(new UUID[] {}, quorum.getMembers());
assertEquals(new UUID[] {}, quorum.getJoined());
assertEquals(new UUID[] {}, quorum.getPipeline());
assertEquals(Collections.emptyMap(), quorum.getVotes());
try {
// Note: Quorum reference was cleared. Client throws exception.
assertFalse(client.isMember());
} catch (IllegalStateException ex) {
log.info("Ignoring expected exception: " + ex);
}
// re-terminate() is safe.
quorum.terminate();
}
/**
* Unit test for the protocol up to a service join, which triggers a leader election. Since the
* singleton quorum has only one member our client will be elected the leader.
*
* @throws InterruptedException
*/
public void test_serviceJoin() throws InterruptedException {
final AbstractQuorum<?, ?> quorum = quorums[0];
final MockQuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
final long lastCommitTime = 0L;
final long lastCommitTime2 = 2L;
// declare the service as a quorum member.
actor.memberAdd();
fixture.awaitDeque();
assertTrue(client.isMember());
assertEquals(new UUID[] {serviceId}, quorum.getMembers());
// add to the pipeline.
actor.pipelineAdd();
fixture.awaitDeque();
assertTrue(client.isPipelineMember());
assertEquals(new UUID[] {serviceId}, quorum.getPipeline());
// cast a vote for a lastCommitTime.
actor.castVote(lastCommitTime);
fixture.awaitDeque();
assertEquals(1, quorum.getVotes().size());
assertEquals(new UUID[] {serviceId}, quorum.getVotes().get(lastCommitTime));
// verify the consensus was updated.
assertEquals(lastCommitTime, client.lastConsensusValue);
// wait for quorum meet.
final long token1 = quorum.awaitQuorum();
// verify service was joined.
assertTrue(client.isJoinedMember(quorum.token()));
assertEquals(new UUID[] {serviceId}, quorum.getJoined());
// validate the token was assigned.
fixture.awaitDeque();
assertEquals(Quorum.NO_QUORUM + 1, quorum.lastValidToken());
assertEquals(Quorum.NO_QUORUM + 1, quorum.token());
assertTrue(quorum.isQuorumMet());
/*
* Do service leave, quorum should break.
*/
actor.serviceLeave();
fixture.awaitDeque();
quorum.awaitBreak();
// vote was withdrawn.
assertEquals(0, quorum.getVotes().size());
assertEquals(null, quorum.getVotes().get(lastCommitTime));
// verify the consensus was updated.
assertEquals(-1L, client.lastConsensusValue);
assertFalse(quorum.isQuorumMet());
assertEquals(Quorum.NO_QUORUM, quorum.token());
assertEquals(token1, quorum.lastValidToken());
assertFalse(client.isJoinedMember(quorum.token()));
assertEquals(new UUID[] {}, quorum.getJoined());
assertFalse(client.isPipelineMember());
assertEquals(new UUID[] {}, quorum.getPipeline());
/*
* Cast another vote, the quorum should meet again.
*/
actor.pipelineAdd();
fixture.awaitDeque();
actor.castVote(lastCommitTime2);
fixture.awaitDeque();
assertEquals(1, quorum.getVotes().size());
assertEquals(null, quorum.getVotes().get(lastCommitTime));
assertEquals(new UUID[] {serviceId}, quorum.getVotes().get(lastCommitTime2));
// verify the consensus was updated.
assertEquals(lastCommitTime2, client.lastConsensusValue);
// await meet.
final long token2 = quorum.awaitQuorum();
// verify the joined services.
assertEquals(new UUID[] {serviceId}, quorum.getJoined());
// validate the token was assigned by the leader.
assertTrue(quorum.isQuorumMet());
assertEquals(token1 + 1, token2);
assertEquals(token1 + 1, quorum.lastValidToken());
assertTrue(client.isJoinedMember(token2));
assertTrue(client.isLeader(token2));
assertFalse(client.isFollower(token2));
/*
* Do service leave, quorum should break again.
*/
actor.serviceLeave();
fixture.awaitDeque();
quorum.awaitBreak();
// vote was withdrawn.
assertEquals(0, quorum.getVotes().size());
assertEquals(null, quorum.getVotes().get(lastCommitTime));
assertEquals(null, quorum.getVotes().get(lastCommitTime2));
// verify the consensus was updated.
assertEquals(-1L, client.lastConsensusValue);
assertFalse(quorum.isQuorumMet());
assertEquals(Quorum.NO_QUORUM, quorum.token());
assertEquals(token2, quorum.lastValidToken());
assertFalse(client.isJoinedMember(quorum.token()));
assertEquals(new UUID[] {}, quorum.getJoined());
assertFalse(client.isPipelineMember());
assertEquals(new UUID[] {}, quorum.getPipeline());
}