@Test
public void testCustomQuorumFunctionIsPresent() throws Exception {
Config config = new Config();
QuorumConfig quorumConfig = new QuorumConfig();
String quorumName = randomString();
quorumConfig.setName(quorumName);
quorumConfig.setEnabled(true);
quorumConfig.setQuorumFunctionImplementation(
new QuorumFunction() {
@Override
public boolean apply(Collection<Member> members) {
return false;
}
});
config.addQuorumConfig(quorumConfig);
String mapName = randomMapName();
MapConfig mapConfig = new MapConfig(mapName);
mapConfig.setQuorumName(quorumName);
config.addMapConfig(mapConfig);
HazelcastInstance hazelcastInstance = createHazelcastInstance(config);
IMap<Object, Object> map = hazelcastInstance.getMap(mapName);
try {
map.put("1", "1");
fail();
} catch (Exception e) {
}
Quorum quorum = hazelcastInstance.getQuorumService().getQuorum(quorumName);
assertFalse(quorum.isPresent());
}
/**
* Unit test for quorum member add/remove.
*
* @throws InterruptedException
*/
public void test_memberAddRemove() throws InterruptedException {
final Quorum<?, ?> quorum = quorums[0];
final QuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
// client is not a member.
assertFalse(client.isMember());
assertEquals(new UUID[] {}, quorum.getMembers());
// instruct actor to add client as a member.
actor.memberAdd();
fixture.awaitDeque();
// client is a member.
assertTrue(client.isMember());
assertEquals(new UUID[] {serviceId}, quorum.getMembers());
// instruct actor to remove client as a member.
actor.memberRemove();
fixture.awaitDeque();
// client is not a member.
assertFalse(client.isMember());
assertEquals(new UUID[] {}, quorum.getMembers());
}
// Verify that Duplicate Channel result in a failure
@Test(expected = IOException.class)
public void testDuplicateChannelFailure() throws IOException {
quorum1 = new Quorum("q1test", hive, channel1);
quorum1.addChannel(channel2);
assertTrue(quorum1.getList().getNodesMap().containsValue(channel2));
quorum1.addChannel(channel2);
fail();
}
// Add Multiple Channels to a Quorum
@Test
public void testAddN() throws IOException {
quorum1 = new Quorum("q1test", hive, channel1);
quorum1.addChannel(channel2);
assertTrue(quorum1.getList().getNodesMap().containsValue(channel2));
quorum1.addChannel(channel3);
assertTrue(quorum1.getList().getNodesMap().containsValue(channel3));
}
// Verify that the Quorum Creation behave appropriately
@Test
public void testQuorumCreation() throws IOException {
quorum1 = new Quorum("q1test", hive, channel1);
assertEquals(quorum1.Id, "q1test");
assertTrue(quorum1.getList().getNodesMap().containsValue(channel1));
assertTrue(hive.contains("q1test"));
assertTrue(quorum1.isAlive());
}
// Fail adding a channel to Non-Existent Room
@Test(expected = IOException.class)
public void testSendMessageToNonExistentRoom() throws IOException {
quorum1 = new Quorum("q1test", hive, channel1);
quorum1.removeChannel(channel1);
Utility.pause(500);
quorum1.addChannel(channel2);
fail();
}
// Verify that Duplicate Quourms added to the Hive Fails
@Test
public void testDuplicateQuorumFailure() {
try {
quorum1 = new Quorum("q1test", hive, channel1);
quorumOne = new Quorum("q1test", hive, channel1);
fail();
} catch (IOException e) {
}
assertTrue(quorum1.getList().getNodesMap().containsValue(channel1));
assertTrue(hive.contains("q1test"));
assertTrue(hive.getQuorumsMap().containsValue(quorum1));
assertFalse(hive.getQuorumsMap().containsValue(quorumOne));
assertTrue(quorum1.isAlive());
}
/**
* Unit test for write pipeline add/remove.
*
* @throws InterruptedException
*/
public void test_pipelineAddRemove() throws InterruptedException {
final Quorum<?, ?> quorum = quorums[0];
final MockQuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
assertFalse(client.isMember());
assertNull(client.downStreamId);
assertFalse(client.isPipelineMember());
assertEquals(new UUID[] {}, quorum.getPipeline());
actor.memberAdd();
fixture.awaitDeque();
/*
* add to the pipeline. since this is a singleton quorum, the downstream
* service will remain null.
*/
assertNull(client.downStreamId);
actor.pipelineAdd();
fixture.awaitDeque();
assertNull(client.downStreamId);
assertTrue(client.isPipelineMember());
assertEquals(new UUID[] {serviceId}, quorum.getPipeline());
/*
* remove from the pipeline. since this is a singleton quorum, the
* downstream service will remain null.
*/
assertNull(client.downStreamId);
actor.pipelineRemove();
fixture.awaitDeque();
assertNull(client.downStreamId);
assertFalse(client.isPipelineMember());
assertEquals(new UUID[] {}, quorum.getPipeline());
actor.memberRemove();
fixture.awaitDeque();
assertFalse(client.isMember());
}
// Send message to the entire room
@Test
public void testSendMessageToQuorum() throws IOException {
quorum1 = new Quorum("q1test", hive, channel1);
quorum1.addChannel(channel2);
assertTrue(quorum1.getList().getNodesMap().containsValue(channel2));
quorum1.addChannel(channel3);
assertTrue(quorum1.getList().getNodesMap().containsValue(channel3));
channel1.getBuffer().clear();
channel2.getBuffer().clear();
channel3.getBuffer().clear();
quorum1.updateBuffer("test-test-test");
Utility.pause(500);
String s1 = channel1.getBuffer().poll();
assertEquals(s1, "Message (q1test): test-test-test");
assertEquals(channel1.getBuffer().poll(), null);
assertEquals(channel2.getBuffer().poll(), "Message (q1test): test-test-test");
assertEquals(channel2.getBuffer().poll(), null);
assertEquals(channel3.getBuffer().poll(), "Message (q1test): test-test-test");
assertEquals(channel3.getBuffer().poll(), null);
}
// Remove all channels from Quorum and verify that the Quorum is removed
@Test
public void testRemoveAllChannels() throws IOException {
quorum1 = new Quorum("q1test", hive, channel1);
quorum1.addChannel(channel2);
assertTrue(quorum1.getList().getNodesMap().containsValue(channel2));
quorum1.addChannel(channel3);
assertTrue(quorum1.getList().getNodesMap().containsValue(channel3));
quorum1.removeChannel(channel1);
quorum1.removeChannel(channel2);
quorum1.removeChannel(channel3);
Utility.pause(500);
assertFalse(quorum1.getList().getNodesMap().containsValue(channel2));
assertFalse(quorum1.getList().getNodesMap().containsValue(channel1));
assertFalse(quorum1.getList().getNodesMap().containsValue(channel3));
assertFalse(hive.contains("q1test"));
assertFalse(hive.getQuorumsMap().containsValue(quorum1));
assertFalse(quorum1.isAlive());
}
/**
* Unit test for quorum member add followed by the termination of the quorum client. This checks
* for proper termination of the client, including the clear down of the quorum's internal state.
*
* @throws InterruptedException
*/
public void test_memberAdd_terminateClient() throws InterruptedException {
final Quorum<?, ?> quorum = quorums[0];
final QuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
// client is not a member.
assertFalse(client.isMember());
assertEquals(new UUID[] {}, quorum.getMembers());
// instruct actor to add client as a member.
actor.memberAdd();
fixture.awaitDeque();
// client is a member.
assertTrue(client.isMember());
assertEquals(new UUID[] {serviceId}, quorum.getMembers());
/*
* Verify termination of the quorum for that client.
*/
assertEquals(client, quorum.getClient());
quorum.terminate();
try {
quorum.getClient();
} catch (IllegalStateException ex) {
log.info("Ignoring expected exception: " + ex);
}
// State was cleared.
assertEquals(Quorum.NO_QUORUM, quorum.token());
assertEquals(Quorum.NO_QUORUM, quorum.lastValidToken());
assertEquals(new UUID[] {}, quorum.getMembers());
assertEquals(new UUID[] {}, quorum.getJoined());
assertEquals(new UUID[] {}, quorum.getPipeline());
assertEquals(Collections.emptyMap(), quorum.getVotes());
try {
// Note: Quorum reference was cleared. Client throws exception.
assertFalse(client.isMember());
} catch (IllegalStateException ex) {
log.info("Ignoring expected exception: " + ex);
}
// Double-termination is safe.
quorum.terminate();
}
/**
* Unit test for the voting protocol for a singleton quorum.
*
* <p>FIXME For some reason this unit test occasionally takes much longer to run than would
* otherwise be expected (up to a few seconds versus a small fraction of a second). You can see
* this in the timestamps of the logger.
*
* <p>The test terminates when the fixture tears down the AbstractQuorum's internal watcher action
* service, which has a hung action. If the WatcherActionService is also single-threaded, then
* this could clearly lead to a deadlock since there would be no thread available to handle new
* events.
*
* <p>It is awaiting the quorumBreak condition in AbstractQuorumActor.clearToken(). This issue may
* be that we have two distinct signals for quorumBreak versus quorumMeet which need to be
* combined and then the various methods modified to also test the condition variable. [I've made
* that change.]
*
* <p>It seems likely that either a concurrent watcherActionService -or- a finite timeout would
* get the unit tests to pass. However, only the former would work around a deadlock due to a
* stuck Condition.
*
* <p>Look again at what Condition is getting stuck and at the stress test in {@link
* #main(String[])} for causes. [nhang=17, nerr=0, nrun=1000]. There are several different causes,
* each of which clearly reflects a different ordering of the events.
*
* <p>It maybe that we see the problem with a singleton quorum because there are no other sources
* of events to kick the quorum into motion again once it fails to join under the initial impetus.
*
* <p>The problem appears to stem from withdrawing the cast vote before the quorum meets.
*/
public void test_voting()
throws InterruptedException, AsynchronousQuorumCloseException, TimeoutException {
/*
* When true, this test also exercises the awaitQuorum() and
* awaitBreak() methods that accept a timeout, but only for the case in
* which the condition should be true on entry. There is another unit
* test in this class that verifies that the TimeoutException is
* correctly thrown if the condition does not become true within the
* timeout.
*/
final boolean awaitMeetsAndBreaks = true;
final Quorum<?, ?> quorum = quorums[0];
final MockQuorumMember<?> client = clients[0];
final QuorumActor<?, ?> actor = actors[0];
final UUID serviceId = client.getServiceId();
final long lastCommitTime1 = 0L;
final long lastCommitTime2 = 2L;
// Verify that no consensus has been achieved yet.
assertEquals(-1L, clients[0].lastConsensusValue);
// add as member service.
actor.memberAdd();
fixture.awaitDeque();
assertTrue(clients[0].isMember());
// join the pipeline.
actor.pipelineAdd();
fixture.awaitDeque();
// Verify that timestamps must be non-negative.
try {
actor.castVote(-1L);
fail("Expected " + IllegalArgumentException.class);
} catch (IllegalArgumentException ex) {
if (log.isInfoEnabled()) log.info("Ignoring expected exception: " + ex);
}
// Should not be any votes.
assertEquals(0, quorum.getVotes().size());
// Cast a vote.
actor.castVote(lastCommitTime1);
fixture.awaitDeque();
// Should be just one vote.
assertEquals(1, quorum.getVotes().size());
// Verify the consensus was updated
assertEquals(lastCommitTime1, client.lastConsensusValue);
if (awaitMeetsAndBreaks)
assertEquals(Quorum.NO_QUORUM + 1, quorum.awaitQuorum(100, TimeUnit.MILLISECONDS));
if (awaitMeetsAndBreaks) {
actor.withdrawVote();
quorum.awaitBreak();
/*
* FIXME I have modified castVote() to automatically add the service
* to the pipeline. Otherwise we must do this explicitly after a
* quorum break. Maybe the better approach to take is to have each
* service self-report its lastCommitTime and have Quorum#start(M)
* automatically strive towards a service join while
* Quorum#terminate() causes leaves and then halts processing for
* the service. Alternatively, each client could set its target
* state in MEMBER, PIPELINE, JOIN. That would allow clients to
* manage resynchronization, which they need to do.
*/
// actor.pipelineAdd();
}
// Cast another vote.
actor.castVote(lastCommitTime2);
fixture.awaitDeque();
// Should be just one vote since a service can only vote for one
// lastCommitTime at a time.
if (quorum.getVotes().size() != 1) {
assertEquals(quorum.getVotes().toString(), 1, quorum.getVotes().size());
}
// Verify the consensus was updated again.
assertEquals(lastCommitTime2, client.lastConsensusValue);
if (awaitMeetsAndBreaks)
assertEquals(Quorum.NO_QUORUM + 2, quorum.awaitQuorum(100, TimeUnit.MILLISECONDS));
// Remove as a member.
actor.memberRemove();
fixture.awaitDeque();
assertFalse(clients[0].isMember());
// The service vote was also removed.
assertEquals(0, quorum.getVotes().size());
if (awaitMeetsAndBreaks) quorum.awaitBreak(100, TimeUnit.MILLISECONDS);
}
