@Test
public void testRemoveMemberExceptions() {
createGroup(2);
ReplicatedEnvironment master = repEnvInfo[0].getEnv();
assertTrue(master.getState().isMaster());
RepNode masterRep = repEnvInfo[0].getRepNode();
try {
masterRep.removeMember(master.getNodeName());
fail("Exception expected.");
} catch (MasterStateException e) {
// Expected
}
try {
masterRep.removeMember("unknown node foobar");
fail("Exception expected.");
} catch (MemberNotFoundException e) {
// Expected
}
masterRep.removeMember(repEnvInfo[1].getRepNode().getNodeName());
try {
masterRep.removeMember(repEnvInfo[1].getRepNode().getNodeName());
fail("Exception expected.");
} catch (MemberNotFoundException e) {
// Expected
}
repEnvInfo[1].closeEnv();
}
@Override
protected void preLogCommitHook() {
super.preLogCommitHook();
RepNode rmMasterNode = repEnvInfo[0].getRepNode();
int size = rmMasterNode.getGroup().getAllElectableMembers().size();
int delNodes = ((size & 1) == 1) ? 2 : 1;
int closeNodeIndex = (size - delNodes) - 1;
/*
* The loop below simulates the concurrent removal of a node while
* a transaction is in progress. It deletes a sufficient number of
* nodes so as to get a lower simple nodes to get to a new lower
* simple majority.
*/
for (int i = repEnvInfo.length - 1; delNodes-- > 0; i--) {
repEnvInfo[i].closeEnv();
rmMasterNode.removeMember(repEnvInfo[i].getRepConfig().getNodeName(), delete);
}
/*
* Shut down an additional undeleted Replica to provoke a
* lack of acks based on the old simple majority.
*/
repEnvInfo[closeNodeIndex].closeEnv();
}
/**
* Bounce the master, causing replica1 to switch roles with the master. If a higher appVersion is
* specified, the bounced node will also be upgraded.
*/
private void bounceMaster(final int appVersion) throws Exception {
/* Disable updates to RepGroupDB due to LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(true);
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv() == masterEnv) {
/*
* Sync up the replication group so that node2 doesn't do
* hard recovery.
*/
RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
/* Disable replay on replicas. */
shutdownFeeder(info.getRepNode(), replicaEnv1);
shutdownFeeder(info.getRepNode(), replicaEnv2);
/* Close the master. */
masterApp.close();
masterApp = null;
info.closeEnv();
masterEnv = null;
/* Force repEnvInfo[2] to the master. */
WaitForMasterListener masterWaiter = new WaitForMasterListener();
replicaEnv2.setStateChangeListener(masterWaiter);
RepNode repNode = repEnvInfo[2].getRepNode();
repNode.forceMaster(true);
/* Enable the LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);
masterWaiter.awaitMastership();
assertTrue(repNode.isMaster());
masterEnv = replicaEnv2;
/* Replica2 was elected, swap names with replica1. */
final ReplicatedEnvironment tmpEnv = replicaEnv1;
replicaEnv1 = replicaEnv2;
replicaEnv2 = tmpEnv;
final AppInterface tmpApp = replicaApp1;
replicaApp1 = replicaApp2;
replicaApp2 = tmpApp;
/* Replica1 (or 2, see above) has been elected master. */
masterApp = newAppObject(appVersion);
masterApp.adopt(replicaApp1);
/* Former master (just upgraded) becomes replica1. */
replicaEnv1 = info.openEnv();
replicaApp1.open(replicaEnv1);
break;
}
}
assertNotNull(masterApp);
assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
}
/* Start the master (the helper node) first */
@Test
public void testGroupCreateMasterFirst() throws DatabaseException {
for (int i = 0; i < repEnvInfo.length; i++) {
ReplicatedEnvironment rep = repEnvInfo[i].openEnv();
State state = rep.getState();
assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
RepNode repNode = RepInternal.getRepImpl(rep).getRepNode();
/* No elections, helper nodes or members queried for master. */
assertEquals(0, repNode.getElections().getElectionCount());
}
}
/*
* Test the API: RepNode.shutdownNetworkBackup/restartNetworkBackup service
* used to disable the service around a replica syncup operation.
*/
@Test
public void testLockout() throws IOException {
setExceptionListener(repEnvInfo[0]);
repEnvInfo[0].openEnv();
RepNode repNode = repEnvInfo[0].getRepNode();
leaveGroupAllButMaster();
repNode.shutdownNetworkBackup();
File backupDir = new File(repEnvInfo[1].getEnvHome().getCanonicalPath() + ".backup");
backupDir.mkdir();
assertTrue(backupDir.exists());
DataChannelFactory channelFactory =
DataChannelFactoryBuilder.construct(RepTestUtils.readRepNetConfig());
EnvironmentImpl envImpl = createEnvImpl(backupDir);
try {
NetworkBackup backup =
new NetworkBackup(
repNode.getSocket(),
backupDir,
new NameIdPair("n1", (short) 1),
true,
envImpl.getFileManager(),
channelFactory);
backup.execute();
fail("expected exception service should not have been available");
} catch (ServiceConnectFailedException e) {
/* Expected. */
} catch (Exception e) {
fail("unexpected exception" + e);
}
repNode.restartNetworkBackup();
try {
NetworkBackup backup =
new NetworkBackup(
repNode.getSocket(),
backupDir,
new NameIdPair("n1", (short) 1),
true,
envImpl.getFileManager(),
channelFactory);
backup.execute();
} catch (Exception e) {
fail("unexpected exception:" + e);
}
envImpl.abnormalClose();
}
/*
* Tests internal node removal APIs.
*/
@Test
public void testRemoveMember() {
createGroup(groupSize);
ReplicatedEnvironment master = repEnvInfo[0].getEnv();
assertTrue(master.getState().isMaster());
RepNode masterRep = repEnvInfo[0].getRepNode();
/* Reduce the group size all the way down to one. */
for (int i = 1; i < groupSize; i++) {
assertTrue(!RepInternal.isClosed(repEnvInfo[i].getEnv()));
masterRep.removeMember(repEnvInfo[i].getEnv().getNodeName());
assertEquals((groupSize - i), masterRep.getGroup().getElectableGroupSize());
}
/* Close the replica handles*/
for (int i = groupSize - 1; i > 0; i--) {
repEnvInfo[i].closeEnv();
}
/* Attempting to re-open them with the same node names should fail. */
for (int i = 1; i < groupSize; i++) {
try {
repEnvInfo[i].openEnv();
fail("Exception expected");
} catch (EnvironmentFailureException e) {
/* Expected, the master should reject the attempt. */
assertEquals(EnvironmentFailureReason.HANDSHAKE_ERROR, e.getReason());
}
}
/* Doing the same but with different node names should be ok. */
for (int i = 1; i < groupSize; i++) {
final RepEnvInfo ri = repEnvInfo[i];
final ReplicationConfig repConfig = ri.getRepConfig();
TestUtils.removeLogFiles("RemoveRepEnvironments", ri.getEnvHome(), false);
repConfig.setNodeName("ReplaceNode_" + i);
ri.openEnv();
assertEquals(i + 1, masterRep.getGroup().getElectableGroupSize());
}
master.close();
}
/**
* Verify that a NetworkBackup that's in progress is aborted by repNode.shutdownNetworkRestore()
* and therefore during a rollback operation.
*/
@Test
public void testNBAbortOnSyncup()
throws IOException, DatabaseException, ServiceConnectFailedException,
LoadThresholdExceededException, InsufficientVLSNRangeException {
setExceptionListener(repEnvInfo[0]);
repEnvInfo[0].openEnv();
final RepNode repNode = repEnvInfo[0].getRepNode();
leaveGroupAllButMaster();
File backupDir = new File(repEnvInfo[1].getEnvHome().getCanonicalPath() + ".backup");
backupDir.mkdir();
DataChannelFactory channelFactory =
DataChannelFactoryBuilder.construct(RepTestUtils.readRepNetConfig());
EnvironmentImpl envImpl = createEnvImpl(backupDir);
NetworkBackup backup =
new NetworkBackup(
repNode.getSocket(),
backupDir,
new NameIdPair("n1", (short) 1),
true,
envImpl.getFileManager(),
channelFactory);
CyclicBarrier testBarrier =
new CyclicBarrier(
1,
new Runnable() {
public void run() {
/* The syncup should kill the NB */
repNode.shutdownNetworkBackup();
}
});
backup.setTestBarrier(testBarrier);
try {
backup.execute();
fail("Expected exception");
} catch (IOException e) {
/* Expected exception as in progress service was terminated. */
}
envImpl.abnormalClose();
}
/*
* Tests internal node deletion APIs.
*/
@Test
public void testDeleteMember() {
createGroup(groupSize);
ReplicatedEnvironment master = repEnvInfo[0].getEnv();
assertTrue(master.getState().isMaster());
RepNode masterRep = repEnvInfo[0].getRepNode();
/* Reduce the group size all the way down to one. */
for (int i = 1; i < groupSize; i++) {
assertTrue(!RepInternal.isClosed(repEnvInfo[i].getEnv()));
final String delName = repEnvInfo[i].getEnv().getNodeName();
repEnvInfo[i].closeEnv();
masterRep.removeMember(delName, true);
assertEquals((groupSize - i), masterRep.getGroup().getElectableGroupSize());
}
/*
* Attempting to re-open them with the same node names should succeed
*/
for (int i = 1; i < groupSize; i++) {
repEnvInfo[i].openEnv();
}
}
/*
* Remove a feeder from the master so that no entries can be replayed on
* this node.
*/
private void shutdownFeeder(RepNode masterNode, ReplicatedEnvironment repEnv) throws Exception {
String nodeName = repEnv.getRepConfig().getNodeName();
RepNodeImpl removeNode = masterNode.getGroup().getNode(nodeName);
masterNode.feederManager().shutdownFeeder(removeNode);
}
