/*
* Provoke sufficient log cleaning to move the entire VLSN right
* sufficiently that the new VLSN range no longer overlaps the VLSN
* range upon entry thus guaranteeing a LogFileRefreshException.
*/
private void shiftVLSNRight(ReplicatedEnvironment menv) {
/* Shift the vlsn range window. */
RepImpl menvImpl = repEnvInfo[0].getRepImpl();
final VLSNIndex vlsnIndex = menvImpl.getRepNode().getVLSNIndex();
VLSN masterHigh = vlsnIndex.getRange().getLast();
CheckpointConfig checkpointConfig = new CheckpointConfig();
checkpointConfig.setForce(true);
do {
/*
* Populate just the master, leaving the replica behind Re-populate
* with the same keys to create Cleaner fodder.
*/
populateDB(menv, TEST_DB_NAME, 100);
/*
* Sleep to permit the cbvlsn on the master to be updated. It's
* done with the period: FeederManager.MASTER_CHANGE_CHECK_TIMEOUT
*/
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
fail("unexpected interrupt");
}
menv.cleanLog();
menv.checkpoint(checkpointConfig);
} while (masterHigh.compareTo(vlsnIndex.getRange().getFirst()) > 0);
}
@Test
public void testRemoveMemberExceptions() {
createGroup(2);
ReplicatedEnvironment master = repEnvInfo[0].getEnv();
assertTrue(master.getState().isMaster());
RepNode masterRep = repEnvInfo[0].getRepNode();
try {
masterRep.removeMember(master.getNodeName());
fail("Exception expected.");
} catch (MasterStateException e) {
// Expected
}
try {
masterRep.removeMember("unknown node foobar");
fail("Exception expected.");
} catch (MemberNotFoundException e) {
// Expected
}
masterRep.removeMember(repEnvInfo[1].getRepNode().getNodeName());
try {
masterRep.removeMember(repEnvInfo[1].getRepNode().getNodeName());
fail("Exception expected.");
} catch (MemberNotFoundException e) {
// Expected
}
repEnvInfo[1].closeEnv();
}
/**
* Bounce the master, causing replica1 to switch roles with the master. If a higher appVersion is
* specified, the bounced node will also be upgraded.
*/
private void bounceMaster(final int appVersion) throws Exception {
/* Disable updates to RepGroupDB due to LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(true);
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv() == masterEnv) {
/*
* Sync up the replication group so that node2 doesn't do
* hard recovery.
*/
RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
/* Disable replay on replicas. */
shutdownFeeder(info.getRepNode(), replicaEnv1);
shutdownFeeder(info.getRepNode(), replicaEnv2);
/* Close the master. */
masterApp.close();
masterApp = null;
info.closeEnv();
masterEnv = null;
/* Force repEnvInfo[2] to the master. */
WaitForMasterListener masterWaiter = new WaitForMasterListener();
replicaEnv2.setStateChangeListener(masterWaiter);
RepNode repNode = repEnvInfo[2].getRepNode();
repNode.forceMaster(true);
/* Enable the LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);
masterWaiter.awaitMastership();
assertTrue(repNode.isMaster());
masterEnv = replicaEnv2;
/* Replica2 was elected, swap names with replica1. */
final ReplicatedEnvironment tmpEnv = replicaEnv1;
replicaEnv1 = replicaEnv2;
replicaEnv2 = tmpEnv;
final AppInterface tmpApp = replicaApp1;
replicaApp1 = replicaApp2;
replicaApp2 = tmpApp;
/* Replica1 (or 2, see above) has been elected master. */
masterApp = newAppObject(appVersion);
masterApp.adopt(replicaApp1);
/* Former master (just upgraded) becomes replica1. */
replicaEnv1 = info.openEnv();
replicaApp1.open(replicaEnv1);
break;
}
}
assertNotNull(masterApp);
assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
}
/* Start the master (the helper node) first */
@Test
public void testGroupCreateMasterFirst() throws DatabaseException {
for (int i = 0; i < repEnvInfo.length; i++) {
ReplicatedEnvironment rep = repEnvInfo[i].openEnv();
State state = rep.getState();
assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
RepNode repNode = RepInternal.getRepImpl(rep).getRepNode();
/* No elections, helper nodes or members queried for master. */
assertEquals(0, repNode.getElections().getElectionCount());
}
}
/** Crash the current master, and wait until the group elects a new one. */
private ReplicatedEnvironment crashMasterAndElectNewMaster(
ReplicatedEnvironment master, RepEnvInfo[] repEnvInfo) {
int masterIndex = RepInternal.getNodeId(master) - 1;
logger.info("Crashing " + master.getNodeName());
repEnvInfo[masterIndex].abnormalCloseEnv();
logger.info("Rejoining");
ReplicatedEnvironment newMaster = RepTestUtils.openRepEnvsJoin(repEnvInfo);
logger.info("New master = " + newMaster.getNodeName());
return newMaster;
}
private void testMemberRemoveAckInteraction(final boolean delete) {
createGroup(groupSize);
Transaction txn;
Database db;
try {
MasterTxn.setFactory(new TxnFactory(delete));
ReplicatedEnvironment master = repEnvInfo[0].getEnv();
txn = master.beginTransaction(null, null);
/* Write to the environment. */
db = master.openDatabase(txn, "random", dbconfig);
db.close();
txn.commit();
} catch (InsufficientAcksException e) {
fail("No exception expected.");
} finally {
MasterTxn.setFactory(null);
}
}
/*
* Tests internal node removal APIs.
*/
@Test
public void testRemoveMember() {
createGroup(groupSize);
ReplicatedEnvironment master = repEnvInfo[0].getEnv();
assertTrue(master.getState().isMaster());
RepNode masterRep = repEnvInfo[0].getRepNode();
/* Reduce the group size all the way down to one. */
for (int i = 1; i < groupSize; i++) {
assertTrue(!RepInternal.isClosed(repEnvInfo[i].getEnv()));
masterRep.removeMember(repEnvInfo[i].getEnv().getNodeName());
assertEquals((groupSize - i), masterRep.getGroup().getElectableGroupSize());
}
/* Close the replica handles*/
for (int i = groupSize - 1; i > 0; i--) {
repEnvInfo[i].closeEnv();
}
/* Attempting to re-open them with the same node names should fail. */
for (int i = 1; i < groupSize; i++) {
try {
repEnvInfo[i].openEnv();
fail("Exception expected");
} catch (EnvironmentFailureException e) {
/* Expected, the master should reject the attempt. */
assertEquals(EnvironmentFailureReason.HANDSHAKE_ERROR, e.getReason());
}
}
/* Doing the same but with different node names should be ok. */
for (int i = 1; i < groupSize; i++) {
final RepEnvInfo ri = repEnvInfo[i];
final ReplicationConfig repConfig = ri.getRepConfig();
TestUtils.removeLogFiles("RemoveRepEnvironments", ri.getEnvHome(), false);
repConfig.setNodeName("ReplaceNode_" + i);
ri.openEnv();
assertEquals(i + 1, masterRep.getGroup().getElectableGroupSize());
}
master.close();
}
@Test
public void testNoQuorum() throws DatabaseException, InterruptedException {
for (int i = 0; i < 3; i++) {
ReplicatedEnvironment rep = repEnvInfo[i].openEnv();
State state = rep.getState();
assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
}
RepTestUtils.syncGroupToLastCommit(repEnvInfo, 3);
repEnvInfo[1].closeEnv();
repEnvInfo[2].closeEnv();
// A new node joining in the absence of a quorum must fail
try {
repEnvInfo[3].openEnv();
fail("Expected exception");
} catch (UnknownMasterException e) {
/* Expected. */
}
}
/*
* Tests internal node deletion APIs.
*/
@Test
public void testDeleteMember() {
createGroup(groupSize);
ReplicatedEnvironment master = repEnvInfo[0].getEnv();
assertTrue(master.getState().isMaster());
RepNode masterRep = repEnvInfo[0].getRepNode();
/* Reduce the group size all the way down to one. */
for (int i = 1; i < groupSize; i++) {
assertTrue(!RepInternal.isClosed(repEnvInfo[i].getEnv()));
final String delName = repEnvInfo[i].getEnv().getNodeName();
repEnvInfo[i].closeEnv();
masterRep.removeMember(delName, true);
assertEquals((groupSize - i), masterRep.getGroup().getElectableGroupSize());
}
/*
* Attempting to re-open them with the same node names should succeed
*/
for (int i = 1; i < groupSize; i++) {
repEnvInfo[i].openEnv();
}
}
/**
* Bounce replica2. No election will take place. If a higher appVersion is specified, the bounced
* node will also be upgraded.
*/
private void bounceReplica2(final int appVersion) throws Exception {
replicaApp2.close();
replicaApp2 = null;
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv() == replicaEnv2) {
info.closeEnv();
replicaEnv2 = info.openEnv();
replicaApp2 = newAppObject(appVersion);
replicaApp2.open(replicaEnv2);
break;
}
}
assertNotNull(replicaApp2);
assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
}
/*
* Remove a feeder from the master so that no entries can be replayed on
* this node.
*/
private void shutdownFeeder(RepNode masterNode, ReplicatedEnvironment repEnv) throws Exception {
String nodeName = repEnv.getRepConfig().getNodeName();
RepNodeImpl removeNode = masterNode.getGroup().getNode(nodeName);
masterNode.feederManager().shutdownFeeder(removeNode);
}
private void experienceLogFlushTask(String sleepTime, boolean flushBeforeCrash) throws Throwable {
try {
createRepEnvInfo(sleepTime);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
long startTime = System.currentTimeMillis();
StatsConfig stConfig = new StatsConfig();
stConfig.setClear(true);
/* Flush the existed dirty data before we do writes. */
for (int i = 0; i < repEnvInfo.length; i++) {
repEnvInfo[i].getEnv().sync();
repEnvInfo[i].getEnv().getStats(stConfig);
}
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
Database db = master.openDatabase(null, dbName, dbConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
for (int i = 1; i <= 100; i++) {
IntegerBinding.intToEntry(i, key);
StringBinding.stringToEntry(value, data);
db.put(null, key, data);
}
assertTrue(System.currentTimeMillis() - startTime < 15000);
Thread.sleep(15000);
long endTime = System.currentTimeMillis();
for (int i = 0; i < repEnvInfo.length; i++) {
EnvironmentStats envStats = repEnvInfo[i].getEnv().getStats(stConfig);
LogFlusher flusher = repEnvInfo[i].getRepNode().getLogFlusher();
if (flushBeforeCrash) {
/* Make sure the LogFlushTask has been invoked. */
assertTrue(flusher.getFlushTask().scheduledExecutionTime() > startTime);
assertTrue(flusher.getFlushTask().scheduledExecutionTime() < endTime);
/*
* Since the log file size is not so big, we can't assure
* all the data will be written in the same log file, but
* we can sure that a flush does happen.
*/
assertTrue(envStats.getNSequentialWrites() >= 1);
assertTrue(envStats.getNLogFSyncs() == 1);
} else {
/*
* Make sure the LogFlushTask is not invoked after making
* the changes.
*/
assertTrue(flusher.getFlushTask().scheduledExecutionTime() < startTime);
assertTrue(envStats.getNSequentialWrites() == 0);
assertTrue(envStats.getNLogFSyncs() == 0);
}
assertTrue(envStats.getNFSyncs() == 0);
}
File[] envHomes = new File[3];
/* Close the replicas without doing a checkpoint. */
for (int i = 0; i < repEnvInfo.length; i++) {
envHomes[i] = repEnvInfo[i].getEnvHome();
repEnvInfo[i].getRepImpl().abnormalClose();
}
/*
* Open a read only standalone Environment on the replicas to see
* whether the data has been synced to the disk.
*/
EnvironmentConfig newConfig = new EnvironmentConfig();
newConfig.setAllowCreate(false);
newConfig.setReadOnly(true);
newConfig.setTransactional(true);
for (int i = 0; i < repEnvInfo.length; i++) {
Environment env = new Environment(envHomes[i], newConfig);
dbConfig.setAllowCreate(false);
dbConfig.setReadOnly(true);
try {
db = env.openDatabase(null, dbName, dbConfig);
} catch (DatabaseNotFoundException e) {
/*
* If the system crashes before the flush, the database is
* not synced to the disk, so this database can't be found
* at all, it's expected.
*/
assertFalse(flushBeforeCrash);
}
if (flushBeforeCrash) {
assertTrue(db.count() == 100);
for (int index = 1; index <= 100; index++) {
IntegerBinding.intToEntry(index, key);
OperationStatus status = db.get(null, key, data, null);
if (flushBeforeCrash) {
assertTrue(status == OperationStatus.SUCCESS);
assertEquals(value, StringBinding.entryToString(data));
}
}
}
if (flushBeforeCrash) {
db.close();
}
env.close();
}
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
/**
* Test that a timeout in the feeder while attempting to read the group database because other
* feeders have it write locked causes the feeder (and replica) to fail, but allows the master to
* continue operating. [#23822]
*/
@Test
public void testJoinGroupReadGroupTimeout() throws DatabaseException, InterruptedException {
/* Start first node as master */
ReplicatedEnvironment repEnv = repEnvInfo[0].openEnv();
assertEquals("Master node state", State.MASTER, repEnv.getState());
RepImpl repImpl = RepInternal.getRepImpl(repEnv);
for (int i = 1; i <= 2; i++) {
/* Get a write lock on the RepGroupDB */
final MasterTxn txn =
new MasterTxn(
repImpl,
new TransactionConfig()
.setDurability(
new Durability(
SyncPolicy.SYNC, SyncPolicy.SYNC, ReplicaAckPolicy.SIMPLE_MAJORITY)),
repImpl.getNameIdPair());
final DatabaseImpl groupDbImpl = repImpl.getGroupDb();
final DatabaseEntry value = new DatabaseEntry();
final Cursor cursor = DbInternal.makeCursor(groupDbImpl, txn, new CursorConfig());
final OperationStatus status = cursor.getNext(RepGroupDB.groupKeyEntry, value, LockMode.RMW);
assertEquals(i + ": Lock group result", OperationStatus.SUCCESS, status);
/* Wait longer than the default 500 ms read timeout */
Thread.sleep(600);
/* Test both electable and secondary nodes */
if (i == 2) {
repEnvInfo[i].getRepConfig().setNodeType(NodeType.SECONDARY);
}
/* Create a thread that attempts to join another environment */
RepNodeThread repNodeThread = new RepNodeThread(i, i != 1);
repNodeThread.start();
/* Wait for attempt to complete */
repNodeThread.join(30000);
assertEquals("RN thread alive", false, repNodeThread.isAlive());
if (i == 1) {
/* Join attempt should fail for primary */
assertNotNull("Expected RN thread exception", repNodeThread.te);
/* Release write lock on RepGroupDB */
cursor.close();
txn.abort();
/* Second join attempt should succeed */
repNodeThread = new RepNodeThread(1);
repNodeThread.start();
repNodeThread.join(30000);
assertEquals("RN thread alive", false, repNodeThread.isAlive());
assertEquals("RN thread exception", null, repNodeThread.te);
} else {
/* Join attempt should succeed for secondary */
assertEquals("RN thread exception", null, repNodeThread.te);
/* Release write lock on RepGroupDB */
cursor.close();
txn.abort();
}
}
}