/*
* Test the API: RepImpl.setBackupProhibited would disable the DbBackup in
* DbBackup.startBackup, may be caused by Replay.rollback().
*/
@Test
public void testRollingBackDbBackupAPI() throws Throwable {
RepEnvInfo[] repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 1);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
RepImpl repImpl = RepInternal.getRepImpl(master);
DbBackup backupHelper = new DbBackup(master);
repImpl.setBackupProhibited(true);
try {
backupHelper.startBackup();
fail("Should throw out a LogOverwriteException here.");
} catch (LogOverwriteException e) {
/* Expect a LogOverwriteException here. */
}
repImpl.setBackupProhibited(false);
try {
backupHelper.startBackup();
backupHelper.endBackup();
} catch (Exception e) {
fail("Shouldn't get an exception here.");
} finally {
RepTestUtils.shutdownRepEnvs(repEnvInfo);
}
}
@Test
public void testDefaultJoinGroupHelper() throws UnknownMasterException, DatabaseException {
for (int i = 0; i < repEnvInfo.length; i++) {
RepEnvInfo ri = repEnvInfo[i];
if ((i + 1) == repEnvInfo.length) {
/* Use a non-master helper for the last replicator. */
ReplicationConfig config = RepTestUtils.createRepConfig((short) (i + 1));
String hpPairs = "";
// Skip the master, use all the other nodes
for (int j = 1; j < i; j++) {
hpPairs += "," + repEnvInfo[j].getRepConfig().getNodeHostPort();
}
hpPairs = hpPairs.substring(1);
config.setHelperHosts(hpPairs);
File envHome = ri.getEnvHome();
ri =
repEnvInfo[i] =
new RepEnvInfo(
envHome, config, RepTestUtils.createEnvConfig(Durability.COMMIT_SYNC));
}
ri.openEnv();
State state = ri.getEnv().getState();
assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
}
}
/* Insert 100 records begins with the beginKey. */
private void doWork(Environment master, String dbName, int beginKey) throws Exception {
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
/* Insert/Update the records of the database. */
Database db = master.openDatabase(null, dbName, dbConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
for (int i = 0; i < 100; i++) {
IntegerBinding.intToEntry(beginKey + i, key);
StringBinding.stringToEntry("herococo", data);
db.put(null, key, data);
}
db.close();
/*
* Do a sync at the end of the stage to make sure master and
* replica have the same data set.
*/
VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo);
}
/**
* Create a log that will have swathes of cleaned files that follow the replication stream, or are
* intermingled in the replication stream.
*
* @return master
*/
private Environment setupLogWithCleanedGaps(boolean multipleGaps) throws Exception {
db = null;
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3, makeEnvConfig());
Environment master = RepTestUtils.joinGroup(repEnvInfo);
int masterIdx = findMasterIndex(master);
db = openDatabase(master);
/* Write some data so there is a replication stream. */
generateData(master, 50, Durability.COMMIT_NO_SYNC, true);
/*
* Make the master have a low-utilization log, and gate cleaning
* with a non-updating global cbvlsn. Shut down the replicas so the
* global cbvlsn remains low, and then fill the master with junk.
* The junk will either entirely be to the right of the last VLSN,
* or (since we can't predict RepGroupDB updates) at least within
* the range of the active VLSN range.
*/
closeReplicas(masterIdx);
fillLogWithTraceMsgs(master, 50);
if (multipleGaps) {
Durability noAck =
new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
/* Write more data */
generateData(master, 50, noAck, true);
/* Make a second cleanup area of junk */
fillLogWithTraceMsgs(master, 50);
}
CheckpointConfig cc = new CheckpointConfig();
cc.setForce(true);
master.checkpoint(cc);
EnvironmentStats stats = master.getStats(clearConfig);
stats = master.getStats(clearConfig);
/* Clean the log */
int totalCleaned = 0;
int cleanedThisPass = 0;
do {
cleanedThisPass = cleanLog(master);
totalCleaned += cleanedThisPass;
master.checkpoint(cc);
stats = master.getStats(clearConfig);
logger.info(
"after cleaning, cleaner backlog = "
+ stats.getCleanerBacklog()
+ " deletionBacklog="
+ stats.getFileDeletionBacklog());
} while (cleanedThisPass > 0);
assertTrue(totalCleaned > 0);
return master;
}
/**
* Make a thread allocate a vlsn, but then fail before it's tracked by the vlsn index. This
* happened in [#20919] when 1.rep environment close was called 2.the repNode was nulled out 3.a
* concurrent writing thread got a NPE within its call to LogManager.log because the repNode was
* null. This thread exited after it had bumped the vlsn, but before it had entered the vlsn in
* the vlsnIndex 4.rep environment close tried to do a checkpoint, but the checkpoint hung. This
* fix works by having (3) invalidate the environment, and by having (4) check for an invalidated
* environment.
*/
@Test
public void testLoggingFailure() throws DatabaseException, IOException {
/* Make a single replicated environment. */
RepEnvInfo[] repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 1);
RepTestUtils.joinGroup(repEnvInfo);
/*
* Disable cleaning and CBVLSN updating, to control vlsn creation
* explicitly.
*/
Environment env = repEnvInfo[0].getEnv();
EnvironmentMutableConfig config = env.getMutableConfig();
config.setConfigParam("je.env.runCleaner", "false");
env.setMutableConfig(config);
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
Database db = env.openDatabase(null, "foo", dbConfig);
DatabaseEntry value = new DatabaseEntry(new byte[4]);
EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env);
LogManager logManager = DbInternal.getEnvironmentImpl(env).getLogManager();
/*
* Inject an exception into the next call to log() that is made
* for a replicated log entry.
*/
logManager.setDelayVLSNRegisterHook(new ForceException());
VLSNIndex vlsnIndex = ((RepImpl) envImpl).getVLSNIndex();
try {
db.put(null, value, value);
fail("Should throw exception");
} catch (Exception expected) {
assertTrue(
"latest="
+ vlsnIndex.getLatestAllocatedVal()
+ " last mapped="
+ vlsnIndex.getRange().getLast().getSequence(),
vlsnIndex.getLatestAllocatedVal() > vlsnIndex.getRange().getLast().getSequence());
}
try {
VLSNIndex.AWAIT_CONSISTENCY_MS = 1000;
envImpl.awaitVLSNConsistency();
fail("Should throw and break out");
} catch (DatabaseException expected) {
}
/* Before the fix, this test hung. */
}
/* Start a replication group with 2 nodes and returns the master. */
private ReplicatedEnvironment getMaster() throws Exception {
Durability durability =
new Durability(
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.ReplicaAckPolicy.ALL);
EnvironmentConfig envConfig = RepTestUtils.createEnvConfig(durability);
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 2, envConfig);
return RepTestUtils.joinGroup(repEnvInfo);
}
/**
* Syncup the group and check for these requirements: - the master has all the data we expect -
* the replicas have all the data that is on the master.
*
* <p>- the last VLSN is not a sync VLSN. We want to ensure that the matchpoint is not the last
* VLSN, so the test will need to do rollback
*
* @throws InterruptedException
* @return lastVLSN on the master
*/
private VLSN checkIfWholeGroupInSync(
ReplicatedEnvironment master, RepEnvInfo[] repEnvInfo, RollbackWorkload workload)
throws InterruptedException {
/*
* Make sure we're testing partial rollbacks, and that the replication
* stream is poised at a place where the last sync VLSN != lastVLSN.
*/
VLSN lastVLSN = ensureDistinctLastAndSyncVLSN(master, repEnvInfo);
RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);
/*
* All nodes in the group should have the same data, and it should
* consist of committed and uncommitted updates.
*/
assertTrue(workload.containsAllData(master));
/*
* TODO: Node equality check is temporarily disabled because it (or
* perhaps just the passing of time that allows for a heartbeat) causes
* a GroupDB record to be written, which becomes the matchpoint and
* defeats the test of partial rollback (because there is none).
*/
// RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/*
* TODO: The following fails if checkNodeEquality is called. Perhaps
* we should just do this here and not above at the top of the method.
*/
lastVLSN = ensureDistinctLastAndSyncVLSN(master, repEnvInfo);
return lastVLSN;
}
public void run() {
try {
/* Get FileLocks. */
FileChannel channel = lockFile.getChannel();
FileLock lockA = channel.lock(1, 1, false);
FileLock lockC = channel.lock(3, 1, false);
ReplicatedEnvironment master = getMaster();
doWork(master, dbName, 1);
/* Release lock A so that read process can do reads. */
lockA.release();
/* Make sure read process get lock B before this process. */
Thread.sleep(sleepTime);
/* Get lock B means read process finish reading, do updates. */
FileLock lockB = getLockWithReTry(channel, 2, 1);
doWork(master, dbName, 101);
/* Release lock B and lock C. */
lockB.release();
lockC.release();
} catch (Exception e) {
/* Dump exceptions and exit with value 6. */
e.printStackTrace();
System.exit(7);
} finally {
RepTestUtils.shutdownRepEnvs(repEnvInfo);
closeLockFile(lockFile);
}
}
@Test
public void testLogProviders() throws Exception {
configureForMaxCleaning(5);
final RepEnvInfo minfo = repEnvInfo[0];
/* Add a secondary node */
repEnvInfo = RepTestUtils.setupExtendEnvInfo(repEnvInfo, 1);
final RepEnvInfo sInfo = repEnvInfo[repEnvInfo.length - 1];
sInfo.getRepConfig().setNodeType(NodeType.SECONDARY);
createGroup();
populateDB(minfo.getEnv(), TEST_DB_NAME, 100);
/* The node that will be use for network restores. */
RepEnvInfo nrInfo = repEnvInfo[1];
/* restore from master. */
doAndCheckRestore(nrInfo, minfo);
/* Check restore from specific Replica. */
doAndCheckRestore(nrInfo, repEnvInfo[2]);
/* restore from self should fail. */
try {
doAndCheckRestore(nrInfo, repEnvInfo[1]);
fail("exception expected");
} catch (EnvironmentFailureException e) {
// Expected. Cannot restore from just yourself.
}
/* Restore secondary */
doAndCheckRestore(sInfo, minfo);
/* Restore from secondary */
doAndCheckRestore(nrInfo, sInfo);
}
private void createRepEnvInfo(String sleepTime) throws Throwable {
/*
* Set a large buffer size and disable the checkpointing, so the
* data in the buffer can only be flushed by the LogFlushTask.
*/
EnvironmentConfig envConfig = RepTestUtils.createEnvConfig(Durability.COMMIT_NO_SYNC);
envConfig.setConfigParam(EnvironmentParams.MAX_MEMORY.getName(), "20000000");
envConfig.setConfigParam(EnvironmentParams.LOG_MEM_SIZE.getName(), "120000000");
envConfig.setConfigParam(EnvironmentParams.NUM_LOG_BUFFERS.getName(), "4");
envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CHECKPOINTER, "false");
/* Configure the log flush task. */
ReplicationConfig repConfig = new ReplicationConfig();
repConfig.setConfigParam(ReplicationConfig.LOG_FLUSH_TASK_INTERVAL, sleepTime);
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3, envConfig, repConfig);
}
public MultiProcessOpenEnvTest() throws Exception {
envRoot = SharedTestUtils.getTestDir();
/* Make rep0 as the environment home. */
File[] envHomes = RepTestUtils.makeRepEnvDirs(envRoot, 2);
masterEnvHome = envHomes[0];
replicaEnvHome = envHomes[1];
lockFile = new File(envRoot, LOCK_FILE_NAME);
}
@Override
@Before
public void setUp() throws Exception {
super.setUp();
channelFactory = DataChannelFactoryBuilder.construct(RepTestUtils.readRepNetConfig());
protocol = new Protocol(GROUP_NAME, new NameIdPair(NODE_NAME, 1), null, channelFactory);
protocol.updateNodeIds(new HashSet<Integer>(Arrays.asList(new Integer(1))));
}
private void openGroup() throws IOException {
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
ReplicationConfig repConfig = new ReplicationConfig();
repConfig.setConfigParam(RepParams.VLSN_LOG_CACHE_SIZE.getName(), "2");
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, nNodes, envConfig, repConfig);
master = RepTestUtils.joinGroup(repEnvInfo);
StoreConfig config = new StoreConfig();
config.setAllowCreate(true);
config.setTransactional(true);
store = new EntityStore(master, "test", config);
primaryIndex = store.getPrimaryIndex(Integer.class, AppData.class);
}
/**
* [#18882] Before this bug fix, this test would result in a java.io.FileNotFoundException out of
* FeederReader$SwitchWindow.fillNext.
*/
@Test
public void testDataInWriteQueue() throws Exception {
openGroup();
ExecutorService appThreads = Executors.newFixedThreadPool(numThreads);
int opsPerThread = numRecords / numThreads;
for (int i = 0; i < numThreads; i++) {
appThreads.execute(new AppWork(i, opsPerThread));
}
appThreads.shutdown();
appThreads.awaitTermination(6000, TimeUnit.SECONDS);
VLSN vlsn = RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
RepTestUtils.checkNodeEquality(vlsn, verbose, repEnvInfo);
closeGroup();
}
@Override
@Before
public void setUp() throws Exception {
super.setUp();
/* Add a secondary node */
repEnvInfo = RepTestUtils.setupExtendEnvInfo(repEnvInfo, 1);
repEnvInfo[repEnvInfo.length - 1].getRepConfig().setNodeType(NodeType.SECONDARY);
}
private void doReplicaHasGapNetworkRestore(boolean multiGaps) throws Throwable {
Durability noAck =
new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
db = null;
try {
Environment master = setupLogWithCleanedGaps(multiGaps);
int masterIdx = findMasterIndex(master);
/*
* Write a record, so that we are sure that there will be a
* network restore, because we have to cross a checkpoint.
*/
generateData(master, 1, noAck, false);
CheckpointConfig cc = new CheckpointConfig();
master.checkpoint(cc);
EnvironmentStats stats = master.getStats(clearConfig);
assertEquals(0, stats.getCleanerBacklog());
if (multiGaps) {
logger.info("Multigap: deletion backlog = " + stats.getFileDeletionBacklog());
} else {
assertEquals(0, stats.getFileDeletionBacklog());
}
db.close();
db = null;
repEnvInfo[masterIdx].closeEnv();
/* Start up the two replicas */
openReplicas(masterIdx);
/* Start the node that had been the master */
try {
repEnvInfo[masterIdx].openEnv();
fail("Should be a network restore");
} catch (InsufficientLogException ile) {
repEnvInfo[masterIdx].closeEnv();
NetworkRestore restore = new NetworkRestore();
NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setRetainLogFiles(true);
restore.execute(ile, config);
repEnvInfo[masterIdx].openEnv();
}
/* Check its last VLSN and size. */
} catch (Throwable t) {
t.printStackTrace();
throw t;
} finally {
if (db != null) {
db.close();
}
RepTestUtils.shutdownRepEnvs(repEnvInfo);
}
}
/*
* Test the API: RepImpl.invalidateDbBackups would disable the DbBackup
* at endBackup, may be caused by Replay.rollback().
*/
@Test
public void testRollBackInvalidateDbBackup() throws Exception {
RepEnvInfo[] repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 1);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
final RepImpl repImpl = RepInternal.getRepImpl(master);
DbBackup backupHelper = new DbBackup(master);
backupHelper.startBackup();
backupHelper.setTestHook(
new TestHook<Object>() {
public void doHook() {
repImpl.invalidateBackups(8L);
}
public Object getHookValue() {
throw new UnsupportedOperationException();
}
public void doIOHook() {
throw new UnsupportedOperationException();
}
public void hookSetup() {
throw new UnsupportedOperationException();
}
public void doHook(Object obj) {
throw new UnsupportedOperationException();
}
});
try {
backupHelper.endBackup();
fail("Should throw out a LogOverwriteException here.");
} catch (LogOverwriteException e) {
/* Expect to get a LogOverwriteException here. */
} finally {
RepTestUtils.shutdownRepEnvs(repEnvInfo);
}
}
private void openReplicas(int masterIndex) {
RepEnvInfo[] restartList = new RepEnvInfo[2];
int a = 0;
for (int i = 0; i < repEnvInfo.length; i++) {
if (i != masterIndex) {
restartList[a++] = repEnvInfo[i];
}
}
RepTestUtils.restartGroup(restartList);
}
/**
* Bounce the master, causing replica1 to switch roles with the master. If a higher appVersion is
* specified, the bounced node will also be upgraded.
*/
private void bounceMaster(final int appVersion) throws Exception {
/* Disable updates to RepGroupDB due to LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(true);
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv() == masterEnv) {
/*
* Sync up the replication group so that node2 doesn't do
* hard recovery.
*/
RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
/* Disable replay on replicas. */
shutdownFeeder(info.getRepNode(), replicaEnv1);
shutdownFeeder(info.getRepNode(), replicaEnv2);
/* Close the master. */
masterApp.close();
masterApp = null;
info.closeEnv();
masterEnv = null;
/* Force repEnvInfo[2] to the master. */
WaitForMasterListener masterWaiter = new WaitForMasterListener();
replicaEnv2.setStateChangeListener(masterWaiter);
RepNode repNode = repEnvInfo[2].getRepNode();
repNode.forceMaster(true);
/* Enable the LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);
masterWaiter.awaitMastership();
assertTrue(repNode.isMaster());
masterEnv = replicaEnv2;
/* Replica2 was elected, swap names with replica1. */
final ReplicatedEnvironment tmpEnv = replicaEnv1;
replicaEnv1 = replicaEnv2;
replicaEnv2 = tmpEnv;
final AppInterface tmpApp = replicaApp1;
replicaApp1 = replicaApp2;
replicaApp2 = tmpApp;
/* Replica1 (or 2, see above) has been elected master. */
masterApp = newAppObject(appVersion);
masterApp.adopt(replicaApp1);
/* Former master (just upgraded) becomes replica1. */
replicaEnv1 = info.openEnv();
replicaApp1.open(replicaEnv1);
break;
}
}
assertNotNull(masterApp);
assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
}
public void openEnv() {
try {
Durability durability =
new Durability(
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.ReplicaAckPolicy.ALL);
EnvironmentConfig envConfig = RepTestUtils.createEnvConfig(durability);
ReplicationConfig repConfig = RepTestUtils.createRepConfig(1);
repEnvInfo = RepTestUtils.setupEnvInfo(envHome, envConfig, repConfig, null);
repEnvInfo.openEnv();
Thread.sleep(sleepTime);
} catch (EnvironmentLockedException e) {
/*
* Exit the process with value 1, don't print out the exception
* since it's expected.
*/
System.exit(1);
} catch (UnsupportedOperationException e) {
/*
* Exit the process with value 2, don't print out the exception
* since it's expected.
*
* Note: this exception thrown because we can't start a
* replicated Environment on an existed standalone Environment.
*/
System.exit(2);
} catch (Exception e) {
/* Dump unexpected exceptions, exit processs with value 3. */
e.printStackTrace();
System.exit(3);
} finally {
if (repEnvInfo.getEnv() != null) {
repEnvInfo.closeEnv();
}
}
}
/** Creates a 3 node group and initializes the app classes. */
private void open() throws Exception {
/*
* ReplicaAckPolicy.ALL is used to ensure that when a master operation
* is committed, the change is immediately available on the replica for
* testing -- no waiting in the test is needed.
*/
repEnvInfo =
RepTestUtils.setupEnvInfos(
envRoot,
3,
RepTestUtils.createEnvConfig(
new Durability(
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.ReplicaAckPolicy.ALL)),
new ReplicationConfig());
masterEnv = RepTestUtils.joinGroup(repEnvInfo);
replicaEnv1 = repEnvInfo[1].getEnv();
replicaEnv2 = repEnvInfo[2].getEnv();
/* Load app classes with custom class loader. */
final File evolveParentDir = new File(System.getProperty("testevolvedir"));
final ClassLoader parentClassLoader = Thread.currentThread().getContextClassLoader();
for (int i = 0; i < N_APP_VERSIONS; i += 1) {
final ClassLoader myLoader =
new SimpleClassLoader(parentClassLoader, new File(evolveParentDir, "dplUpgrade." + i));
appClasses[i] = Class.forName(APP_IMPL, true /*initialize*/, myLoader);
}
/* Open v0 app objects. */
masterApp = newAppObject(0);
masterApp.open(masterEnv);
replicaApp1 = newAppObject(0);
replicaApp1.open(replicaEnv1);
replicaApp2 = newAppObject(0);
replicaApp2.open(replicaEnv2);
}
/*
* Test the API: RepNode.shutdownNetworkBackup/restartNetworkBackup service
* used to disable the service around a replica syncup operation.
*/
@Test
public void testLockout() throws IOException {
setExceptionListener(repEnvInfo[0]);
repEnvInfo[0].openEnv();
RepNode repNode = repEnvInfo[0].getRepNode();
leaveGroupAllButMaster();
repNode.shutdownNetworkBackup();
File backupDir = new File(repEnvInfo[1].getEnvHome().getCanonicalPath() + ".backup");
backupDir.mkdir();
assertTrue(backupDir.exists());
DataChannelFactory channelFactory =
DataChannelFactoryBuilder.construct(RepTestUtils.readRepNetConfig());
EnvironmentImpl envImpl = createEnvImpl(backupDir);
try {
NetworkBackup backup =
new NetworkBackup(
repNode.getSocket(),
backupDir,
new NameIdPair("n1", (short) 1),
true,
envImpl.getFileManager(),
channelFactory);
backup.execute();
fail("expected exception service should not have been available");
} catch (ServiceConnectFailedException e) {
/* Expected. */
} catch (Exception e) {
fail("unexpected exception" + e);
}
repNode.restartNetworkBackup();
try {
NetworkBackup backup =
new NetworkBackup(
repNode.getSocket(),
backupDir,
new NameIdPair("n1", (short) 1),
true,
envImpl.getFileManager(),
channelFactory);
backup.execute();
} catch (Exception e) {
fail("unexpected exception:" + e);
}
envImpl.abnormalClose();
}
/** Crash the current master, and wait until the group elects a new one. */
private ReplicatedEnvironment crashMasterAndElectNewMaster(
ReplicatedEnvironment master, RepEnvInfo[] repEnvInfo) {
int masterIndex = RepInternal.getNodeId(master) - 1;
logger.info("Crashing " + master.getNodeName());
repEnvInfo[masterIndex].abnormalCloseEnv();
logger.info("Rejoining");
ReplicatedEnvironment newMaster = RepTestUtils.openRepEnvsJoin(repEnvInfo);
logger.info("New master = " + newMaster.getNodeName());
return newMaster;
}
/** Simulates the scenario where an entire group goes down and is restarted. */
@Test
public void testAllJoinLeaveJoinGroup() throws DatabaseException, InterruptedException {
createGroup();
ReplicatedEnvironment masterRep = repEnvInfo[0].getEnv();
populateDB(masterRep, TEST_DB_NAME, 100);
RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
/* Shutdown the entire group. */
closeNodes(repEnvInfo);
/*
* Restart the group, using a longer join wait time to allow the
* secondary to query the primaries a second time after the election is
* complete. See RepNode.MASTER_QUERY_INTERVAL.
*/
final long masterQueryInterval = 10000;
restartNodes(JOIN_WAIT_TIME + masterQueryInterval, repEnvInfo);
}
@Test
public void testNoQuorum() throws DatabaseException, InterruptedException {
for (int i = 0; i < 3; i++) {
ReplicatedEnvironment rep = repEnvInfo[i].openEnv();
State state = rep.getState();
assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
}
RepTestUtils.syncGroupToLastCommit(repEnvInfo, 3);
repEnvInfo[1].closeEnv();
repEnvInfo[2].closeEnv();
// A new node joining in the absence of a quorum must fail
try {
repEnvInfo[3].openEnv();
fail("Expected exception");
} catch (UnknownMasterException e) {
/* Expected. */
}
}
/**
* Verify that a NetworkBackup that's in progress is aborted by repNode.shutdownNetworkRestore()
* and therefore during a rollback operation.
*/
@Test
public void testNBAbortOnSyncup()
throws IOException, DatabaseException, ServiceConnectFailedException,
LoadThresholdExceededException, InsufficientVLSNRangeException {
setExceptionListener(repEnvInfo[0]);
repEnvInfo[0].openEnv();
final RepNode repNode = repEnvInfo[0].getRepNode();
leaveGroupAllButMaster();
File backupDir = new File(repEnvInfo[1].getEnvHome().getCanonicalPath() + ".backup");
backupDir.mkdir();
DataChannelFactory channelFactory =
DataChannelFactoryBuilder.construct(RepTestUtils.readRepNetConfig());
EnvironmentImpl envImpl = createEnvImpl(backupDir);
NetworkBackup backup =
new NetworkBackup(
repNode.getSocket(),
backupDir,
new NameIdPair("n1", (short) 1),
true,
envImpl.getFileManager(),
channelFactory);
CyclicBarrier testBarrier =
new CyclicBarrier(
1,
new Runnable() {
public void run() {
/* The syncup should kill the NB */
repNode.shutdownNetworkBackup();
}
});
backup.setTestBarrier(testBarrier);
try {
backup.execute();
fail("Expected exception");
} catch (IOException e) {
/* Expected exception as in progress service was terminated. */
}
envImpl.abnormalClose();
}
/**
* On the master, generate a log that has section A: a lot of records packed together section B: a
* lot of junk that gets cleaned away, creating a gap in the log section C: a new section of data
*
* <p>Bring the replicas down after A is replicated, but before C is written. When the replicas
* come up, they will have to be fed by the feeder from point A.
*/
@Test
public void testFeederHasGap() throws Throwable {
Durability noAck =
new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
db = null;
try {
Environment master = setupLogWithCleanedGaps(false);
int masterIdx = findMasterIndex(master);
/*
* Write a single record, and then junk, so that we are sure there
* is a new VLSN, and that the replicas will have to sync up to
* this point, across the gap of cleaned junk.
*/
generateData(master, 1, noAck, false);
EnvironmentStats stats = master.getStats(clearConfig);
assertEquals(0, stats.getCleanerBacklog());
assertEquals(0, stats.getFileDeletionBacklog());
/* Start up the two replicas */
for (int i = 0; i < repEnvInfo.length; i++) {
if (i != masterIdx) {
repEnvInfo[i].openEnv();
/* make sure we have up to date data */
readData(repEnvInfo[i].getEnv(), 50);
}
}
} catch (Throwable t) {
t.printStackTrace();
throw t;
} finally {
if (db != null) {
db.close();
}
RepTestUtils.shutdownRepEnvs(repEnvInfo);
}
}
private void close(boolean normalShutdown) {
try {
if (normalShutdown) {
replicaApp1.close();
replicaApp2.close();
masterApp.close();
RepTestUtils.shutdownRepEnvs(repEnvInfo);
} else {
for (RepEnvInfo info : repEnvInfo) {
info.abnormalCloseEnv();
}
}
} finally {
repEnvInfo = null;
masterEnv = null;
replicaEnv1 = null;
replicaEnv2 = null;
masterApp = null;
replicaApp1 = null;
replicaApp2 = null;
}
}
@Test
public void testReplicaHasGap() throws Throwable {
db = null;
try {
Environment master = setupLogWithCleanedGaps(false);
int masterIdx = findMasterIndex(master);
db.close();
db = null;
repEnvInfo[masterIdx].closeEnv();
/* Start up the two replicas */
openReplicas(masterIdx);
/* Start the master */
try {
repEnvInfo[masterIdx].openEnv();
} catch (InsufficientLogException ile) {
repEnvInfo[masterIdx].closeEnv();
NetworkRestore restore = new NetworkRestore();
NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setRetainLogFiles(true);
restore.execute(ile, config);
repEnvInfo[masterIdx].openEnv();
}
} catch (Throwable t) {
t.printStackTrace();
throw t;
} finally {
if (db != null) {
db.close();
}
RepTestUtils.shutdownRepEnvs(repEnvInfo);
}
}
@Override
public void setUp() throws Exception {
super.setUp();
RepTestUtils.removeRepEnvironments(envRoot);
}
