/**
* Create a log that will have swathes of cleaned files that follow the replication stream, or are
* intermingled in the replication stream.
*
* @return master
*/
private Environment setupLogWithCleanedGaps(boolean multipleGaps) throws Exception {
db = null;
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3, makeEnvConfig());
Environment master = RepTestUtils.joinGroup(repEnvInfo);
int masterIdx = findMasterIndex(master);
db = openDatabase(master);
/* Write some data so there is a replication stream. */
generateData(master, 50, Durability.COMMIT_NO_SYNC, true);
/*
* Make the master have a low-utilization log, and gate cleaning
* with a non-updating global cbvlsn. Shut down the replicas so the
* global cbvlsn remains low, and then fill the master with junk.
* The junk will either entirely be to the right of the last VLSN,
* or (since we can't predict RepGroupDB updates) at least within
* the range of the active VLSN range.
*/
closeReplicas(masterIdx);
fillLogWithTraceMsgs(master, 50);
if (multipleGaps) {
Durability noAck =
new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
/* Write more data */
generateData(master, 50, noAck, true);
/* Make a second cleanup area of junk */
fillLogWithTraceMsgs(master, 50);
}
CheckpointConfig cc = new CheckpointConfig();
cc.setForce(true);
master.checkpoint(cc);
EnvironmentStats stats = master.getStats(clearConfig);
stats = master.getStats(clearConfig);
/* Clean the log */
int totalCleaned = 0;
int cleanedThisPass = 0;
do {
cleanedThisPass = cleanLog(master);
totalCleaned += cleanedThisPass;
master.checkpoint(cc);
stats = master.getStats(clearConfig);
logger.info(
"after cleaning, cleaner backlog = "
+ stats.getCleanerBacklog()
+ " deletionBacklog="
+ stats.getFileDeletionBacklog());
} while (cleanedThisPass > 0);
assertTrue(totalCleaned > 0);
return master;
}
private void expectNothingToClean() {
env.cleanLog();
final EnvironmentStats stats = env.getStats(null);
final String msg =
String.format(
"%d probes, %d non-probes", stats.getNCleanerProbeRuns(), stats.getNCleanerRuns());
assertEquals(msg, 0, stats.getNCleanerRuns() - stats.getNCleanerProbeRuns());
}
private void doReplicaHasGapNetworkRestore(boolean multiGaps) throws Throwable {
Durability noAck =
new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
db = null;
try {
Environment master = setupLogWithCleanedGaps(multiGaps);
int masterIdx = findMasterIndex(master);
/*
* Write a record, so that we are sure that there will be a
* network restore, because we have to cross a checkpoint.
*/
generateData(master, 1, noAck, false);
CheckpointConfig cc = new CheckpointConfig();
master.checkpoint(cc);
EnvironmentStats stats = master.getStats(clearConfig);
assertEquals(0, stats.getCleanerBacklog());
if (multiGaps) {
logger.info("Multigap: deletion backlog = " + stats.getFileDeletionBacklog());
} else {
assertEquals(0, stats.getFileDeletionBacklog());
}
db.close();
db = null;
repEnvInfo[masterIdx].closeEnv();
/* Start up the two replicas */
openReplicas(masterIdx);
/* Start the node that had been the master */
try {
repEnvInfo[masterIdx].openEnv();
fail("Should be a network restore");
} catch (InsufficientLogException ile) {
repEnvInfo[masterIdx].closeEnv();
NetworkRestore restore = new NetworkRestore();
NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setRetainLogFiles(true);
restore.execute(ile, config);
repEnvInfo[masterIdx].openEnv();
}
/* Check its last VLSN and size. */
} catch (Throwable t) {
t.printStackTrace();
throw t;
} finally {
if (db != null) {
db.close();
}
RepTestUtils.shutdownRepEnvs(repEnvInfo);
}
}
private void expectBackgroundCleaning() {
final long endTime = System.currentTimeMillis() + (30 * 1000);
while (System.currentTimeMillis() < endTime) {
final EnvironmentStats stats = env.getStats(null);
if (stats.getNCleanerRuns() > 0) {
return;
}
}
close();
fail("Cleaner did not run");
}
/**
* On the master, generate a log that has section A: a lot of records packed together section B: a
* lot of junk that gets cleaned away, creating a gap in the log section C: a new section of data
*
* <p>Bring the replicas down after A is replicated, but before C is written. When the replicas
* come up, they will have to be fed by the feeder from point A.
*/
@Test
public void testFeederHasGap() throws Throwable {
Durability noAck =
new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
db = null;
try {
Environment master = setupLogWithCleanedGaps(false);
int masterIdx = findMasterIndex(master);
/*
* Write a single record, and then junk, so that we are sure there
* is a new VLSN, and that the replicas will have to sync up to
* this point, across the gap of cleaned junk.
*/
generateData(master, 1, noAck, false);
EnvironmentStats stats = master.getStats(clearConfig);
assertEquals(0, stats.getCleanerBacklog());
assertEquals(0, stats.getFileDeletionBacklog());
/* Start up the two replicas */
for (int i = 0; i < repEnvInfo.length; i++) {
if (i != masterIdx) {
repEnvInfo[i].openEnv();
/* make sure we have up to date data */
readData(repEnvInfo[i].getEnv(), 50);
}
}
} catch (Throwable t) {
t.printStackTrace();
throw t;
} finally {
if (db != null) {
db.close();
}
RepTestUtils.shutdownRepEnvs(repEnvInfo);
}
}
void loadStats(StatsConfig config, EnvironmentStats stats) {
stats.setCacheDataBytes(getCacheMemoryUsage());
}
private void experienceLogFlushTask(String sleepTime, boolean flushBeforeCrash) throws Throwable {
try {
createRepEnvInfo(sleepTime);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
long startTime = System.currentTimeMillis();
StatsConfig stConfig = new StatsConfig();
stConfig.setClear(true);
/* Flush the existed dirty data before we do writes. */
for (int i = 0; i < repEnvInfo.length; i++) {
repEnvInfo[i].getEnv().sync();
repEnvInfo[i].getEnv().getStats(stConfig);
}
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
Database db = master.openDatabase(null, dbName, dbConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
for (int i = 1; i <= 100; i++) {
IntegerBinding.intToEntry(i, key);
StringBinding.stringToEntry(value, data);
db.put(null, key, data);
}
assertTrue(System.currentTimeMillis() - startTime < 15000);
Thread.sleep(15000);
long endTime = System.currentTimeMillis();
for (int i = 0; i < repEnvInfo.length; i++) {
EnvironmentStats envStats = repEnvInfo[i].getEnv().getStats(stConfig);
LogFlusher flusher = repEnvInfo[i].getRepNode().getLogFlusher();
if (flushBeforeCrash) {
/* Make sure the LogFlushTask has been invoked. */
assertTrue(flusher.getFlushTask().scheduledExecutionTime() > startTime);
assertTrue(flusher.getFlushTask().scheduledExecutionTime() < endTime);
/*
* Since the log file size is not so big, we can't assure
* all the data will be written in the same log file, but
* we can sure that a flush does happen.
*/
assertTrue(envStats.getNSequentialWrites() >= 1);
assertTrue(envStats.getNLogFSyncs() == 1);
} else {
/*
* Make sure the LogFlushTask is not invoked after making
* the changes.
*/
assertTrue(flusher.getFlushTask().scheduledExecutionTime() < startTime);
assertTrue(envStats.getNSequentialWrites() == 0);
assertTrue(envStats.getNLogFSyncs() == 0);
}
assertTrue(envStats.getNFSyncs() == 0);
}
File[] envHomes = new File[3];
/* Close the replicas without doing a checkpoint. */
for (int i = 0; i < repEnvInfo.length; i++) {
envHomes[i] = repEnvInfo[i].getEnvHome();
repEnvInfo[i].getRepImpl().abnormalClose();
}
/*
* Open a read only standalone Environment on the replicas to see
* whether the data has been synced to the disk.
*/
EnvironmentConfig newConfig = new EnvironmentConfig();
newConfig.setAllowCreate(false);
newConfig.setReadOnly(true);
newConfig.setTransactional(true);
for (int i = 0; i < repEnvInfo.length; i++) {
Environment env = new Environment(envHomes[i], newConfig);
dbConfig.setAllowCreate(false);
dbConfig.setReadOnly(true);
try {
db = env.openDatabase(null, dbName, dbConfig);
} catch (DatabaseNotFoundException e) {
/*
* If the system crashes before the flush, the database is
* not synced to the disk, so this database can't be found
* at all, it's expected.
*/
assertFalse(flushBeforeCrash);
}
if (flushBeforeCrash) {
assertTrue(db.count() == 100);
for (int index = 1; index <= 100; index++) {
IntegerBinding.intToEntry(index, key);
OperationStatus status = db.get(null, key, data, null);
if (flushBeforeCrash) {
assertTrue(status == OperationStatus.SUCCESS);
assertEquals(value, StringBinding.entryToString(data));
}
}
}
if (flushBeforeCrash) {
db.close();
}
env.close();
}
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}