/* Insert 100 records begins with the beginKey. */ private void doWork(Environment master, String dbName, int beginKey) throws Exception { DatabaseConfig dbConfig = new DatabaseConfig(); dbConfig.setAllowCreate(true); dbConfig.setTransactional(true); /* Insert/Update the records of the database. */ Database db = master.openDatabase(null, dbName, dbConfig); DatabaseEntry key = new DatabaseEntry(); DatabaseEntry data = new DatabaseEntry(); for (int i = 0; i < 100; i++) { IntegerBinding.intToEntry(beginKey + i, key); StringBinding.stringToEntry("herococo", data); db.put(null, key, data); } db.close(); /* * Do a sync at the end of the stage to make sure master and * replica have the same data set. */ VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length); RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo); }
/** * [#18882] Before this bug fix, this test would result in a java.io.FileNotFoundException out of * FeederReader$SwitchWindow.fillNext. */ @Test public void testDataInWriteQueue() throws Exception { openGroup(); ExecutorService appThreads = Executors.newFixedThreadPool(numThreads); int opsPerThread = numRecords / numThreads; for (int i = 0; i < numThreads; i++) { appThreads.execute(new AppWork(i, opsPerThread)); } appThreads.shutdown(); appThreads.awaitTermination(6000, TimeUnit.SECONDS); VLSN vlsn = RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length); RepTestUtils.checkNodeEquality(vlsn, verbose, repEnvInfo); closeGroup(); }
/** * Since the master never dies in this test, no rollbacks should occur, but no data should be lost * either. * * <p>TODO: Should the workload param be of a different class (not a RollbackWorkload), since its * masterSteadyWork method is only called here? */ private void replicasDieAndRejoin(RollbackWorkload workload, int numIterations) throws Throwable { RepEnvInfo[] repEnvInfo = null; try { /* Create a 3 node group. Assign identities. */ repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3); ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo); logger.severe("master=" + master); RepEnvInfo replicaA = null; RepEnvInfo replicaB = null; for (RepEnvInfo info : repEnvInfo) { if (info.getEnv().getState().isMaster()) { continue; } if (replicaA == null) { replicaA = info; } else { replicaB = info; } } /* * For the sake of easy test writing, make sure numIterations is an * even number. */ assertTrue((numIterations % 2) == 0); replicaA.abnormalCloseEnv(); for (int i = 0; i < numIterations; i++) { workload.masterSteadyWork(master); waitForReplicaToSync(master, repEnvInfo); if ((i % 2) == 0) { flushLogAndCrash(replicaB); replicaA.openEnv(); } else { flushLogAndCrash(replicaA); replicaB.openEnv(); } waitForReplicaToSync(master, repEnvInfo); } replicaA.openEnv(); VLSN lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast(); RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN); assertTrue(workload.containsAllData(master)); RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo); workload.close(); for (RepEnvInfo repi : repEnvInfo) { /* * We're done with the test. Bringing down these replicators * forcibly, without closing transactions and whatnot. */ repi.abnormalCloseEnv(); } } catch (Throwable e) { e.printStackTrace(); throw e; } }
/** * Create 3 nodes and replicate operations. Kill off the master, and make the other two resume. * This will require a syncup and a rollback of any operations after the matchpoint. */ private void masterDiesAndRejoins(RollbackWorkload workload) throws Throwable { RepEnvInfo[] repEnvInfo = null; try { /* Create a 3 node group */ repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3); ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo); logger.severe("master=" + master); /* * Run a workload against the master. Sync up the group and check * that all nodes have the same contents. This first workload must * end with in-progress, uncommitted transactions. */ workload.beforeMasterCrash(master); VLSN lastVLSN = VLSN.NULL_VLSN; if (workload.noLockConflict()) { lastVLSN = checkIfWholeGroupInSync(master, repEnvInfo, workload); } /* * Crash the master, find a new master. */ RepEnvInfo oldMaster = repEnvInfo[RepInternal.getNodeId(master) - 1]; master = crashMasterAndElectNewMaster(master, repEnvInfo); RepEnvInfo newMaster = repEnvInfo[RepInternal.getNodeId(master) - 1]; logger.severe("newmaster=" + master); RepEnvInfo alwaysReplica = null; for (RepEnvInfo info : repEnvInfo) { if ((info != oldMaster) && (info != newMaster)) { alwaysReplica = info; break; } } /* * Check that the remaining two nodes only contain committed * updates. * TODO: check that the number of group members is 2. */ assertTrue(workload.containsSavedData(master)); RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo); /* * Do some work against the new master, while the old master is * asleep. Note that the first workload may have contained * in-flight transactions, so this may result in the rollback of * some transactions in the first workload. */ workload.afterMasterCrashBeforeResumption(master); /* * The intent of this test is that the work after crash will end on * an incomplete txn. Check for that. */ lastVLSN = ensureDistinctLastAndSyncVLSN(master, repEnvInfo); /* Now bring up the old master. */ logger.info("Bring up old master"); oldMaster.openEnv(); logger.info("Old master joined"); RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN); logger.info("Old master synced"); /* * Check that all nodes only contain committed updates. */ workload.releaseDbLocks(); assertTrue(workload.containsSavedData(master)); RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo); /* * Now crash the node that has never been a master. Do some work * without it, then recover that node, then do a verification * check. This exercises the recovery of a log that has syncups in * it. */ alwaysReplica.abnormalCloseEnv(); workload.afterReplicaCrash(master); lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast(); RepTestUtils.syncGroupToVLSN(repEnvInfo, 2, lastVLSN); alwaysReplica.openEnv(); RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN); assertTrue(workload.containsSavedData(master)); RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo); RepTestUtils.checkUtilizationProfile(repEnvInfo); workload.close(); /* * We're done with the test. Bringing down these replicators * forcibly, without closing transactions and whatnot. */ for (RepEnvInfo repi : repEnvInfo) { repi.abnormalCloseEnv(); } /* * Open and verify the environments one last time, to ensure that * rollbacks in the recovery interval don't cause problems. */ master = RepTestUtils.restartGroup(repEnvInfo); lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast(); RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN); RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo); /* Final close. */ for (RepEnvInfo repi : repEnvInfo) { repi.closeEnv(); } } catch (Throwable e) { e.printStackTrace(); throw e; } }
/** * This is really multiple tests in one. It tests network restore with a replica in each of the * following three states: * * <p>1) A brand new node joining the group and needing a network restore. * * <p>2) An existing node with its own unique log needing a network restore. * * <p>3) Repeated network restores, reflecting a mature node. */ @Test public void testBasic() throws DatabaseException, Exception { /* * The cleaner thread can see InsufficientLogExceptions so just stifle * those exceptions from stderr. */ DaemonThread.stifleExceptionChatter = true; configureForMaxCleaning(2); final RepEnvInfo info1 = repEnvInfo[0]; RepEnvInfo info2 = repEnvInfo[1]; ReplicatedEnvironment masterRep = info1.openEnv(); Environment menv = masterRep; EnvironmentMutableConfig mconfig = menv.getMutableConfig(); mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "false"); menv.setMutableConfig(mconfig); /* * Have just the master join first. We do this to test the special case * of a brand new node joining a group and needing VLSN 1. The same * node then rejoins with its VLSN > 1 to test subsequent rejoins * where the node has already participated in the replication. */ populateDB(masterRep, TEST_DB_NAME, 100); mconfig = menv.getMutableConfig(); mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "true"); menv.setMutableConfig(mconfig); File cenvDir = info2.getEnvHome(); final int cid = 2; for (int i = 0; i < RESTORE_CYCLES; i++) { leaveGroupAllButMaster(); shiftVLSNRight(masterRep); RepNodeImpl memberPrev = info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName()); /* Node1 is not known on the first iteration. */ final VLSN prevSync = (i == 0) ? null : memberPrev.getBarrierState().getLastCBVLSN(); try { /* Should force a network restore. */ setExceptionListener(info2); info2.openEnv(); fail("exception expected"); } catch (InsufficientLogException e) { RepNodeImpl member = info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName()); /* * The sync state should have been advanced to help contribute * to the global CBVLSN and prevent it from advancing. */ final VLSN currSync = member.getBarrierState().getLastCBVLSN(); assertTrue((i == 0) || currSync.compareTo(prevSync) >= 0); NetworkRestore networkRestore = new NetworkRestore(); networkRestore.execute(e, new NetworkRestoreConfig()); final NetworkBackupStats stats = networkRestore.getNetworkBackupStats(); assertThat(stats.getExpectedBytes(), greaterThan(0)); assertThat(stats.getTransferredBytes(), greaterThan(0)); /* Create a replacement replicator. */ info2 = RepTestUtils.setupEnvInfo(cenvDir, RepTestUtils.DEFAULT_DURABILITY, cid, info1); setExceptionListener(info2); info2.openEnv(); } /* Verify that we can continue with the "restored" log files. */ populateDB(masterRep, TEST_DB_NAME, 100, 100); VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, 2); RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo); info2.closeEnv(); } }