@Test(timeout = 600000)
public void expire_maxSize_with_TTL_get() throws InterruptedException {
if (UtilsTest.scale() == 0) return;
File f = UtilsTest.tempDbFile();
for (int o = 0; o < 2; o++) {
final DB db = DBMaker.fileDB(f).transactionDisable().make();
final HTreeMap<Object, Object> map =
db.hashMapCreate("foo")
.expireMaxSize(1000)
.expireAfterAccess(3, TimeUnit.SECONDS)
.makeOrGet();
map.put("foo", "bar");
for (int i = 0; i < 10; i++) assertEquals("bar", map.get("foo"));
Thread.sleep(6000);
map.get("aa"); // so internal tasks have change to run
assertEquals(null, map.get("foo"));
db.commit();
db.close();
Thread.sleep(1100);
}
}
@Test
public void circular_queue_persisted() {
// i put disk limit 4 objects ,
File f = UtilsTest.tempDbFile();
DB db = DBMaker.newFileDB(f).transactionDisable().cacheDisable().make();
Queue queue = db.createCircularQueue("test", null, 4);
// when i put 6 objects to queue
queue.add(0);
queue.add(1);
queue.add(2);
queue.add(3);
// now deletes 0 on first
queue.add(4);
// now deletes 1
queue.add(5);
db.close();
db = DBMaker.newFileDB(f).transactionDisable().cacheDisable().deleteFilesAfterClose().make();
queue = db.getCircularQueue("test");
assertEquals(2, queue.poll());
assertEquals(3, queue.poll());
assertEquals(4, queue.poll());
assertEquals(5, queue.poll());
assertNull(queue.poll());
db.close();
}
Map<Long, String> fill(StoreWAL e) {
Map<Long, String> ret = new LinkedHashMap<Long, String>();
for (int i = 0; i < 1000; i++) {
String s = UtilsTest.randomString((int) (Math.random() * 10000));
long recid = e.put(s, Serializer.STRING);
ret.put(recid, s);
}
return ret;
}
public class Issue89Test {
private static final String MY_TEST_DATA_FILE = UtilsTest.tempDbFile().getAbsolutePath();
private static final String MAP_DB_DATA_FILE_TO_REMOVE = MY_TEST_DATA_FILE + ".0";
private static final String TEST_TREE_SET = "TestTreeSet";
private static final String DUMMY_CONTENT = "DummyContent";
@Before
public void setUp() throws Exception {
deleteFile();
}
@After
public void tearDown() throws Exception {
deleteFile();
}
@Test
public void testAppend() throws Exception {
appendToDataFile();
appendToDataFile();
appendToDataFile();
appendToDataFile();
}
private void appendToDataFile() {
final DB myTestDataFile = createMapDB(MY_TEST_DATA_FILE);
addData(myTestDataFile);
myTestDataFile.close();
}
private void addData(DB myTestDataFile) {
final NavigableSet<Object> testTreeSet = myTestDataFile.treeSet(TEST_TREE_SET);
testTreeSet.add(DUMMY_CONTENT);
myTestDataFile.commit();
}
private DB createMapDB(String fileName) {
final File file = new File(fileName);
return createMapDB(file);
}
private DB createMapDB(File file) {
return DBMaker.appendFileDB(file).closeOnJvmShutdown().make();
}
private void deleteFile() {
final File file = new File(MAP_DB_DATA_FILE_TO_REMOVE);
if (file.exists()) {
file.delete();
}
}
}
@Test
public void expire_maxSize_with_TTL() throws InterruptedException {
if (UtilsTest.scale() == 0) return;
File f = UtilsTest.tempDbFile();
for (int o = 0; o < 2; o++) {
final DB db = DBMaker.fileDB(f).transactionDisable().make();
final HTreeMap<Object, Object> map =
db.hashMapCreate("foo")
.expireMaxSize(1000)
.expireAfterWrite(1, TimeUnit.DAYS)
.makeOrGet();
map.put("foo", "bar");
assertEquals("bar", map.get("foo"));
Thread.sleep(1100);
assertEquals("bar", map.get("foo"));
db.commit();
db.close();
Thread.sleep(1100);
}
}
@Test
public void queue_persisted() {
File f = UtilsTest.tempDbFile();
DB db = DBMaker.newFileDB(f).transactionDisable().cacheDisable().make();
Queue<Object> queue = db.getQueue("test");
queue.add("1");
queue.add("2");
queue.add("3");
queue.add("4");
db.close();
db = DBMaker.newFileDB(f).transactionDisable().cacheDisable().deleteFilesAfterClose().make();
queue = db.getQueue("test");
assertEquals("1", queue.poll());
assertEquals("2", queue.poll());
assertEquals("3", queue.poll());
assertEquals("4", queue.poll());
assertNull(queue.poll());
db.close();
}
public static void main(String[] args) {
if (1 == 1) return;
// create inMemory store which does not use serialization,
// and has speed comparable to `java.util` collections
DB inMemory = new DB(new StoreHeap());
Map m = inMemory.getTreeMap("test");
Random r = new Random();
// insert random stuff, keep on mind it needs to fit into memory
for (int i = 0; i < 10000; i++) {
m.put(r.nextInt(), "dwqas" + i);
}
// now create on-disk store, it needs to be completely empty
File targetFile = UtilsTest.tempDbFile();
DB target = DBMaker.newFileDB(targetFile).make();
Pump.copy(inMemory, target);
inMemory.close();
target.close();
}
/**
* The test has been passed once the "terminateAfterString" has been seen.
*
* @param args Command line arguments for the runner
* @param terminateAfterString the runner is searching the stdout and stderr for this string. as
* soon as it appears, the test has passed
* @param failOnPatterns The runner is searching stdout and stderr for the pattern (regexp)
* specified here. If one appears, the test has failed
* @param type Set the type of the runner
* @param expectedReturnValue Expected return code from the runner.
* @param checkLogForTerminateString If true, the runner checks also the log4j logger for the
* terminate string
*/
protected void runWithArgs(
String[] args,
String terminateAfterString,
String[] failOnPatterns,
RunTypes type,
int expectedReturnValue,
boolean checkLogForTerminateString) {
LOG.info("Running with args {}", Arrays.toString(args));
outContent = new ByteArrayOutputStream();
errContent = new ByteArrayOutputStream();
System.setOut(new PrintStream(outContent));
System.setErr(new PrintStream(errContent));
// we wait for at most three minutes
final int START_TIMEOUT_SECONDS = 180;
final long deadline = System.currentTimeMillis() + (START_TIMEOUT_SECONDS * 1000);
Runner runner = new Runner(args, type, expectedReturnValue);
runner.start();
boolean expectedStringSeen = false;
boolean testPassedFromLog4j = false;
do {
sleep(1000);
String outContentString = outContent.toString();
String errContentString = errContent.toString();
if (failOnPatterns != null) {
for (String failOnString : failOnPatterns) {
Pattern pattern = Pattern.compile(failOnString);
if (pattern.matcher(outContentString).find()
|| pattern.matcher(errContentString).find()) {
LOG.warn("Failing test. Output contained illegal string '" + failOnString + "'");
sendOutput();
// stopping runner.
runner.sendStop();
Assert.fail("Output contained illegal string '" + failOnString + "'");
}
}
}
// check output for the expected terminateAfterString.
if (checkLogForTerminateString) {
LoggingEvent matchedEvent = UtilsTest.getEventContainingString(terminateAfterString);
if (matchedEvent != null) {
testPassedFromLog4j = true;
LOG.info("Found expected output in logging event {}", matchedEvent);
}
}
if (outContentString.contains(terminateAfterString)
|| errContentString.contains(terminateAfterString)
|| testPassedFromLog4j) {
expectedStringSeen = true;
LOG.info("Found expected output in redirected streams");
// send "stop" command to command line interface
LOG.info("RunWithArgs: request runner to stop");
runner.sendStop();
// wait for the thread to stop
try {
runner.join(30000);
} catch (InterruptedException e) {
LOG.warn("Interrupted while stopping runner", e);
}
LOG.warn("RunWithArgs runner stopped.");
} else {
// check if thread died
if (!runner.isAlive()) {
// leave loop: the runner died, so we can not expect new strings to show up.
break;
}
}
} while (runner.getRunnerError() == null
&& !expectedStringSeen
&& System.currentTimeMillis() < deadline);
sendOutput();
if (runner.getRunnerError() != null) {
// this lets the test fail.
throw new RuntimeException("Runner failed", runner.getRunnerError());
}
Assert.assertTrue(
"During the timeout period of "
+ START_TIMEOUT_SECONDS
+ " seconds the "
+ "expected string did not show up",
expectedStringSeen);
LOG.info("Test was successful");
}
@SuppressWarnings({"rawtypes", "unchecked"})
public class StoreWALTest<E extends StoreWAL> extends StoreCachedTest<E> {
@Override
boolean canRollback() {
return true;
}
File f = UtilsTest.tempDbFile();
@Override
protected E openEngine() {
StoreWAL e = new StoreWAL(f.getPath());
e.init();
return (E) e;
}
@Test
public void WAL_created() {
File wal0 = new File(f.getPath() + ".wal.0");
File wal1 = new File(f.getPath() + ".wal.1");
File wal2 = new File(f.getPath() + ".wal.2");
e = openEngine();
assertTrue(wal0.exists());
assertFalse(wal1.exists());
e.put("aa", Serializer.STRING);
e.commit();
assertTrue(wal0.exists());
assertTrue(wal1.exists());
assertFalse(wal2.exists());
e.put("aa", Serializer.STRING);
e.commit();
assertTrue(wal0.exists());
assertTrue(wal1.exists());
assertTrue(wal2.exists());
}
@Test
public void WAL_replay_long() {
e = openEngine();
long v = e.composeIndexVal(1000, e.round16Up(10000), true, true, true);
long offset = 0xF0000;
e.walPutLong(offset, v);
e.commit();
e.structuralLock.lock();
e.commitLock.lock();
e.replayWAL();
assertEquals(v, e.vol.getLong(offset));
}
@Test
public void WAL_replay_mixed() {
e = openEngine();
e.structuralLock.lock();
for (int i = 0; i < 3; i++) {
long v = e.composeIndexVal(100 + i, e.round16Up(10000) + i * 16, true, true, true);
e.walPutLong(0xF0000 + i * 8, v);
byte[] d = new byte[9];
Arrays.fill(d, (byte) i);
e.putDataSingleWithoutLink(-1, e.round16Up(100000) + 64 + i * 16, d, 0, d.length);
}
e.commit();
e.structuralLock.lock();
e.commitLock.lock();
e.replayWAL();
for (int i = 0; i < 3; i++) {
long v = e.composeIndexVal(100 + i, e.round16Up(10000) + i * 16, true, true, true);
assertEquals(v, e.vol.getLong(0xF0000 + i * 8));
byte[] d = new byte[9];
Arrays.fill(d, (byte) i);
byte[] d2 = new byte[9];
e.vol.getData(e.round16Up(100000) + 64 + i * 16, d2, 0, d2.length);
assertTrue(Serializer.BYTE_ARRAY.equals(d, d2));
}
}
Map<Long, String> fill(StoreWAL e) {
Map<Long, String> ret = new LinkedHashMap<Long, String>();
for (int i = 0; i < 1000; i++) {
String s = UtilsTest.randomString((int) (Math.random() * 10000));
long recid = e.put(s, Serializer.STRING);
ret.put(recid, s);
}
return ret;
}
@Test
public void compact_file_swap_if_seal() {
walCompactSwap(true);
}
@Test
public void compact_file_notswap_if_notseal() {
walCompactSwap(false);
}
protected void walCompactSwap(boolean seal) {
e = openEngine();
Map<Long, String> m = fill(e);
e.commit();
e.close();
// copy file into new location
String compactTarget = e.getWalFileName("c.compactXXX");
Volume f0 = new Volume.FileChannelVol(f);
Volume f = new Volume.FileChannelVol(new File(compactTarget));
f0.copyEntireVolumeTo(f);
f0.close();
f.sync();
f.close();
e = openEngine();
// modify orig file and close
Long recid = m.keySet().iterator().next();
e.update(recid, "aaa", Serializer.STRING);
if (!seal) m.put(recid, "aaa");
e.commit();
e.close();
// now move file so it is valid compacted file
assertTrue(new File(compactTarget).renameTo(new File(e.getWalFileName("c.compact"))));
// create compaction seal
String compactSeal = e.getWalFileName("c");
Volume sealVol = new Volume.FileChannelVol(new File(compactSeal));
sealVol.ensureAvailable(16);
sealVol.putLong(8, StoreWAL.WAL_SEAL + (seal ? 0 : 1));
sealVol.sync();
sealVol.close();
// now reopen file and check its content
// change should be reverted, since compaction file was used
e = openEngine();
for (Long recid2 : m.keySet()) {
assertEquals(m.get(recid2), e.get(recid2, Serializer.STRING));
}
}
@Test(timeout = 100000)
public void compact_commit_works_during_compact() throws InterruptedException {
compact_tx_works(false, true);
}
@Test(timeout = 100000)
public void compact_commit_works_after_compact() throws InterruptedException {
compact_tx_works(false, false);
}
@Test(timeout = 100000)
public void compact_rollback_works_during_compact() throws InterruptedException {
compact_tx_works(true, true);
}
@Test(timeout = 100000)
public void compact_rollback_works_after_compact() throws InterruptedException {
compact_tx_works(true, false);
}
void compact_tx_works(final boolean rollbacks, final boolean pre) throws InterruptedException {
e = openEngine();
Map<Long, String> m = fill(e);
e.commit();
if (pre) e.$_TEST_HACK_COMPACT_PRE_COMMIT_WAIT = true;
else e.$_TEST_HACK_COMPACT_POST_COMMIT_WAIT = true;
Thread t =
new Thread() {
@Override
public void run() {
e.compact();
}
};
t.start();
Thread.sleep(1000);
// we should be able to commit while compaction is running
for (Long recid : m.keySet()) {
boolean revert = rollbacks && Math.random() < 0.5;
e.update(recid, "ZZZ", Serializer.STRING);
if (revert) {
e.rollback();
} else {
e.commit();
m.put(recid, "ZZZ");
}
}
if (pre) assertTrue(t.isAlive());
Thread.sleep(1000);
e.$_TEST_HACK_COMPACT_PRE_COMMIT_WAIT = false;
e.$_TEST_HACK_COMPACT_POST_COMMIT_WAIT = false;
t.join();
for (Long recid : m.keySet()) {
assertEquals(m.get(recid), e.get(recid, Serializer.STRING));
}
e.close();
}
@Test
public void compact_record_file_used() throws IOException {
e = openEngine();
Map<Long, String> m = fill(e);
e.commit();
e.close();
// now create fake compaction file, that should be ignored since seal is broken
String csealFile = e.getWalFileName("c");
Volume cseal = new Volume.FileChannelVol(new File(csealFile));
cseal.ensureAvailable(16);
cseal.putLong(8, 234238492376748923L);
cseal.close();
// create record wal file
String r0 = e.getWalFileName("r0");
Volume r = new Volume.FileChannelVol(new File(r0));
r.ensureAvailable(100000);
r.putLong(8, StoreWAL.WAL_SEAL);
long offset = 16;
// modify all records in map via record wal
for (long recid : m.keySet()) {
r.putUnsignedByte(offset++, 5 << 4);
r.putSixLong(offset, recid);
offset += 6;
String val = "aa" + recid;
m.put(recid, val);
DataIO.DataOutputByteArray b = new DataIO.DataOutputByteArray();
Serializer.STRING.serialize(b, val);
int size = b.pos;
r.putInt(offset, size);
offset += 4;
r.putData(offset, b.buf, 0, size);
offset += size;
}
r.putUnsignedByte(offset, 0);
r.sync();
r.putLong(8, StoreWAL.WAL_SEAL);
r.sync();
r.close();
// reopen engine, record WAL should be replayed
e = openEngine();
// check content of log file replayed into main store
for (long recid : m.keySet()) {
assertEquals(m.get(recid), e.get(recid, Serializer.STRING));
}
e.close();
}
}
