/** Tests that we can load a store after the index has been corrupted */
@Test
public void testLoadPersistentStoreAfterCorruption() throws IOException, InterruptedException {
// initialise
String cacheName = "testPersistent";
Store diskStore = createPersistentDiskStore(cacheName);
diskStore.removeAll();
for (int i = 0; i < 100; i++) {
byte[] data = new byte[1024];
diskStore.put(new Element("key" + (i + 100), data));
}
waitShorter();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
assertEquals(100, diskStore.getSize());
manager.removeCache(cacheName);
File indexFile = ((DiskPersistentStore) diskStore).getIndexFile();
FileOutputStream fout = new FileOutputStream(indexFile);
// corrupt the index file
fout.write(new byte[] {'q', 'w', 'e', 'r', 't', 'y'});
fout.close();
diskStore = createPersistentDiskStore(cacheName);
File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
assertTrue("File exists", dataFile.exists());
// Make sure the data file got recreated since the index was corrupt
assertEquals("Data file was not recreated", 0, dataFile.length());
assertEquals(0, diskStore.getSize());
}
/** Any disk store with an auto generated random directory should not be able to be loaded. */
@Test
public void testCannotLoadPersistentStoreWithAutoDir() throws IOException, InterruptedException {
// initialise
String cacheName = "testPersistent";
Store diskStore = createAutoPersistentDiskStore(cacheName);
diskStore.removeAll();
for (int i = 0; i < 100; i++) {
byte[] data = new byte[1024];
diskStore.put(new Element("key" + (i + 100), data));
}
waitLonger();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
assertEquals(100, diskStore.getSize());
manager2.removeCache(cacheName);
Thread.sleep(1000);
Cache cache = new Cache(cacheName, 10000, true, false, 5, 1, true, 600);
manager2.addCache(cache);
File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
assertTrue("File exists", dataFile.exists());
assertEquals(0, dataFile.length());
assertEquals(0, cache.getSize());
manager.removeCache(cacheName);
assertTrue("File exists", dataFile.exists());
assertEquals(0, dataFile.length());
}
/** Tests that we can save and load a persistent store in a repeatable way */
@Test
public void testLoadPersistentStore() throws IOException, InterruptedException {
// initialise
String cacheName = "testLoadPersistent";
Store store = createPersistentDiskStore(cacheName);
store.removeAll();
waitShorter();
for (int i = 0; i < 100; i++) {
byte[] data = new byte[1024];
store.put(new Element("key" + (i + 100), data));
}
store.flush();
waitShorter();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, store.getOnDiskSizeInBytes());
assertEquals(100, store.getSize());
manager.removeCache(cacheName);
Thread.sleep(3000);
// check that we can create and dispose several times with no problems and no lost data
for (int i = 0; i < 10; i++) {
store = createPersistentDiskStore(cacheName);
File dataFile = ((DiskPersistentStore) store).getDataFile();
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(100, store.getSize());
manager.removeCache(cacheName);
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
}
}
/** This method tries to get the store too leak. */
protected long thrashCache() throws Exception {
long startingSize = measureMemoryUse();
LOG.info("Starting memory used is: " + startingSize);
final String value = "value";
final String key = "key";
// Add the entry
final Element element = new Element(key, value);
store.put(element);
// Create 15 threads that read the keys;
final List executables = new ArrayList();
for (int i = 0; i < 15; i++) {
final LruMemoryStoreTest.Executable executable =
new MemoryStoreTester.Executable() {
public void execute() throws Exception {
for (int i = 0; i < 500; i++) {
final String key = "key" + i;
store.get(key);
}
store.get("key");
}
};
executables.add(executable);
}
// Create 15 threads that are insert 500 keys with large byte[] as values
for (int i = 0; i < 15; i++) {
final MemoryStoreTester.Executable executable =
new MemoryStoreTester.Executable() {
public void execute() throws Exception {
// Add a bunch of entries
for (int i = 0; i < 500; i++) {
// Use a random length value
final String key = "key" + i;
byte[] value = new byte[10000];
Element element = new Element(key, value);
store.put(element);
}
}
};
executables.add(executable);
}
runThreads(executables);
store.removeAll();
long finishingSize = measureMemoryUse();
LOG.info("Ending memory used is: " + finishingSize);
return finishingSize - startingSize;
}
/**
* Tests that we can save and load a persistent store in a repeatable way, and delete and add
* data.
*/
@Test
public void testFreeSpaceBehaviour() throws IOException, InterruptedException {
// initialise
String cacheName = "testPersistent";
Store diskStore = createPersistentDiskStore(cacheName);
diskStore.removeAll();
byte[] data = new byte[1024];
for (int i = 0; i < 100; i++) {
diskStore.put(new Element("key" + (i + 100), data));
}
waitShorter();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
assertEquals(100, diskStore.getSize());
manager.removeCache(cacheName);
diskStore = createPersistentDiskStore(cacheName);
File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(100, diskStore.getSize());
diskStore.remove("key100");
diskStore.remove("key101");
diskStore.remove("key102");
diskStore.remove("key103");
diskStore.remove("key104");
diskStore.put(new Element("key100", data));
diskStore.put(new Element("key101", data));
waitShorter();
// The file does not shrink.
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(97, diskStore.getSize());
diskStore.put(new Element("key102", data));
diskStore.put(new Element("key103", data));
diskStore.put(new Element("key104", data));
diskStore.put(new Element("key201", data));
diskStore.put(new Element("key202", data));
waitShorter();
assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(102, diskStore.getSize());
manager.removeCache(cacheName);
assertTrue("File exists", dataFile.exists());
assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
}
/**
* Tests that a file is created with the right size after puts, and that the file is not deleted
* on disposal
*
* <p>This test uses a preconfigured cache from the test cache.xml. Note that teardown causes an
* exception because the disk store is being shut down twice.
*/
@Test
public void testPersistentStore() throws IOException, InterruptedException, CacheException {
// initialise
Store store = createPersistentDiskStoreFromCacheManager();
store.removeAll();
File dataFile = ((DiskPersistentStore) store).getDataFile();
for (int i = 0; i < 100; i++) {
byte[] data = new byte[1024];
store.put(new Element("key" + (i + 100), data));
}
waitShorter();
assertEquals(100, store.getSize());
store.dispose();
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
}
/**
* Checks that the expiry thread runs and expires elements which has the effect of preventing the
* disk store from continously growing. Ran for 6 hours through 10000 outer loops. No memory use
* increase. Using a key of "key" + i * outer) you get early slots that cannot be reused. The
* DiskStore actual size therefore starts at 133890 and ends at 616830. There is quite a lot of
* space that cannot be used because of fragmentation. Question? Should an effort be made to
* coalesce fragmented space? Unlikely in production to get contiguous fragments as in the first
* form of this test.
*
* <p>Using a key of Integer.valueOf(i * outer) the size stays constant at 140800.
*
* @throws InterruptedException
*/
@Test
public void testExpiryWithSize() throws InterruptedException {
Store diskStore = createDiskStore();
diskStore.removeAll();
byte[] data = new byte[1024];
for (int outer = 1; outer <= 10; outer++) {
for (int i = 0; i < 101; i++) {
Element element = new Element(Integer.valueOf(i * outer), data);
element.setTimeToLive(1);
diskStore.put(element);
}
waitLonger();
int predictedSize = (ELEMENT_ON_DISK_SIZE + 68) * 100;
long actualSize = diskStore.getOnDiskSizeInBytes();
LOG.info("Predicted Size: " + predictedSize + " Actual Size: " + actualSize);
assertEquals(predictedSize, actualSize);
LOG.info("Memory Use: " + measureMemoryUse());
}
}
/** Tests removing all the entries. */
@Test
public void testRemoveAll() throws Exception {
final String value = "value";
final String key = "key";
// Add the entry
Element element = new Element(key, value);
store.put(element);
// Check the entry is there
element = store.get(key);
assertNotNull(element);
// Remove it
store.removeAll();
// Check the entry is not there
assertEquals(0, store.getSize());
element = store.get(key);
assertNull(element);
}
/** Tests removing all the entries. */
@Test
public void testRemoveAll() throws Exception {
final Store diskStore = createDiskStore();
// Add the entry
final String value = "value";
diskStore.put(new Element("key1", value));
diskStore.put(new Element("key2", value));
// Check the entry is there
assertNotNull(diskStore.get("key1"));
assertNotNull(diskStore.get("key2"));
// Remove it
diskStore.removeAll();
// Check the entry is not there
assertEquals(0, diskStore.getSize());
assertEquals(0, diskStore.getOnDiskSize());
assertNull(diskStore.get("key1"));
assertNull(diskStore.get("key2"));
}
/**
* Tests that we can save and load a persistent store in a repeatable way, and delete and add
* data.
*/
@Test
public void testLoadPersistentStoreWithDelete() throws IOException, InterruptedException {
// initialise
String cacheName = "testPersistentWithDelete";
Store diskStore = createPersistentDiskStore(cacheName);
diskStore.removeAll();
for (int i = 0; i < 100; i++) {
byte[] data = new byte[1024];
diskStore.put(new Element("key" + (i + 100), data));
}
waitShorter();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
assertEquals(100, diskStore.getSize());
manager.removeCache(cacheName);
diskStore = createPersistentDiskStore(cacheName);
File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(100, diskStore.getSize());
diskStore.remove("key100");
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(99, diskStore.getSize());
manager.removeCache(cacheName);
diskStore = createPersistentDiskStore(cacheName);
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(99, diskStore.getSize());
diskStore.put(new Element("key100", new byte[1024]));
diskStore.flush();
waitShorter();
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(100, diskStore.getSize());
}
