/** Tests that we can load a store after the index has been corrupted */
@Test
public void testLoadPersistentStoreAfterCorruption() throws IOException, InterruptedException {
// initialise
String cacheName = "testPersistent";
Store diskStore = createPersistentDiskStore(cacheName);
diskStore.removeAll();
for (int i = 0; i < 100; i++) {
byte[] data = new byte[1024];
diskStore.put(new Element("key" + (i + 100), data));
}
waitShorter();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
assertEquals(100, diskStore.getSize());
manager.removeCache(cacheName);
File indexFile = ((DiskPersistentStore) diskStore).getIndexFile();
FileOutputStream fout = new FileOutputStream(indexFile);
// corrupt the index file
fout.write(new byte[] {'q', 'w', 'e', 'r', 't', 'y'});
fout.close();
diskStore = createPersistentDiskStore(cacheName);
File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
assertTrue("File exists", dataFile.exists());
// Make sure the data file got recreated since the index was corrupt
assertEquals("Data file was not recreated", 0, dataFile.length());
assertEquals(0, diskStore.getSize());
}
/** Any disk store with an auto generated random directory should not be able to be loaded. */
@Test
public void testCannotLoadPersistentStoreWithAutoDir() throws IOException, InterruptedException {
// initialise
String cacheName = "testPersistent";
Store diskStore = createAutoPersistentDiskStore(cacheName);
diskStore.removeAll();
for (int i = 0; i < 100; i++) {
byte[] data = new byte[1024];
diskStore.put(new Element("key" + (i + 100), data));
}
waitLonger();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
assertEquals(100, diskStore.getSize());
manager2.removeCache(cacheName);
Thread.sleep(1000);
Cache cache = new Cache(cacheName, 10000, true, false, 5, 1, true, 600);
manager2.addCache(cache);
File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
assertTrue("File exists", dataFile.exists());
assertEquals(0, dataFile.length());
assertEquals(0, cache.getSize());
manager.removeCache(cacheName);
assertTrue("File exists", dataFile.exists());
assertEquals(0, dataFile.length());
}
/** Tests that we can save and load a persistent store in a repeatable way */
@Test
public void testLoadPersistentStore() throws IOException, InterruptedException {
// initialise
String cacheName = "testLoadPersistent";
Store store = createPersistentDiskStore(cacheName);
store.removeAll();
waitShorter();
for (int i = 0; i < 100; i++) {
byte[] data = new byte[1024];
store.put(new Element("key" + (i + 100), data));
}
store.flush();
waitShorter();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, store.getOnDiskSizeInBytes());
assertEquals(100, store.getSize());
manager.removeCache(cacheName);
Thread.sleep(3000);
// check that we can create and dispose several times with no problems and no lost data
for (int i = 0; i < 10; i++) {
store = createPersistentDiskStore(cacheName);
File dataFile = ((DiskPersistentStore) store).getDataFile();
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(100, store.getSize());
manager.removeCache(cacheName);
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
}
}
/**
* Tests that we can save and load a persistent store in a repeatable way, and delete and add
* data.
*/
@Test
public void testFreeSpaceBehaviour() throws IOException, InterruptedException {
// initialise
String cacheName = "testPersistent";
Store diskStore = createPersistentDiskStore(cacheName);
diskStore.removeAll();
byte[] data = new byte[1024];
for (int i = 0; i < 100; i++) {
diskStore.put(new Element("key" + (i + 100), data));
}
waitShorter();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
assertEquals(100, diskStore.getSize());
manager.removeCache(cacheName);
diskStore = createPersistentDiskStore(cacheName);
File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(100, diskStore.getSize());
diskStore.remove("key100");
diskStore.remove("key101");
diskStore.remove("key102");
diskStore.remove("key103");
diskStore.remove("key104");
diskStore.put(new Element("key100", data));
diskStore.put(new Element("key101", data));
waitShorter();
// The file does not shrink.
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(97, diskStore.getSize());
diskStore.put(new Element("key102", data));
diskStore.put(new Element("key103", data));
diskStore.put(new Element("key104", data));
diskStore.put(new Element("key201", data));
diskStore.put(new Element("key202", data));
waitShorter();
assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(102, diskStore.getSize());
manager.removeCache(cacheName);
assertTrue("File exists", dataFile.exists());
assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
}
/**
* Tests that a file is created with the right size after puts, and that the file is deleted on
* disposal
*/
@Test
public void testNonPersistentStore() throws IOException, InterruptedException {
Store diskStore = createNonExpiringDiskStore();
File dataFile = ((OverflowToDiskStore) diskStore).getDataFile();
// 100 + 1 for the in-memory capacity
for (int i = 0; i < 101; i++) {
byte[] data = new byte[1024];
diskStore.put(new Element("key" + (i + 100), data));
}
waitShorter();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
assertEquals(100, diskStore.getOnDiskSize());
assertEquals(101, diskStore.getSize());
diskStore.dispose();
Thread.sleep(1);
assertFalse("File exists", dataFile.exists());
}
/**
* Checks that the expiry thread runs and expires elements which has the effect of preventing the
* disk store from continously growing. Ran for 6 hours through 10000 outer loops. No memory use
* increase. Using a key of "key" + i * outer) you get early slots that cannot be reused. The
* DiskStore actual size therefore starts at 133890 and ends at 616830. There is quite a lot of
* space that cannot be used because of fragmentation. Question? Should an effort be made to
* coalesce fragmented space? Unlikely in production to get contiguous fragments as in the first
* form of this test.
*
* <p>Using a key of Integer.valueOf(i * outer) the size stays constant at 140800.
*
* @throws InterruptedException
*/
@Test
public void testExpiryWithSize() throws InterruptedException {
Store diskStore = createDiskStore();
diskStore.removeAll();
byte[] data = new byte[1024];
for (int outer = 1; outer <= 10; outer++) {
for (int i = 0; i < 101; i++) {
Element element = new Element(Integer.valueOf(i * outer), data);
element.setTimeToLive(1);
diskStore.put(element);
}
waitLonger();
int predictedSize = (ELEMENT_ON_DISK_SIZE + 68) * 100;
long actualSize = diskStore.getOnDiskSizeInBytes();
LOG.info("Predicted Size: " + predictedSize + " Actual Size: " + actualSize);
assertEquals(predictedSize, actualSize);
LOG.info("Memory Use: " + measureMemoryUse());
}
}
/**
* Tests that we can save and load a persistent store in a repeatable way, and delete and add
* data.
*/
@Test
public void testLoadPersistentStoreWithDelete() throws IOException, InterruptedException {
// initialise
String cacheName = "testPersistentWithDelete";
Store diskStore = createPersistentDiskStore(cacheName);
diskStore.removeAll();
for (int i = 0; i < 100; i++) {
byte[] data = new byte[1024];
diskStore.put(new Element("key" + (i + 100), data));
}
waitShorter();
assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
assertEquals(100, diskStore.getSize());
manager.removeCache(cacheName);
diskStore = createPersistentDiskStore(cacheName);
File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(100, diskStore.getSize());
diskStore.remove("key100");
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(99, diskStore.getSize());
manager.removeCache(cacheName);
diskStore = createPersistentDiskStore(cacheName);
assertTrue("File exists", dataFile.exists());
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(99, diskStore.getSize());
diskStore.put(new Element("key100", new byte[1024]));
diskStore.flush();
waitShorter();
assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
assertEquals(100, diskStore.getSize());
}
