Esempio n. 1
0
  /** Tests that we can load a store after the index has been corrupted */
  @Test
  public void testLoadPersistentStoreAfterCorruption() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testPersistent";
    Store diskStore = createPersistentDiskStore(cacheName);
    diskStore.removeAll();

    for (int i = 0; i < 100; i++) {
      byte[] data = new byte[1024];
      diskStore.put(new Element("key" + (i + 100), data));
    }
    waitShorter();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
    assertEquals(100, diskStore.getSize());
    manager.removeCache(cacheName);

    File indexFile = ((DiskPersistentStore) diskStore).getIndexFile();
    FileOutputStream fout = new FileOutputStream(indexFile);
    // corrupt the index file
    fout.write(new byte[] {'q', 'w', 'e', 'r', 't', 'y'});
    fout.close();
    diskStore = createPersistentDiskStore(cacheName);
    File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
    assertTrue("File exists", dataFile.exists());

    // Make sure the data file got recreated since the index was corrupt
    assertEquals("Data file was not recreated", 0, dataFile.length());
    assertEquals(0, diskStore.getSize());
  }
Esempio n. 2
0
  /** Any disk store with an auto generated random directory should not be able to be loaded. */
  @Test
  public void testCannotLoadPersistentStoreWithAutoDir() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testPersistent";
    Store diskStore = createAutoPersistentDiskStore(cacheName);
    diskStore.removeAll();

    for (int i = 0; i < 100; i++) {
      byte[] data = new byte[1024];
      diskStore.put(new Element("key" + (i + 100), data));
    }
    waitLonger();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
    assertEquals(100, diskStore.getSize());
    manager2.removeCache(cacheName);
    Thread.sleep(1000);

    Cache cache = new Cache(cacheName, 10000, true, false, 5, 1, true, 600);
    manager2.addCache(cache);

    File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
    assertTrue("File exists", dataFile.exists());
    assertEquals(0, dataFile.length());
    assertEquals(0, cache.getSize());
    manager.removeCache(cacheName);
    assertTrue("File exists", dataFile.exists());
    assertEquals(0, dataFile.length());
  }
Esempio n. 3
0
  /** Tests that we can save and load a persistent store in a repeatable way */
  @Test
  public void testLoadPersistentStore() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testLoadPersistent";
    Store store = createPersistentDiskStore(cacheName);
    store.removeAll();
    waitShorter();

    for (int i = 0; i < 100; i++) {
      byte[] data = new byte[1024];
      store.put(new Element("key" + (i + 100), data));
    }
    store.flush();
    waitShorter();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, store.getOnDiskSizeInBytes());
    assertEquals(100, store.getSize());
    manager.removeCache(cacheName);
    Thread.sleep(3000);
    // check that we can create and dispose several times with no problems and no lost data
    for (int i = 0; i < 10; i++) {
      store = createPersistentDiskStore(cacheName);
      File dataFile = ((DiskPersistentStore) store).getDataFile();
      assertTrue("File exists", dataFile.exists());
      assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
      assertEquals(100, store.getSize());

      manager.removeCache(cacheName);

      assertTrue("File exists", dataFile.exists());
      assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    }
  }
Esempio n. 4
0
  /** This method tries to get the store too leak. */
  protected long thrashCache() throws Exception {

    long startingSize = measureMemoryUse();
    LOG.info("Starting memory used is: " + startingSize);

    final String value = "value";
    final String key = "key";

    // Add the entry
    final Element element = new Element(key, value);
    store.put(element);

    // Create 15 threads that read the keys;
    final List executables = new ArrayList();
    for (int i = 0; i < 15; i++) {
      final LruMemoryStoreTest.Executable executable =
          new MemoryStoreTester.Executable() {
            public void execute() throws Exception {
              for (int i = 0; i < 500; i++) {
                final String key = "key" + i;
                store.get(key);
              }
              store.get("key");
            }
          };
      executables.add(executable);
    }
    // Create 15 threads that are insert 500 keys with large byte[] as values
    for (int i = 0; i < 15; i++) {
      final MemoryStoreTester.Executable executable =
          new MemoryStoreTester.Executable() {
            public void execute() throws Exception {

              // Add a bunch of entries
              for (int i = 0; i < 500; i++) {
                // Use a random length value
                final String key = "key" + i;
                byte[] value = new byte[10000];
                Element element = new Element(key, value);
                store.put(element);
              }
            }
          };
      executables.add(executable);
    }

    runThreads(executables);
    store.removeAll();

    long finishingSize = measureMemoryUse();
    LOG.info("Ending memory used is: " + finishingSize);
    return finishingSize - startingSize;
  }
Esempio n. 5
0
  /**
   * Tests that we can save and load a persistent store in a repeatable way, and delete and add
   * data.
   */
  @Test
  public void testFreeSpaceBehaviour() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testPersistent";
    Store diskStore = createPersistentDiskStore(cacheName);
    diskStore.removeAll();

    byte[] data = new byte[1024];
    for (int i = 0; i < 100; i++) {
      diskStore.put(new Element("key" + (i + 100), data));
    }
    waitShorter();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
    assertEquals(100, diskStore.getSize());
    manager.removeCache(cacheName);

    diskStore = createPersistentDiskStore(cacheName);
    File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
    assertTrue("File exists", dataFile.exists());
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(100, diskStore.getSize());

    diskStore.remove("key100");
    diskStore.remove("key101");
    diskStore.remove("key102");
    diskStore.remove("key103");
    diskStore.remove("key104");

    diskStore.put(new Element("key100", data));
    diskStore.put(new Element("key101", data));
    waitShorter();

    // The file does not shrink.
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(97, diskStore.getSize());

    diskStore.put(new Element("key102", data));
    diskStore.put(new Element("key103", data));
    diskStore.put(new Element("key104", data));
    diskStore.put(new Element("key201", data));
    diskStore.put(new Element("key202", data));
    waitShorter();
    assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(102, diskStore.getSize());
    manager.removeCache(cacheName);
    assertTrue("File exists", dataFile.exists());
    assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
  }
Esempio n. 6
0
  /**
   * Tests that a file is created with the right size after puts, and that the file is not deleted
   * on disposal
   *
   * <p>This test uses a preconfigured cache from the test cache.xml. Note that teardown causes an
   * exception because the disk store is being shut down twice.
   */
  @Test
  public void testPersistentStore() throws IOException, InterruptedException, CacheException {
    // initialise
    Store store = createPersistentDiskStoreFromCacheManager();
    store.removeAll();

    File dataFile = ((DiskPersistentStore) store).getDataFile();

    for (int i = 0; i < 100; i++) {
      byte[] data = new byte[1024];
      store.put(new Element("key" + (i + 100), data));
    }
    waitShorter();
    assertEquals(100, store.getSize());
    store.dispose();

    assertTrue("File exists", dataFile.exists());
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
  }
Esempio n. 7
0
  /**
   * Checks that the expiry thread runs and expires elements which has the effect of preventing the
   * disk store from continously growing. Ran for 6 hours through 10000 outer loops. No memory use
   * increase. Using a key of "key" + i * outer) you get early slots that cannot be reused. The
   * DiskStore actual size therefore starts at 133890 and ends at 616830. There is quite a lot of
   * space that cannot be used because of fragmentation. Question? Should an effort be made to
   * coalesce fragmented space? Unlikely in production to get contiguous fragments as in the first
   * form of this test.
   *
   * <p>Using a key of Integer.valueOf(i * outer) the size stays constant at 140800.
   *
   * @throws InterruptedException
   */
  @Test
  public void testExpiryWithSize() throws InterruptedException {
    Store diskStore = createDiskStore();
    diskStore.removeAll();

    byte[] data = new byte[1024];
    for (int outer = 1; outer <= 10; outer++) {
      for (int i = 0; i < 101; i++) {
        Element element = new Element(Integer.valueOf(i * outer), data);
        element.setTimeToLive(1);
        diskStore.put(element);
      }
      waitLonger();
      int predictedSize = (ELEMENT_ON_DISK_SIZE + 68) * 100;
      long actualSize = diskStore.getOnDiskSizeInBytes();
      LOG.info("Predicted Size: " + predictedSize + " Actual Size: " + actualSize);
      assertEquals(predictedSize, actualSize);
      LOG.info("Memory Use: " + measureMemoryUse());
    }
  }
Esempio n. 8
0
  /** Tests removing all the entries. */
  @Test
  public void testRemoveAll() throws Exception {
    final String value = "value";
    final String key = "key";

    // Add the entry
    Element element = new Element(key, value);
    store.put(element);

    // Check the entry is there
    element = store.get(key);
    assertNotNull(element);

    // Remove it
    store.removeAll();

    // Check the entry is not there
    assertEquals(0, store.getSize());
    element = store.get(key);
    assertNull(element);
  }
Esempio n. 9
0
  /** Tests removing all the entries. */
  @Test
  public void testRemoveAll() throws Exception {
    final Store diskStore = createDiskStore();

    // Add the entry
    final String value = "value";
    diskStore.put(new Element("key1", value));
    diskStore.put(new Element("key2", value));

    // Check the entry is there
    assertNotNull(diskStore.get("key1"));
    assertNotNull(diskStore.get("key2"));

    // Remove it
    diskStore.removeAll();

    // Check the entry is not there
    assertEquals(0, diskStore.getSize());
    assertEquals(0, diskStore.getOnDiskSize());
    assertNull(diskStore.get("key1"));
    assertNull(diskStore.get("key2"));
  }
Esempio n. 10
0
  /**
   * Tests that we can save and load a persistent store in a repeatable way, and delete and add
   * data.
   */
  @Test
  public void testLoadPersistentStoreWithDelete() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testPersistentWithDelete";
    Store diskStore = createPersistentDiskStore(cacheName);
    diskStore.removeAll();

    for (int i = 0; i < 100; i++) {
      byte[] data = new byte[1024];
      diskStore.put(new Element("key" + (i + 100), data));
    }
    waitShorter();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
    assertEquals(100, diskStore.getSize());
    manager.removeCache(cacheName);

    diskStore = createPersistentDiskStore(cacheName);
    File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
    assertTrue("File exists", dataFile.exists());
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(100, diskStore.getSize());

    diskStore.remove("key100");
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(99, diskStore.getSize());

    manager.removeCache(cacheName);

    diskStore = createPersistentDiskStore(cacheName);
    assertTrue("File exists", dataFile.exists());
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(99, diskStore.getSize());

    diskStore.put(new Element("key100", new byte[1024]));
    diskStore.flush();
    waitShorter();
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(100, diskStore.getSize());
  }