Example #1
0
  /** Tests that we can load a store after the index has been corrupted */
  @Test
  public void testLoadPersistentStoreAfterCorruption() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testPersistent";
    Store diskStore = createPersistentDiskStore(cacheName);
    diskStore.removeAll();

    for (int i = 0; i < 100; i++) {
      byte[] data = new byte[1024];
      diskStore.put(new Element("key" + (i + 100), data));
    }
    waitShorter();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
    assertEquals(100, diskStore.getSize());
    manager.removeCache(cacheName);

    File indexFile = ((DiskPersistentStore) diskStore).getIndexFile();
    FileOutputStream fout = new FileOutputStream(indexFile);
    // corrupt the index file
    fout.write(new byte[] {'q', 'w', 'e', 'r', 't', 'y'});
    fout.close();
    diskStore = createPersistentDiskStore(cacheName);
    File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
    assertTrue("File exists", dataFile.exists());

    // Make sure the data file got recreated since the index was corrupt
    assertEquals("Data file was not recreated", 0, dataFile.length());
    assertEquals(0, diskStore.getSize());
  }
Example #2
0
  /** Any disk store with an auto generated random directory should not be able to be loaded. */
  @Test
  public void testCannotLoadPersistentStoreWithAutoDir() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testPersistent";
    Store diskStore = createAutoPersistentDiskStore(cacheName);
    diskStore.removeAll();

    for (int i = 0; i < 100; i++) {
      byte[] data = new byte[1024];
      diskStore.put(new Element("key" + (i + 100), data));
    }
    waitLonger();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
    assertEquals(100, diskStore.getSize());
    manager2.removeCache(cacheName);
    Thread.sleep(1000);

    Cache cache = new Cache(cacheName, 10000, true, false, 5, 1, true, 600);
    manager2.addCache(cache);

    File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
    assertTrue("File exists", dataFile.exists());
    assertEquals(0, dataFile.length());
    assertEquals(0, cache.getSize());
    manager.removeCache(cacheName);
    assertTrue("File exists", dataFile.exists());
    assertEquals(0, dataFile.length());
  }
Example #3
0
  /** Tests that we can save and load a persistent store in a repeatable way */
  @Test
  public void testLoadPersistentStore() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testLoadPersistent";
    Store store = createPersistentDiskStore(cacheName);
    store.removeAll();
    waitShorter();

    for (int i = 0; i < 100; i++) {
      byte[] data = new byte[1024];
      store.put(new Element("key" + (i + 100), data));
    }
    store.flush();
    waitShorter();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, store.getOnDiskSizeInBytes());
    assertEquals(100, store.getSize());
    manager.removeCache(cacheName);
    Thread.sleep(3000);
    // check that we can create and dispose several times with no problems and no lost data
    for (int i = 0; i < 10; i++) {
      store = createPersistentDiskStore(cacheName);
      File dataFile = ((DiskPersistentStore) store).getDataFile();
      assertTrue("File exists", dataFile.exists());
      assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
      assertEquals(100, store.getSize());

      manager.removeCache(cacheName);

      assertTrue("File exists", dataFile.exists());
      assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    }
  }
Example #4
0
  /**
   * Tests that we can save and load a persistent store in a repeatable way, and delete and add
   * data.
   */
  @Test
  public void testFreeSpaceBehaviour() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testPersistent";
    Store diskStore = createPersistentDiskStore(cacheName);
    diskStore.removeAll();

    byte[] data = new byte[1024];
    for (int i = 0; i < 100; i++) {
      diskStore.put(new Element("key" + (i + 100), data));
    }
    waitShorter();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
    assertEquals(100, diskStore.getSize());
    manager.removeCache(cacheName);

    diskStore = createPersistentDiskStore(cacheName);
    File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
    assertTrue("File exists", dataFile.exists());
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(100, diskStore.getSize());

    diskStore.remove("key100");
    diskStore.remove("key101");
    diskStore.remove("key102");
    diskStore.remove("key103");
    diskStore.remove("key104");

    diskStore.put(new Element("key100", data));
    diskStore.put(new Element("key101", data));
    waitShorter();

    // The file does not shrink.
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(97, diskStore.getSize());

    diskStore.put(new Element("key102", data));
    diskStore.put(new Element("key103", data));
    diskStore.put(new Element("key104", data));
    diskStore.put(new Element("key201", data));
    diskStore.put(new Element("key202", data));
    waitShorter();
    assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(102, diskStore.getSize());
    manager.removeCache(cacheName);
    assertTrue("File exists", dataFile.exists());
    assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
  }
Example #5
0
  /**
   * Tests that a file is created with the right size after puts, and that the file is deleted on
   * disposal
   */
  @Test
  public void testNonPersistentStore() throws IOException, InterruptedException {
    Store diskStore = createNonExpiringDiskStore();
    File dataFile = ((OverflowToDiskStore) diskStore).getDataFile();

    // 100 + 1 for the in-memory capacity
    for (int i = 0; i < 101; i++) {
      byte[] data = new byte[1024];
      diskStore.put(new Element("key" + (i + 100), data));
    }
    waitShorter();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());

    assertEquals(100, diskStore.getOnDiskSize());
    assertEquals(101, diskStore.getSize());
    diskStore.dispose();
    Thread.sleep(1);
    assertFalse("File exists", dataFile.exists());
  }
Example #6
0
  /**
   * Checks that the expiry thread runs and expires elements which has the effect of preventing the
   * disk store from continously growing. Ran for 6 hours through 10000 outer loops. No memory use
   * increase. Using a key of "key" + i * outer) you get early slots that cannot be reused. The
   * DiskStore actual size therefore starts at 133890 and ends at 616830. There is quite a lot of
   * space that cannot be used because of fragmentation. Question? Should an effort be made to
   * coalesce fragmented space? Unlikely in production to get contiguous fragments as in the first
   * form of this test.
   *
   * <p>Using a key of Integer.valueOf(i * outer) the size stays constant at 140800.
   *
   * @throws InterruptedException
   */
  @Test
  public void testExpiryWithSize() throws InterruptedException {
    Store diskStore = createDiskStore();
    diskStore.removeAll();

    byte[] data = new byte[1024];
    for (int outer = 1; outer <= 10; outer++) {
      for (int i = 0; i < 101; i++) {
        Element element = new Element(Integer.valueOf(i * outer), data);
        element.setTimeToLive(1);
        diskStore.put(element);
      }
      waitLonger();
      int predictedSize = (ELEMENT_ON_DISK_SIZE + 68) * 100;
      long actualSize = diskStore.getOnDiskSizeInBytes();
      LOG.info("Predicted Size: " + predictedSize + " Actual Size: " + actualSize);
      assertEquals(predictedSize, actualSize);
      LOG.info("Memory Use: " + measureMemoryUse());
    }
  }
Example #7
0
  /**
   * Tests that we can save and load a persistent store in a repeatable way, and delete and add
   * data.
   */
  @Test
  public void testLoadPersistentStoreWithDelete() throws IOException, InterruptedException {
    // initialise
    String cacheName = "testPersistentWithDelete";
    Store diskStore = createPersistentDiskStore(cacheName);
    diskStore.removeAll();

    for (int i = 0; i < 100; i++) {
      byte[] data = new byte[1024];
      diskStore.put(new Element("key" + (i + 100), data));
    }
    waitShorter();
    assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
    assertEquals(100, diskStore.getSize());
    manager.removeCache(cacheName);

    diskStore = createPersistentDiskStore(cacheName);
    File dataFile = ((DiskPersistentStore) diskStore).getDataFile();
    assertTrue("File exists", dataFile.exists());
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(100, diskStore.getSize());

    diskStore.remove("key100");
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(99, diskStore.getSize());

    manager.removeCache(cacheName);

    diskStore = createPersistentDiskStore(cacheName);
    assertTrue("File exists", dataFile.exists());
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(99, diskStore.getSize());

    diskStore.put(new Element("key100", new byte[1024]));
    diskStore.flush();
    waitShorter();
    assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
    assertEquals(100, diskStore.getSize());
  }