@Override
public void onEntryRemoved(Object key) {
try {
CacheStore cacheStore = clm.getCacheStore();
if (cacheStore != null) cacheStore.remove(key);
} catch (CacheLoaderException e) {
throw new CacheException(e);
}
}
protected void assertNumberOfEntries(int cacheIndex) throws Exception {
CacheStore cacheStore =
TestingUtil.extractComponent(cache(cacheIndex), CacheLoaderManager.class).getCacheStore();
assertEquals(2, cacheStore.loadAllKeys(null).size()); // two entries in store
DataContainer dataContainer = cache(cacheIndex).getAdvancedCache().getDataContainer();
assertEquals(
1, dataContainer.size(null)); // only one entry in memory (the other one was evicted)
}
public void testClear() throws Exception {
prepareClearTest();
c1.clear();
for (Cache<Object, String> c : caches) assert c.isEmpty();
for (int i = 0; i < 5; i++) {
String key = "k" + i;
for (Cache<Object, String> c : caches) {
CacheStore store =
TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
assert !store.containsKey(key);
}
}
}
@Override
public Object perform(InvocationContext ctx) throws Throwable {
// To avoid reliance of a thread local flag, get a reference for the
// cache store to be able to clear it after cache has stopped.
CacheStore store = cacheLoaderManager.getCacheStore();
cacheManager.getCache(cacheName).stop();
// After stopping the cache, clear it
if (store != null) store.clear();
registry.removeCache(cacheName);
return null;
}
public void testRemoveFromNonOwnerWithFlags() throws Exception {
String key = "k1", value = "value";
initAndTest();
Object retval =
getFirstNonOwner(key).getAdvancedCache().withFlags(Flag.SKIP_CACHE_STORE).remove(key);
if (testRetVals) assert value.equals(retval);
for (Cache<Object, String> c : caches) {
if (isOwner(c, key)) {
CacheStore store =
TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
assert store.containsKey(key);
}
}
}
public void testPutAllWithFlags() throws Exception {
Map<String, String> data = makePutAllTestData();
c1.getAdvancedCache().withFlags(Flag.SKIP_CACHE_STORE).putAll(data);
for (Cache<Object, String> c : caches) {
CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
for (String key : keys) {
assert !store.containsKey(key);
if (isOwner(c, key)) {
assertIsInContainerImmortal(c, key);
}
}
}
}
public void testClearWithFlag() throws Exception {
prepareClearTest();
c1.getAdvancedCache().withFlags(Flag.SKIP_CACHE_STORE).clear();
for (Cache<Object, String> c : caches) {
assert c.isEmpty()
: "Data container " + c + " should be empty, instead it contains keys " + c.keySet();
CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
for (int i = 0; i < 5; i++) {
String key = "k" + i;
if (isOwner(c, key)) {
assert store.containsKey(key);
}
}
}
}
public void testPutFromOwner() throws Exception {
String key = "k3", value = "value3";
for (Cache<Object, String> c : caches) assert c.isEmpty();
getOwners(key)[0].put(key, value);
for (Cache<Object, String> c : caches) {
CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
if (isOwner(c, key)) {
assertIsInContainerImmortal(c, key);
assert store.containsKey(key);
} else {
assertIsNotInL1(c, key);
assert !store.containsKey(key);
}
}
}
public void testPutAll() throws Exception {
c1.putAll(makePutAllTestData());
for (Cache<Object, String> c : caches) {
CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
for (String key : keys) {
if (isOwner(c, key)) {
assertIsInContainerImmortal(c, key);
assert store.containsKey(key);
} else {
assert !store.containsKey(key);
}
}
}
}
public void testAtomicReplaceFromNonOwner() throws Exception {
String key = "k1", value = "value", value2 = "v2";
initAndTest();
boolean replaced = getFirstNonOwner(key).replace(key, value2, value);
assert !replaced;
replaced = getFirstNonOwner(key).replace(key, value, value2);
assert replaced;
for (Cache<Object, String> c : caches) {
c.get(key).equals(value2);
if (isOwner(c, key)) {
CacheStore store =
TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
assert store.containsKey(key);
assert store.load(key).getValue().equals(value2);
}
}
}
public void testPropagatingTwoPhaseCommit() throws Exception {
List<Modification> list = new LinkedList<Modification>();
list.add(new Store(InternalEntryFactory.create("k1", "v1")));
list.add(new Store(InternalEntryFactory.create("k2", "v2", lifespan)));
list.add(new Store(InternalEntryFactory.create("k3", "v3")));
list.add(new Remove("k3"));
list.add(new Clear());
list.add(new Store(InternalEntryFactory.create("k4", "v4")));
list.add(new Store(InternalEntryFactory.create("k5", "v5", lifespan)));
list.add(new Store(InternalEntryFactory.create("k6", "v6")));
list.add(new Remove("k6"));
GlobalTransaction tx = gtf.newGlobalTransaction(null, false);
cs.prepare(list, tx, false);
CacheStore[] allStores = new CacheStore[] {cs, store1, store2}; // for iteration
for (int i = 1; i < 7; i++) {
for (CacheStore s : allStores) assert !s.containsKey("k" + i);
}
cs.commit(tx);
for (int i = 1; i < 7; i++) {
if (i < 4 || i == 6) {
// these have been deleted
for (CacheStore s : allStores) assert !s.containsKey("k" + i);
} else {
for (CacheStore s : allStores) {
assert s.containsKey("k" + i);
assert s.load("k" + i).getValue().equals("v" + i);
assert s.load("k" + i).getLifespan() == (i == 5 ? lifespan : -1);
}
}
}
}
public void testAtomicPutIfAbsentFromNonOwnerWithFlag() throws Exception {
String key = "k1", value = "value";
for (Cache<Object, String> c : caches) assert c.isEmpty();
String replaced =
getFirstNonOwner(key)
.getAdvancedCache()
.withFlags(Flag.SKIP_CACHE_STORE)
.putIfAbsent("k1", value);
assert replaced == null;
// interesting case: fails to put as value exists, put actually missing in Store
replaced = getFirstNonOwner(key).putIfAbsent("k1", value);
assert value.equals(replaced);
for (Cache<Object, String> c : caches) {
c.get(key).equals(replaced);
CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
assert !store.containsKey(key);
}
}
public void testPropagatingStreams()
throws IOException, ClassNotFoundException, CacheLoaderException {
store2.store(InternalEntryFactory.create("k1", "v1"));
store2.store(InternalEntryFactory.create("k2", "v2", lifespan));
assert cs.containsKey("k1");
assert cs.containsKey("k2");
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(byteStream);
cs.toStream(new UnclosableObjectOutputStream(oos));
oos.close();
byteStream.close();
cs.clear();
assert !cs.containsKey("k1");
assert !cs.containsKey("k2");
for (CacheStore s : stores) {
assert !s.containsKey("k1");
assert !s.containsKey("k2");
}
cs.fromStream(
new UnclosableObjectInputStream(
new ObjectInputStream(new ByteArrayInputStream(byteStream.toByteArray()))));
assert cs.containsKey("k1");
assert cs.containsKey("k2");
assert cs.load("k1").getValue().equals("v1");
assert cs.load("k2").getValue().equals("v2");
assert cs.load("k1").getLifespan() == -1;
assert cs.load("k2").getLifespan() == lifespan;
assert store2.containsKey("k1");
assert store2.containsKey("k2");
assert store2.load("k1").getValue().equals("v1");
assert store2.load("k2").getValue().equals("v2");
assert store2.load("k1").getLifespan() == -1;
assert store2.load("k2").getLifespan() == lifespan;
// store 1 has not been set up to support fetching state
assert !store1.containsKey("k1");
assert !store1.containsKey("k2");
}
public void testGetFromNonOwnerWithFlags() throws Exception {
String key = "k2", value = "value2";
for (Cache<Object, String> c : caches) assert c.isEmpty();
Cache<Object, String> nonOwner = getFirstNonOwner(key);
Cache<Object, String> owner = getFirstOwner(key);
CacheStore ownerStore =
TestingUtil.extractComponent(owner, CacheLoaderManager.class).getCacheStore();
owner.put(key, value);
assert value.equals(ownerStore.load(key).getValue());
owner.getAdvancedCache().withFlags(Flag.SKIP_CACHE_STORE).clear();
assert value.equals(ownerStore.load(key).getValue());
assert owner.getAdvancedCache().withFlags(Flag.SKIP_CACHE_STORE).get(key) == null;
assert nonOwner.getAdvancedCache().withFlags(Flag.SKIP_CACHE_STORE).get(key) == null;
assert value.equals(nonOwner.get(key));
// need to do the get() on all the owners first to load the values, otherwise
// assertOwnershipAndNonOwnership might fail
assertOnAllCaches(key, value);
assertOwnershipAndNonOwnership(key, true);
}
@Test(
enabled = false,
description = "Temporary disabled : https://issues.jboss.org/browse/ISPN-2249")
public void testAtomicPutIfAbsentFromNonOwner() throws Exception {
String key = "k1", value = "value", value2 = "v2";
for (Cache<Object, String> c : caches) assert c.isEmpty();
String replaced = getFirstNonOwner(key).putIfAbsent("k1", value);
assert replaced == null;
replaced = getFirstNonOwner(key).putIfAbsent("k1", value2);
assert value.equals(replaced);
for (Cache<Object, String> c : caches) {
c.get(key).equals(replaced);
if (isOwner(c, key)) {
CacheStore store =
TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
assert store.containsKey(key);
assert store.load(key).getValue().equals(value);
}
}
}
public void testReplaceFromNonOwnerWithFlag() throws Exception {
String key = "k1", value = "value", value2 = "v2";
initAndTest();
Object retval =
getFirstNonOwner(key)
.getAdvancedCache()
.withFlags(Flag.SKIP_CACHE_STORE)
.replace(key, value2);
if (testRetVals) assert value.equals(retval);
for (Cache<Object, String> c : caches) {
CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
if (isOwner(c, key)) {
assertIsInContainerImmortal(c, key);
assert c.get(key).equals(value2);
assert store.load(key).getValue().equals(value);
} else {
assert !store.containsKey(key);
}
}
}
private void prepareClearTest() throws CacheLoaderException {
for (Cache<Object, String> c : caches)
assert c.isEmpty()
: "Data container " + c + " should be empty, instead it contains keys " + c.keySet();
for (int i = 0; i < 5; i++) {
getOwners("k" + i)[0].put("k" + i, "value" + i);
}
// this will fill up L1 as well
for (int i = 0; i < 5; i++) assertOnAllCachesAndOwnership("k" + i, "value" + i);
for (Cache<Object, String> c : caches) {
assert !c.isEmpty();
CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
for (int i = 0; i < 5; i++) {
String key = "k" + i;
if (isOwner(c, key)) {
assert store.containsKey(key) : "Cache store " + c + " does not contain key " + key;
}
}
}
}
public void testRemoveFromNonOwner() throws Exception {
String key = "k1", value = "value";
initAndTest();
for (Cache<Object, String> c : caches) {
CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
if (isOwner(c, key)) {
assertIsInContainerImmortal(c, key);
assert store.load(key).getValue().equals(value);
} else {
assert !store.containsKey(key);
}
}
Object retval = getFirstNonOwner(key).remove(key);
if (testRetVals) assert "value".equals(retval);
for (Cache<Object, String> c : caches) {
CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
assert !store.containsKey(key);
}
}
/**
* {@inheritDoc} This implementation iterates through a list of work represented by {@link
* Modification} objects and executes it against the {@link CacheStore}.
*
* <p>Current commands supported are:
*
* <ul>
* <li>STORE
* <li>CLEAR
* <li>REMOVE
* <li>PURGE_EXPIRED
* </ul>
*/
public void doWork() throws Exception {
for (Modification modification : mods)
switch (modification.getType()) {
case STORE:
Store s = (Store) modification;
cs.store(s.getStoredEntry());
break;
case CLEAR:
cs.clear();
break;
case REMOVE:
Remove r = (Remove) modification;
cs.remove(r.getKey());
break;
case PURGE_EXPIRED:
cs.purgeExpired();
break;
default:
throw new IllegalArgumentException("Unknown modification type " + modification.getType());
}
}
public void testPersistence() throws CacheLoaderException, ParseException {
verifyFullTextHasMatches(0);
Country italy = new Country();
italy.countryName = "Italy";
City rome = new City();
rome.name = "Rome";
italy.cities.add(rome);
cache.put("IT", italy);
assert !store.containsKey("IT");
verifyFullTextHasMatches(1);
cache.evict("IT");
assert store.containsKey("IT");
InternalCacheEntry internalCacheEntry = cache.getAdvancedCache().getDataContainer().get("IT");
assert internalCacheEntry == null;
verifyFullTextHasMatches(1);
Country country = cache.get("IT");
assert country != null;
assert "Italy".equals(country.countryName);
verifyFullTextHasMatches(1);
cache.stop();
assert ((SearchFactoryIntegrator) search.getSearchFactory()).isStopped();
TestingUtil.killCacheManagers(cm);
// Now let's check the entry is not re-indexed during data preloading:
recreateCacheManager();
// People should generally use a persistent index; we use RAMDirectory for
// test cleanup, so for our configuration it needs now to contain zero
// matches: on filesystem it would be exactly one as expected (two when ISPN-1179 was open)
verifyFullTextHasMatches(0);
}
@Test
public void testSkip() throws IOException, CacheLoaderException {
final long expected = (DATA_SIZE - 1);
final byte[] data = randomData(DATA_SIZE);
when(mockEntry.getValue()).thenReturn(data);
when(mockStore.load(MOCK_FIRST_CHUNK)).thenReturn(mockEntry);
final long actual = testObj.skip(expected);
assertEquals(expected, actual);
verify(mockStore).load(anyString());
verify(mockStore).load(MOCK_FIRST_CHUNK);
verify(mockEntry).getValue();
assertTrue(testObj.read() > -1);
assertEquals(-1, testObj.read());
}
@Test
public void testAvailable() throws IOException, CacheLoaderException {
final byte[] data = randomData(DATA_SIZE);
when(mockEntry.getValue()).thenReturn(data);
when(mockStore.load(MOCK_FIRST_CHUNK)).thenReturn(mockEntry);
assertEquals(0, testObj.available());
final int partition = 435;
testObj.skip(partition);
// part of the first buffer remains
assertEquals(DATA_SIZE - partition, testObj.available());
testObj.skip(DATA_SIZE - partition);
// none of the first buffer remains
assertEquals(0, testObj.available());
testObj.skip(1);
// no buffers remain
assertEquals(-1, testObj.available());
}
@Test
public void testBufferedRead() throws IOException, CacheLoaderException {
final byte[] data = randomData(DATA_SIZE);
when(mockEntry.getValue()).thenReturn(data);
when(mockStore.load(anyString())).thenReturn(mockEntry).thenReturn(mockEntry).thenReturn(null);
final int partition = 234;
final int expected = (DATA_SIZE - partition);
final byte[] buffer = new byte[DATA_SIZE];
long actual = testObj.read(buffer, 0, expected);
// can read less than a block of data
assertEquals(expected, actual);
// will not load the next chunk if more data is available
actual = testObj.read(buffer, 0, DATA_SIZE);
assertEquals(partition, actual);
actual = testObj.read(buffer, 0, DATA_SIZE);
// will load the next chunk if no data is available
assertEquals(DATA_SIZE, actual);
// and will report the end of the data accurately
actual = testObj.read(buffer, 0, DATA_SIZE);
assertEquals(-1, actual);
}
@Test
public void testSkipMultipleBuffers() throws IOException, CacheLoaderException {
final byte[] data = randomData(DATA_SIZE);
when(mockEntry.getValue()).thenReturn(data);
when(mockStore.load(anyString())).thenReturn(mockEntry).thenReturn(mockEntry).thenReturn(null);
long expected = (DATA_SIZE);
// ask for more than the buffer
long actual = testObj.skip(DATA_SIZE + 1);
// we should skip only one complete buffer
assertEquals(expected, actual);
// ok, skip all but the last byte remaining
expected = (DATA_SIZE - 1);
actual = testObj.skip(expected);
// new buffer, mostly skipped
assertEquals(expected, actual);
// we should still have 1 more byte
assertTrue(testObj.read() > -1);
// but only the one
assertEquals(-1, testObj.read());
// and we only had two cacheEntries
verify(mockEntry, times(2)).getValue();
}
public void testPutFromNonOwner() throws Exception {
String key = "k2", value = "value2";
for (Cache<Object, String> c : caches) assert c.isEmpty();
Cache<Object, String> nonOwner = getFirstNonOwner(key);
Cache<Object, String> owner = getFirstOwner(key);
CacheStore nonOwnerStore =
TestingUtil.extractComponent(nonOwner, CacheLoaderManager.class).getCacheStore();
CacheStore ownerStore =
TestingUtil.extractComponent(owner, CacheLoaderManager.class).getCacheStore();
assert !nonOwnerStore.containsKey(key);
assert !ownerStore.containsKey(key);
Object retval = nonOwner.put(key, value);
assert !nonOwnerStore.containsKey(key);
assert ownerStore.containsKey(key);
if (testRetVals) assert retval == null;
assertOnAllCachesAndOwnership(key, value);
}
public void testPropagatingWrites() throws Exception {
// put something in the store
cs.store(InternalEntryFactory.create("k1", "v1"));
cs.store(InternalEntryFactory.create("k2", "v2", lifespan));
int i = 1;
for (CacheStore s : stores) {
assert s.containsKey("k1") : "Key k1 missing on store " + i;
assert s.containsKey("k2") : "Key k2 missing on store " + i;
assert s.load("k1").getValue().equals("v1");
assert s.load("k2").getValue().equals("v2");
assert s.load("k1").getLifespan() == -1;
assert s.load("k2").getLifespan() == lifespan;
i++;
}
cs.remove("k1");
for (CacheStore s : stores) {
assert !s.containsKey("k1");
assert s.containsKey("k2");
assert s.load("k1") == null;
assert s.load("k2").getValue().equals("v2");
assert s.load("k2").getLifespan() == lifespan;
}
cs.clear();
for (CacheStore s : stores) {
assert !s.containsKey("k1");
assert !s.containsKey("k2");
assert s.load("k1") == null;
assert s.load("k2") == null;
}
cs.store(InternalEntryFactory.create("k1", "v1"));
cs.store(InternalEntryFactory.create("k2", "v2", lifespan));
cs.store(InternalEntryFactory.create("k3", "v3", 1000)); // short lifespan!
for (CacheStore s : stores) {
assert s.containsKey("k1");
assert s.containsKey("k2");
assert s.containsKey("k3");
}
Thread.sleep(1100);
cs.purgeExpired();
for (CacheStore s : stores) {
assert s.containsKey("k1");
assert s.containsKey("k2");
assert !s.containsKey("k3");
}
}
@Before
public void setUp() throws CacheLoaderException {
initMocks(this);
when(mockStore.containsKey(MOCK_FIRST_CHUNK)).thenReturn(true);
testObj = new StoreChunkInputStream(mockStore, MOCK_KEY);
}
private void invalidateSegments(Set<Integer> newSegments, Set<Integer> segmentsToL1) {
// The actual owners keep track of the nodes that hold a key in L1 ("requestors") and
// they invalidate the key on every requestor after a change.
// But this information is only present on the owners where the ClusteredGetKeyValueCommand
// got executed - if the requestor only contacted one owner, and that node is no longer an owner
// (perhaps because it left the cluster), the other owners will not know to invalidate the key
// on that requestor. Furthermore, the requestors list is not copied to the new owners during
// state transfers.
// To compensate for this, we delete all L1 entries in segments that changed ownership during
// this topology update. We can't actually differentiate between L1 entries and regular entries,
// so we delete all entries that don't belong to this node in the current OR previous topology.
Set<Object> keysToL1 = new HashSet<Object>();
Set<Object> keysToRemove = new HashSet<Object>();
// gather all keys from data container that belong to the segments that are being removed/moved
// to L1
for (InternalCacheEntry ice : dataContainer) {
Object key = ice.getKey();
int keySegment = getSegment(key);
if (segmentsToL1.contains(keySegment)) {
keysToL1.add(key);
} else if (!newSegments.contains(keySegment)) {
keysToRemove.add(key);
}
}
// gather all keys from cache store that belong to the segments that are being removed/moved to
// L1
CacheStore cacheStore = getCacheStore();
if (cacheStore != null) {
// todo [anistor] extend CacheStore interface to be able to specify a filter when loading keys
// (ie. keys should belong to desired segments)
try {
Set<Object> storedKeys =
cacheStore.loadAllKeys(new ReadOnlyDataContainerBackedKeySet(dataContainer));
for (Object key : storedKeys) {
int keySegment = getSegment(key);
if (segmentsToL1.contains(keySegment)) {
keysToL1.add(key);
} else if (!newSegments.contains(keySegment)) {
keysToRemove.add(key);
}
}
} catch (CacheLoaderException e) {
log.failedLoadingKeysFromCacheStore(e);
}
}
if (configuration.clustering().l1().onRehash()) {
log.debugf("Moving to L1 state for segments %s of cache %s", segmentsToL1, cacheName);
} else {
log.debugf("Removing state for segments %s of cache %s", segmentsToL1, cacheName);
}
if (!keysToL1.isEmpty()) {
try {
InvalidateCommand invalidateCmd =
commandsFactory.buildInvalidateFromL1Command(
true, EnumSet.of(CACHE_MODE_LOCAL, SKIP_LOCKING), keysToL1);
InvocationContext ctx = icc.createNonTxInvocationContext();
interceptorChain.invoke(ctx, invalidateCmd);
log.debugf(
"Invalidated %d keys, data container now has %d keys",
keysToL1.size(), dataContainer.size());
if (trace) log.tracef("Invalidated keys: %s", keysToL1);
} catch (CacheException e) {
log.failedToInvalidateKeys(e);
}
}
log.debugf(
"Removing L1 state for segments not in %s or %s for cache %s",
newSegments, segmentsToL1, cacheName);
if (!keysToRemove.isEmpty()) {
try {
InvalidateCommand invalidateCmd =
commandsFactory.buildInvalidateFromL1Command(
false, EnumSet.of(CACHE_MODE_LOCAL, SKIP_LOCKING), keysToRemove);
InvocationContext ctx = icc.createNonTxInvocationContext();
interceptorChain.invoke(ctx, invalidateCmd);
log.debugf(
"Invalidated %d keys, data container of cache %s now has %d keys",
keysToRemove.size(), cacheName, dataContainer.size());
if (trace) log.tracef("Invalidated keys: %s", keysToRemove);
} catch (CacheException e) {
log.failedToInvalidateKeys(e);
}
}
// todo [anistor] call CacheNotifier.notifyDataRehashed
}