public void testWriteDuringPartition() throws Exception { DISCARD[] discards = new DISCARD[NUM_NODES]; for (int i = 0; i < NUM_NODES; i++) { discards[i] = TestingUtil.getDiscardForCache(cache(i)); } final List<Future<Object>> futures = new ArrayList<>(NUM_NODES); final ConcurrentMap<String, Integer> insertedKeys = CollectionFactory.makeConcurrentMap(); final AtomicBoolean stop = new AtomicBoolean(false); for (int i = 0; i < NUM_NODES; i++) { final int cacheIndex = i; Future<Object> future = fork( new Callable<Object>() { @Override public Object call() throws Exception { Cache<String, Integer> cache = cache(cacheIndex); int count = 0; while (!stop.get()) { String key = "key" + cacheIndex + "_" + count; try { cache.put(key, count); insertedKeys.put(key, count); } catch (AvailabilityException e) { // expected, ignore } catch (CacheException e) { if (e.getCause() instanceof XAException && e.getCause().getCause() instanceof AvailabilityException) { // expected, ignore } } count++; Thread.sleep(0); } return count; } }); futures.add(future); } long startTime = TIME_SERVICE.time(); int splitIndex = 0; while (splitIndex < NUM_NODES) { List<Address> partitionOne = new ArrayList<Address>(NUM_NODES); List<Address> partitionTwo = new ArrayList<Address>(NUM_NODES); List<EmbeddedCacheManager> partitionOneManagers = new ArrayList<>(); List<EmbeddedCacheManager> partitionTwoManagers = new ArrayList<>(); for (int i = 0; i < NUM_NODES; i++) { if ((i + splitIndex) % NUM_NODES < NUM_NODES / 2) { partitionTwo.add(address(i)); partitionTwoManagers.add(manager(i)); } else { partitionOne.add(address(i)); partitionOneManagers.add(manager(i)); } } assertEquals(NUM_NODES / 2, partitionTwo.size()); log.infof( "Cache is available, splitting cluster at index %d. First partition is %s, second partition is %s", splitIndex, partitionOne, partitionTwo); for (int i = 0; i < NUM_NODES; i++) { if (partitionOne.contains(address(i))) { for (Address a : partitionTwo) { discards[i].addIgnoreMember(((JGroupsAddress) a).getJGroupsAddress()); } } else { for (Address a : partitionOne) { discards[i].addIgnoreMember(((JGroupsAddress) a).getJGroupsAddress()); } } } TestingUtil.blockForMemberToFail(30000, partitionOneManagers.toArray(new CacheContainer[0])); TestingUtil.blockForMemberToFail(30000, partitionTwoManagers.toArray(new CacheContainer[0])); log.infof("Nodes split, waiting for the caches to become degraded"); eventually( new Condition() { @Override public boolean isSatisfied() throws Exception { return TestingUtil.extractComponent(cache(0), PartitionHandlingManager.class) .getAvailabilityMode() == AvailabilityMode.DEGRADED_MODE; } }); assertFuturesRunning(futures); log.infof("Cache is degraded, merging partitions %s and %s", partitionOne, partitionTwo); for (int i = 0; i < NUM_NODES; i++) { discards[i].resetIgnoredMembers(); } TestingUtil.blockUntilViewsReceived( 60000, true, cacheManagers.toArray(new CacheContainer[0])); log.infof("Partitions merged, waiting for the caches to become available"); eventually( new Condition() { @Override public boolean isSatisfied() throws Exception { return TestingUtil.extractComponent(cache(0), PartitionHandlingManager.class) .getAvailabilityMode() == AvailabilityMode.AVAILABLE; } }); TestingUtil.waitForRehashToComplete(caches()); assertFuturesRunning(futures); splitIndex++; } stop.set(true); for (Future<Object> future : futures) { future.get(10, TimeUnit.SECONDS); } for (String key : insertedKeys.keySet()) { for (int i = 0; i < NUM_NODES; i++) { assertEquals( "Failure for key " + key + " on " + cache(i), insertedKeys.get(key), cache(i).get(key)); } } long duration = TIME_SERVICE.timeDuration(startTime, TimeUnit.SECONDS); log.infof("Test finished in %d seconds", duration); }