private void closeConnectionBetween(HazelcastInstance h1, HazelcastInstance h2) {
if (h1 == null || h2 == null) return;
final Node n1 = TestUtil.getNode(h1);
final Node n2 = TestUtil.getNode(h2);
n1.clusterService.removeAddress(n2.address);
n2.clusterService.removeAddress(n1.address);
}
@Test(timeout = 120000)
public void testOneMemberWriteThroughWithIndex() throws Exception {
TestMapStore testMapStore = new TestMapStore(1, 1, 1);
testMapStore.setLoadAllKeys(false);
Config config = newConfig(testMapStore, 0);
TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);
HazelcastInstance h1 = nodeFactory.newHazelcastInstance(config);
testMapStore.insert("1", "value1");
IMap map = h1.getMap("default");
assertEquals(0, map.size());
assertTrue(map.tryLock("1", 1, TimeUnit.SECONDS));
assertEquals("value1", map.get("1"));
map.unlock("1");
assertEquals("value1", map.put("1", "value2"));
assertEquals("value2", map.get("1"));
assertEquals("value2", testMapStore.getStore().get("1"));
assertEquals(1, map.size());
assertTrue(map.evict("1"));
assertEquals(0, map.size());
assertEquals(1, testMapStore.getStore().size());
assertEquals("value2", map.get("1"));
assertEquals(1, map.size());
map.remove("1");
assertEquals(0, map.size());
assertEquals(0, testMapStore.getStore().size());
testMapStore.assertAwait(1);
assertEquals(1, testMapStore.getInitCount());
assertEquals("default", testMapStore.getMapName());
assertEquals(TestUtil.getNode(h1), TestUtil.getNode(testMapStore.getHazelcastInstance()));
}
/*
* Sets up a situation where the member list is out of order on node2. Both
* node2 and node1 think they are masters and both think each other are in
* their clusters.
*/
@Test
public void testOutOfSyncMemberListTwoMasters() throws Exception {
Config c1 = buildConfig(false);
Config c2 = buildConfig(false);
Config c3 = buildConfig(false);
c1.getNetworkConfig().setPort(45701);
c2.getNetworkConfig().setPort(45702);
c3.getNetworkConfig().setPort(45703);
List<String> allMembers = Arrays.asList("127.0.0.1:45701, 127.0.0.1:45702, 127.0.0.1:45703");
c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1);
final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2);
final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3);
final MemberImpl m1 = (MemberImpl) h1.getCluster().getLocalMember();
final MemberImpl m2 = (MemberImpl) h2.getCluster().getLocalMember();
final MemberImpl m3 = (MemberImpl) h3.getCluster().getLocalMember();
// All three nodes join into one cluster
assertEquals(3, h1.getCluster().getMembers().size());
assertEquals(3, h2.getCluster().getMembers().size());
assertEquals(3, h3.getCluster().getMembers().size());
final Node n2 = TestUtil.getNode(h2);
// Simulates node2 getting an out of order member list. That causes node2 to think it's the
// master.
List<MemberInfo> members = new ArrayList<MemberInfo>();
members.add(new MemberInfo(m2.getAddress(), m2.getUuid()));
members.add(new MemberInfo(m3.getAddress(), m3.getUuid()));
members.add(new MemberInfo(m1.getAddress(), m1.getUuid()));
n2.clusterService.updateMembers(members);
n2.setMasterAddress(m2.getAddress());
// Give the cluster some time to figure things out. The merge and heartbeat code should have
// kicked in by this point
Thread.sleep(30 * 1000);
assertEquals(m1, h1.getCluster().getMembers().iterator().next());
assertEquals(m1, h2.getCluster().getMembers().iterator().next());
assertEquals(m1, h3.getCluster().getMembers().iterator().next());
assertEquals(3, h1.getCluster().getMembers().size());
assertEquals(3, h2.getCluster().getMembers().size());
assertEquals(3, h3.getCluster().getMembers().size());
}
/*
* Sets up a situation where node3 removes the master and sets node2 as the
* master but none of the other nodes do. This means that node3 thinks node2
* is master but node2 thinks node1 is master.
*/
@Test
public void testOutOfSyncMemberList() throws Exception {
Config c1 = buildConfig(false);
Config c2 = buildConfig(false);
Config c3 = buildConfig(false);
c1.getNetworkConfig().setPort(35701);
c2.getNetworkConfig().setPort(35702);
c3.getNetworkConfig().setPort(35703);
List<String> allMembers = Arrays.asList("127.0.0.1:35701, 127.0.0.1:35702, 127.0.0.1:35703");
c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1);
final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2);
final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3);
// All three nodes join into one cluster
assertEquals(3, h1.getCluster().getMembers().size());
assertEquals(3, h2.getCluster().getMembers().size());
assertEquals(3, h3.getCluster().getMembers().size());
// This simulates each node reading from the other nodes in the list at regular intervals
// This prevents the heart beat code from timing out
final HazelcastInstance[] instances = new HazelcastInstance[] {h1, h2, h3};
final AtomicBoolean doingWork = new AtomicBoolean(true);
Thread[] workThreads = new Thread[instances.length];
for (int i = 0; i < instances.length; i++) {
final int threadNum = i;
workThreads[threadNum] =
new Thread(
new Runnable() {
public void run() {
while (doingWork.get()) {
final HazelcastInstance hz = instances[threadNum];
Set<Member> members = new HashSet<Member>(hz.getCluster().getMembers());
members.remove(hz.getCluster().getLocalMember());
final Map<Member, Future<String>> futures =
hz.getExecutorService("test").submitToMembers(new PingCallable(), members);
for (Future<String> f : futures.values()) {
try {
f.get();
} catch (MemberLeftException ignored) {
} catch (Exception e) {
e.printStackTrace();
}
}
try {
Thread.sleep(2000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
});
workThreads[threadNum].start();
}
final Node n3 = TestUtil.getNode(h3);
n3.clusterService.removeAddress(((MemberImpl) h1.getCluster().getLocalMember()).getAddress());
// Give the cluster some time to figure things out. The merge and heartbeat code should have
// kicked in by this point
Thread.sleep(30 * 1000);
doingWork.set(false);
for (Thread t : workThreads) {
t.join();
}
assertEquals(3, h1.getCluster().getMembers().size());
assertEquals(3, h2.getCluster().getMembers().size());
assertEquals(3, h3.getCluster().getMembers().size());
}
public static Node getNode(HazelcastInstance hz) {
return TestUtil.getNode(hz);
}
private NearCache getNearCache(String mapName, HazelcastInstance instance) {
NodeEngineImpl nodeEngine = TestUtil.getNode(instance).nodeEngine;
MapService service = nodeEngine.getService(MapService.SERVICE_NAME);
return service.getNearCache(mapName);
}
protected NodeEngine getNodeEngine(HazelcastInstance instance) {
Node node = TestUtil.getNode(instance);
return node.getNodeEngine();
}
protected ICacheService getCacheService(HazelcastInstance instance) {
Node node = TestUtil.getNode(instance);
return node.getNodeEngine().getService(ICacheService.SERVICE_NAME);
}