Exemple #1
0
  @Test
  public void testWaitingIndefinitely() throws InterruptedException {
    TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(5);
    final Config config = new Config();
    config.setProperty(GroupProperties.PROP_OPERATION_CALL_TIMEOUT_MILLIS, "2000");
    final HazelcastInstance[] instances = factory.newInstances(config);

    instances[0].getLock("testWaitingIndefinitely").lock();

    final CountDownLatch latch = new CountDownLatch(1);
    new Thread() {
      public void run() {
        try {
          // because max timeout=2000 we get timeout exception which we should not
          instances[1].getLock("testWaitingIndefinitely").lock();
          latch.countDown();
        } catch (Exception ignored) {
        }
      }
    }.start();

    // wait for enough time which is greater than max-timeout (2000)
    Thread.sleep(10000);

    instances[0].getLock("testWaitingIndefinitely").unlock();

    assertTrue(latch.await(5, TimeUnit.SECONDS));
  }
 @Test(timeout = 120000)
 public void testIssue583MapReplaceShouldTriggerMapStore() {
   final ConcurrentMap<String, Long> store = new ConcurrentHashMap<String, Long>();
   final MapStore<String, Long> myMapStore = new SimpleMapStore<String, Long>(store);
   Config config = getConfig();
   config
       .getMapConfig("myMap")
       .setMapStoreConfig(new MapStoreConfig().setImplementation(myMapStore));
   TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);
   HazelcastInstance hc = nodeFactory.newHazelcastInstance(config);
   IMap<String, Long> myMap = hc.getMap("myMap");
   myMap.put("one", 1L);
   assertEquals(1L, myMap.get("one").longValue());
   assertEquals(1L, store.get("one").longValue());
   myMap.putIfAbsent("two", 2L);
   assertEquals(2L, myMap.get("two").longValue());
   assertEquals(2L, store.get("two").longValue());
   myMap.putIfAbsent("one", 5L);
   assertEquals(1L, myMap.get("one").longValue());
   assertEquals(1L, store.get("one").longValue());
   myMap.replace("one", 1L, 111L);
   assertEquals(111L, myMap.get("one").longValue());
   assertEquals(111L, store.get("one").longValue());
   myMap.replace("one", 1L);
   assertEquals(1L, myMap.get("one").longValue());
   assertEquals(1L, store.get("one").longValue());
 }
  @Test(timeout = 120000)
  public void testIssue1142ExceptionWhenLoadAllReturnsNull() {
    Config config = getConfig();
    String mapname = "testIssue1142ExceptionWhenLoadAllReturnsNull";
    MapStoreConfig mapStoreConfig = new MapStoreConfig();
    mapStoreConfig.setImplementation(
        new MapStoreAdapter<String, String>() {
          @Override
          public Set<String> loadAllKeys() {
            Set keys = new HashSet();
            keys.add("key");
            return keys;
          }

          public Map loadAll(Collection keys) {
            return null;
          }
        });
    config.getMapConfig(mapname).setMapStoreConfig(mapStoreConfig);
    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(2);
    HazelcastInstance instance = nodeFactory.newHazelcastInstance(config);
    HazelcastInstance instance2 = nodeFactory.newHazelcastInstance(config);
    final IMap map = instance.getMap(mapname);
    for (int i = 0; i < 300; i++) {
      map.put(i, i);
    }
    assertEquals(300, map.size());
  }
  @Test
  public void testSubmitToAllMembersCallable() throws InterruptedException {
    final int k = simpleTestNodeCount;
    TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(k);
    final HazelcastInstance[] instances = factory.newInstances(new Config());
    final AtomicInteger count = new AtomicInteger(0);
    final CountDownLatch countDownLatch = new CountDownLatch(k * k);
    final MultiExecutionCallback callback =
        new MultiExecutionCallback() {
          public void onResponse(Member member, Object value) {
            count.incrementAndGet();
            countDownLatch.countDown();
          }

          public void onComplete(Map<Member, Object> values) {}
        };
    for (int i = 0; i < k; i++) {
      final IExecutorService service =
          instances[i].getExecutorService("testSubmitToAllMembersCallable");
      final String script =
          "hazelcast.getAtomicLong('testSubmitToAllMembersCallable').incrementAndGet();";
      service.submitToAllMembers(new ScriptCallable(script, null), callback);
    }
    countDownLatch.await(30, TimeUnit.SECONDS);
    final IAtomicLong result = instances[0].getAtomicLong("testSubmitToAllMembersCallable");
    assertEquals(k * k, result.get());
    assertEquals(k * k, count.get());
  }
  @Test
  public void testRemovedEntry_shouldNotBeReached_afterMigration() throws Exception {
    String mapName = randomMapName();
    TestHazelcastInstanceFactory factory = new TestHazelcastInstanceFactory(2);
    MapStoreTest.SimpleMapStore<Integer, Integer> store =
        new MapStoreTest.SimpleMapStore<Integer, Integer>();
    store.store.put(1, 0);

    Config config = createConfig(mapName, store);
    HazelcastInstance node1 = factory.newHazelcastInstance(config);

    IMap<Integer, Integer> map = node1.getMap(mapName);

    map.put(1, 1);
    map.delete(1);

    HazelcastInstance node2 = factory.newHazelcastInstance(config);

    map = node2.getMap(mapName);

    Integer value = map.get(1);
    factory.shutdownAll();

    assertNull(value);
  }
  @Test(timeout = 30000)
  public void testNullFromObjectCombiner() throws Exception {

    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

    HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
    HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
    HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

    IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
    for (int i = 0; i < 100; i++) {
      m1.put(i, i);
    }

    JobTracker jobTracker = h1.getJobTracker("default");
    Job<Integer, Integer> job = jobTracker.newJob(KeyValueSource.fromMap(m1));
    JobCompletableFuture<Map<String, BigInteger>> future =
        job.chunkSize(1)
            .mapper(new GroupingTestMapper())
            .combiner(new ObjectCombinerFactory())
            .reducer(new ObjectReducerFactory())
            .submit();

    int[] expectedResults = new int[4];
    for (int i = 0; i < 100; i++) {
      int index = i % 4;
      expectedResults[index] += i;
    }

    Map<String, BigInteger> map = future.get();
    for (int i = 0; i < 4; i++) {
      assertEquals(BigInteger.valueOf(expectedResults[i]), map.get(String.valueOf(i)));
    }
  }
  @Test
  public void testSubmitToMemberRunnable() throws InterruptedException {
    final int k = simpleTestNodeCount;
    TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(k);
    final HazelcastInstance[] instances = factory.newInstances(new Config());
    final AtomicInteger count = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(k);
    final ExecutionCallback callback =
        new ExecutionCallback() {
          public void onResponse(Object response) {
            if (response == null) {
              count.incrementAndGet();
            }
            latch.countDown();
          }

          public void onFailure(Throwable t) {}
        };
    for (int i = 0; i < k; i++) {
      final HazelcastInstance instance = instances[i];
      final IExecutorService service = instance.getExecutorService("testSubmitToMemberRunnable");
      final String script =
          "if(!hazelcast.getCluster().getLocalMember().equals(member)) "
              + "hazelcast.getAtomicLong('testSubmitToMemberRunnable').incrementAndGet();";
      final HashMap map = new HashMap();
      map.put("member", instance.getCluster().getLocalMember());
      service.submitToMember(
          new ScriptRunnable(script, map), instance.getCluster().getLocalMember(), callback);
    }
    latch.await(10, TimeUnit.SECONDS);
    assertEquals(0, instances[0].getAtomicLong("testSubmitToMemberRunnable").get());
    assertEquals(k, count.get());
  }
 @Test
 public void testEmptyMapIsEmpty() throws Exception {
   TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(1);
   HazelcastInstance instance = nodeFactory.newHazelcastInstance();
   ReplicatedMap<Integer, Integer> map = instance.getReplicatedMap(randomName());
   assertTrue("map should be empty", map.isEmpty());
 }
  private void testSize(Config config) throws Exception {
    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(2);

    HazelcastInstance instance1 = nodeFactory.newHazelcastInstance(config);
    HazelcastInstance instance2 = nodeFactory.newHazelcastInstance(config);

    final ReplicatedMap<Integer, Integer> map1 = instance1.getReplicatedMap("default");
    final ReplicatedMap<Integer, Integer> map2 = instance2.getReplicatedMap("default");

    final AbstractMap.SimpleEntry<Integer, Integer>[] testValues = buildTestValues();

    WatchedOperationExecutor executor = new WatchedOperationExecutor();
    executor.execute(
        new Runnable() {
          @Override
          public void run() {
            int half = testValues.length / 2;
            for (int i = 0; i < testValues.length; i++) {
              final ReplicatedMap<Integer, Integer> map = i < half ? map1 : map2;
              final AbstractMap.SimpleEntry<Integer, Integer> entry = testValues[i];
              map.put(entry.getKey(), entry.getValue());
            }
          }
        },
        60,
        EntryEventType.ADDED,
        100,
        1,
        map1,
        map2);

    assertMatchSuccessfulOperationQuota(1, map1.size(), map2.size());
  }
 @Test(expected = java.lang.IllegalArgumentException.class)
 public void removeNullListener() throws Exception {
   TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(1);
   HazelcastInstance instance1 = nodeFactory.newHazelcastInstance();
   ReplicatedMap<Object, Object> map1 = instance1.getReplicatedMap("default");
   map1.removeEntryListener(null);
 }
  private void testPutAll(Config config) throws TimeoutException {
    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(2);
    HazelcastInstance instance1 = nodeFactory.newHazelcastInstance(config);
    HazelcastInstance instance2 = nodeFactory.newHazelcastInstance(config);
    final ReplicatedMap<String, String> map1 = instance1.getReplicatedMap("default");
    final ReplicatedMap<String, String> map2 = instance2.getReplicatedMap("default");

    final Map<String, String> mapTest = new HashMap<String, String>();
    for (int i = 0; i < 100; i++) {
      mapTest.put("foo-" + i, "bar");
    }

    WatchedOperationExecutor executor = new WatchedOperationExecutor();
    executor.execute(
        new Runnable() {
          @Override
          public void run() {
            map1.putAll(mapTest);
          }
        },
        60,
        EntryEventType.ADDED,
        map1,
        map2);

    for (Entry<String, String> entry : map2.entrySet()) {
      assertStartsWith("foo-", entry.getKey());
      assertEquals("bar", entry.getValue());
    }
    for (Entry<String, String> entry : map1.entrySet()) {
      assertStartsWith("foo-", entry.getKey());
      assertEquals("bar", entry.getValue());
    }
  }
 @Test
 public void removeEmptyListener() throws Exception {
   TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(1);
   HazelcastInstance instance1 = nodeFactory.newHazelcastInstance();
   ReplicatedMap<Object, Object> map1 = instance1.getReplicatedMap("default");
   assertFalse(map1.removeEntryListener("2"));
 }
  private void testAddEntryListener(Config config) throws TimeoutException {
    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(2);

    HazelcastInstance instance1 = nodeFactory.newHazelcastInstance(config);
    HazelcastInstance instance2 = nodeFactory.newHazelcastInstance(config);

    final ReplicatedMap<String, String> map1 = instance1.getReplicatedMap("default");
    final ReplicatedMap<String, String> map2 = instance2.getReplicatedMap("default");

    SimpleEntryListener listener = new SimpleEntryListener(1, 0);
    map2.addEntryListener(listener, "foo-18");

    final int operations = 100;
    WatchedOperationExecutor executor = new WatchedOperationExecutor();
    executor.execute(
        new Runnable() {
          @Override
          public void run() {
            for (int i = 0; i < operations; i++) {
              map1.put("foo-" + i, "bar");
            }
          }
        },
        60,
        EntryEventType.ADDED,
        operations,
        1,
        map1,
        map2);

    assertOpenEventually(listener.addLatch);
  }
  @Test(timeout = 60000)
  public void testMapReduceWithCustomKeyValueSource() throws Exception {

    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

    final HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
    final HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
    final HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);

    JobTracker jobTracker = h1.getJobTracker("default");
    Job<String, Integer> job = jobTracker.newJob(new CustomKeyValueSource());
    ICompletableFuture<Map<String, Integer>> completableFuture =
        job.chunkSize(10)
            .mapper(new CustomMapper())
            .combiner(new CustomCombinerFactory())
            .reducer(new CustomReducerFactory())
            .submit();

    Map<String, Integer> result = completableFuture.get();

    assertEquals(1000, result.size());

    List<Map.Entry<String, Integer>> entrySet = new ArrayList(result.entrySet());
    Collections.sort(entrySet, ENTRYSET_COMPARATOR);

    int count = 0;
    for (Map.Entry<String, Integer> entry : entrySet) {
      assertEquals(String.valueOf(count), entry.getKey());
      assertEquals(count++ * 6, (int) entry.getValue());
    }
  }
  @Test(timeout = 30000)
  public void testMapperReducerCollator() throws Exception {

    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

    HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
    HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
    HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

    IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
    for (int i = 0; i < 100; i++) {
      m1.put(i, i);
    }

    JobTracker tracker = h1.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(KeyValueSource.fromMap(m1));
    ICompletableFuture<Integer> future =
        job.mapper(new GroupingTestMapper())
            .reducer(new TestReducerFactory())
            .submit(new TestCollator());

    int result = future.get();

    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
      expectedResult += i;
    }

    for (int i = 0; i < 4; i++) {
      assertEquals(expectedResult, result);
    }
  }
 @Test(expected = IllegalArgumentException.class)
 public void testNegativeTtlThrowsException() throws Exception {
   TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(1);
   HazelcastInstance instance = nodeFactory.newHazelcastInstance();
   ReplicatedMap<Integer, Integer> map = instance.getReplicatedMap(randomName());
   map.put(1, 1, -1, TimeUnit.DAYS);
 }
  @Test(timeout = 30000)
  public void testPartitionPostpone() throws Exception {

    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

    final HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
    final HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
    final HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

    assertClusterSizeEventually(3, h1);

    IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
    for (int i = 0; i < 100; i++) {
      m1.put(i, i);
    }

    JobTracker tracker = h1.getJobTracker("default");
    KeyValueSource<Integer, Integer> kvs = KeyValueSource.fromMap(m1);
    KeyValueSource<Integer, Integer> wrapper = new MapKeyValueSourceAdapter<Integer, Integer>(kvs);
    Job<Integer, Integer> job = tracker.newJob(wrapper);
    ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new TestMapper()).submit();

    Map<String, List<Integer>> result = future.get();

    assertEquals(100, result.size());
    for (List<Integer> value : result.values()) {
      assertEquals(1, value.size());
    }
  }
  private void testHitsAreZeroInitiallyWithSingleNode(Config config) throws Exception {
    final TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(1);
    final HazelcastInstance instance1 = nodeFactory.newHazelcastInstance(config);

    final ReplicatedMap<Integer, Integer> map = instance1.getReplicatedMap(randomMapName());
    final int operations = 100;
    execute(
        new Runnable() {
          @Override
          public void run() {
            for (int i = 0; i < operations; i++) {
              map.put(i, i);
            }
          }
        },
        ADDED,
        operations,
        1,
        map);

    for (int i = 0; i < operations; i++) {
      final ReplicatedRecord<Integer, Integer> replicatedRecord = getReplicatedRecord(map, i);
      assertEquals(0, replicatedRecord.getHits());
    }
  }
  @Test(timeout = 30000)
  public void testDataSerializableIntermediateObject() throws Exception {

    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

    HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
    HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
    HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

    IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
    for (int i = 0; i < 100; i++) {
      m1.put(i, i);
    }

    JobTracker jobTracker = h1.getJobTracker("default");
    Job<Integer, Integer> job = jobTracker.newJob(KeyValueSource.fromMap(m1));
    ICompletableFuture<Integer> future =
        job.mapper(new TestMapper())
            .combiner(new DataSerializableIntermediateCombinerFactory())
            .reducer(new DataSerializableIntermediateReducerFactory())
            .submit(new DataSerializableIntermediateCollator());

    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
      expectedResult += i;
    }
    expectedResult = (int) ((double) expectedResult / 100);

    assertEquals(expectedResult, (int) future.get());
  }
  @Test
  public void testWaitingIndefinitely() throws InterruptedException {
    final Config config = new Config();
    config.setProperty(GroupProperties.PROP_OPERATION_CALL_TIMEOUT_MILLIS, "3000");

    TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(2);
    final HazelcastInstance[] instances = factory.newInstances(config);

    // need to warm-up partitions,
    // since waiting for lock backup can take up to 5 seconds
    // and that may cause OperationTimeoutException with "No response for 4000 ms" error.
    warmUpPartitions(instances);

    final String name = "testWaitingIndefinitely";
    ILock lock = instances[0].getLock(name);
    lock.lock();

    final CountDownLatch latch = new CountDownLatch(1);
    new Thread() {
      public void run() {
        try {
          // because max timeout=3000 we get timeout exception which we should not
          instances[1].getLock(name).lock();
          latch.countDown();
        } catch (Exception ignored) {
        }
      }
    }.start();

    // wait for enough time which is greater than max-timeout (3000)
    sleepSeconds(10);
    lock.unlock();

    assertTrue(latch.await(20, TimeUnit.SECONDS));
  }
  @Test
  public void testSubmitToKeyOwnerCallable() throws Exception {
    final int k = simpleTestNodeCount;
    TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(k);
    final HazelcastInstance[] instances = factory.newInstances(new Config());
    final AtomicInteger count = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(k / 2);
    final ExecutionCallback callback =
        new ExecutionCallback() {
          public void onResponse(Object response) {
            if ((Boolean) response) count.incrementAndGet();
            latch.countDown();
          }

          public void onFailure(Throwable t) {}
        };
    for (int i = 0; i < k; i++) {
      final HazelcastInstance instance = instances[i];
      final IExecutorService service = instance.getExecutorService("testSubmitToKeyOwnerCallable");
      final String script = "hazelcast.getCluster().getLocalMember().equals(member)";
      final HashMap map = new HashMap();
      final Member localMember = instance.getCluster().getLocalMember();
      map.put("member", localMember);
      int key = 0;
      while (!localMember.equals(instance.getPartitionService().getPartition(++key).getOwner())) ;
      if (i % 2 == 0) {
        final Future f = service.submitToKeyOwner(new ScriptCallable(script, map), key);
        assertTrue((Boolean) f.get(5, TimeUnit.SECONDS));
      } else {
        service.submitToKeyOwner(new ScriptCallable(script, map), key, callback);
      }
    }
    assertTrue(latch.await(30, TimeUnit.SECONDS));
    assertEquals(k / 2, count.get());
  }
  @Test
  public void test_rollingRestart() {
    final TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory();
    final int nodeCount = 3;
    final HazelcastInstance[] instances = new HazelcastInstance[nodeCount];
    instances[0] = factory.newHazelcastInstance();

    if (partitionAssignmentType == PartitionAssignmentType.DURING_STARTUP) {
      warmUpPartitions(instances[0]);
    }

    for (int i = 1; i < nodeCount; i++) {
      instances[i] = factory.newHazelcastInstance();
    }

    if (partitionAssignmentType == PartitionAssignmentType.AT_THE_END) {
      warmUpPartitions(instances);
    }

    changeClusterStateEventually(instances[0], clusterState);

    Address address = getNode(instances[0]).getThisAddress();
    instances[0].shutdown();
    instances[0] = factory.newHazelcastInstance(address);

    for (HazelcastInstance instance : instances) {
      assertClusterSizeEventually(nodeCount, instance);
      assertEquals(clusterState, instance.getCluster().getClusterState());
    }

    changeClusterStateEventually(instances[0], ClusterState.ACTIVE);
  }
  private void testHitsAndLastAccessTimeAreSetWithSingleNode(Config config) throws Exception {
    final TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(1);
    final HazelcastInstance instance1 = nodeFactory.newHazelcastInstance(config);

    final ReplicatedMap<Integer, Integer> map = instance1.getReplicatedMap(randomMapName());
    final int operations = 100;
    execute(
        new Runnable() {
          @Override
          public void run() {
            for (int i = 0; i < operations; i++) {
              map.put(i, i);
            }
          }
        },
        ADDED,
        operations,
        1,
        map);

    for (int i = 0; i < operations; i++) {
      map.containsKey(i);
    }

    for (int i = 0; i < operations; i++) {
      final ReplicatedRecord<Integer, Integer> replicatedRecord = getReplicatedRecord(map, i);
      assertEquals(1, replicatedRecord.getHits());
      assertTrue(
          "Last access time should be set for " + i, replicatedRecord.getLastAccessTime() > 0);
    }
  }
  @Test
  public void testDataIntegrity() throws InterruptedException {
    setLoggingLog4j();
    System.out.println("nodeCount = " + nodeCount);
    System.out.println("operations = " + operations);
    System.out.println("keyCount = " + keyCount);
    Config config = new Config();
    config.getReplicatedMapConfig("test").setReplicationDelayMillis(0);
    TestHazelcastInstanceFactory factory = new TestHazelcastInstanceFactory(nodeCount);
    final HazelcastInstance[] instances = factory.newInstances(config);
    String replicatedMapName = "test";
    final List<ReplicatedMap> maps = createMapOnEachInstance(instances, replicatedMapName);
    ArrayList<Integer> keys = generateRandomIntegerList(keyCount);
    Thread[] threads = createThreads(nodeCount, maps, keys, operations);
    for (Thread thread : threads) {
      thread.start();
    }
    for (Thread thread : threads) {
      thread.join();
    }
    for (int i = 0; i < keyCount; i++) {
      final String key = "foo-" + keys.get(i);
      assertTrueEventually(
          new AssertTask() {
            @Override
            public void run() throws Exception {
              System.out.println("---------------------");
              System.out.println("key = " + key);
              printValues();
              assertValuesAreEqual();
            }

            private void printValues() throws Exception {
              for (int j = 0; j < maps.size(); j++) {
                ReplicatedMap map = maps.get(j);
                System.out.println(
                    "value["
                        + j
                        + "] = "
                        + map.get(key)
                        + " , store version : "
                        + getStore(map, key).getVersion());
              }
            }

            private void assertValuesAreEqual() {
              for (int i = 0; i < maps.size() - 1; i++) {
                ReplicatedMap map1 = maps.get(i);
                ReplicatedMap map2 = maps.get(i + 1);
                Object v1 = map1.get(key);
                Object v2 = map2.get(key);
                assertNotNull(v1);
                assertNotNull(v2);
                assertEquals(v1, v2);
              }
            }
          },
          120);
    }
  }
  @Test(timeout = 120000)
  public void testInitialLoadModeEagerMultipleThread() {
    final String mapName = "default";
    final int instanceCount = 2;
    final int size = 10000;
    final TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(instanceCount);
    final CountDownLatch countDownLatch = new CountDownLatch(instanceCount - 1);
    final Config config = getConfig();
    GroupConfig groupConfig = new GroupConfig("testEager");
    config.setGroupConfig(groupConfig);
    MapStoreConfig mapStoreConfig = new MapStoreConfig();
    mapStoreConfig.setEnabled(true);
    mapStoreConfig.setImplementation(new SimpleMapLoader(size, true));
    mapStoreConfig.setInitialLoadMode(MapStoreConfig.InitialLoadMode.EAGER);
    config.getMapConfig(mapName).setMapStoreConfig(mapStoreConfig);

    HazelcastInstance instance1 = nodeFactory.newHazelcastInstance(config);
    Runnable runnable =
        new Runnable() {
          public void run() {
            HazelcastInstance instance2 = nodeFactory.newHazelcastInstance(config);
            final IMap<Object, Object> map = instance2.getMap(mapName);
            assertEquals(size, map.size());
            countDownLatch.countDown();
          }
        };
    new Thread(runnable).start();

    assertOpenEventually(countDownLatch, 120);
    IMap map = instance1.getMap(mapName);
    assertEquals(size, map.size());
  }
  @Test(timeout = 30000, expected = CancellationException.class)
  public void testInProcessCancellation() throws Exception {

    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

    final HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
    final HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
    final HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

    assertClusterSizeEventually(3, h1);

    IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
    for (int i = 0; i < 100; i++) {
      m1.put(i, i);
    }

    JobTracker tracker = h1.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(KeyValueSource.fromMap(m1));
    ICompletableFuture<Map<String, List<Integer>>> future =
        job.mapper(new TimeConsumingMapper()).submit();

    future.cancel(true);

    try {
      Map<String, List<Integer>> result = future.get();
      fail();

    } catch (Exception e) {
      e.printStackTrace();
      throw e;
    }
  }
  @Test(timeout = 120000)
  public void testOneMemberFlush() throws Exception {
    TestMapStore testMapStore = new TestMapStore(1, 1, 1);
    testMapStore.setLoadAllKeys(false);
    int size = 100;
    Config config = newConfig(testMapStore, 200);
    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);
    HazelcastInstance h1 = nodeFactory.newHazelcastInstance(config);
    IMap map = h1.getMap("default");
    assertEquals(0, map.size());
    for (int i = 0; i < size; i++) {
      map.put(i, i);
    }
    assertEquals(size, map.size());
    assertEquals(0, testMapStore.getStore().size());
    assertEquals(size, map.getLocalMapStats().getDirtyEntryCount());
    map.flush();
    assertEquals(size, testMapStore.getStore().size());
    assertEquals(0, map.getLocalMapStats().getDirtyEntryCount());
    assertEquals(size, map.size());

    for (int i = 0; i < size / 2; i++) {
      map.remove(i);
    }
    assertEquals(size / 2, map.size());
    assertEquals(size, testMapStore.getStore().size());
    map.flush();
    assertEquals(size / 2, testMapStore.getStore().size());
    assertEquals(size / 2, map.size());
  }
  @Test(timeout = 30000)
  public void testMapperReducer() throws Exception {

    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

    HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
    HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
    HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

    IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
    for (int i = 0; i < 100; i++) {
      m1.put(i, i);
    }

    JobTracker tracker = h1.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(KeyValueSource.fromMap(m1));
    ICompletableFuture<Map<String, Integer>> future =
        job.mapper(new GroupingTestMapper()).reducer(new TestReducerFactory()).submit();

    Map<String, Integer> result = future.get();

    // Precalculate results
    int[] expectedResults = new int[4];
    for (int i = 0; i < 100; i++) {
      int index = i % 4;
      expectedResults[index] += i;
    }

    for (int i = 0; i < 4; i++) {
      assertEquals(expectedResults[i], (int) result.get(String.valueOf(i)));
    }
  }
  @Test(timeout = 120000)
  public void testMapStoreNotCalledFromEntryProcessorBackup() throws Exception {
    final String mapName = "testMapStoreNotCalledFromEntryProcessorBackup_" + randomString();
    final int instanceCount = 2;
    Config config = getConfig();
    // Configure map with one backup and dummy map store
    MapConfig mapConfig = config.getMapConfig(mapName);
    mapConfig.setBackupCount(1);
    MapStoreConfig mapStoreConfig = new MapStoreConfig();
    MapStoreWithStoreCount mapStore = new MapStoreWithStoreCount(1, 120);
    mapStoreConfig.setImplementation(mapStore);
    mapConfig.setMapStoreConfig(mapStoreConfig);

    TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(instanceCount);
    HazelcastInstance instance1 = nodeFactory.newHazelcastInstance(config);
    HazelcastInstance instance2 = nodeFactory.newHazelcastInstance(config);

    final IMap<String, String> map = instance1.getMap(mapName);
    final String key = "key";
    final String value = "value";
    // executeOnKey
    map.executeOnKey(key, new ValueSetterEntryProcessor(value));
    mapStore.awaitStores();

    assertEquals(value, map.get(key));
    assertEquals(1, mapStore.getCount());
  }
  @Test(timeout = MINUTE)
  public void testQueryDuringAndAfterMigration() throws Exception {
    HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
    int count = 500;
    IMap imap = h1.getMap("employees");
    for (int i = 0; i < count; i++) {
      imap.put(String.valueOf(i), new Employee("joe" + i, i % 60, ((i & 1) == 1), (double) i));
    }

    nodeFactory.newInstances(new Config(), 3);

    final IMap employees = h1.getMap("employees");
    assertTrueAllTheTime(
        new AssertTask() {
          @Override
          public void run() throws Exception {
            Collection<Employee> values =
                employees.values(new SqlPredicate("active and name LIKE 'joe15%'"));
            for (Employee employee : values) {
              assertTrue(employee.isActive());
            }
            assertEquals(6, values.size());
          }
        },
        3);
  }