/* github issue #183 */
@Test
public void testKeyBasedListeners() throws InterruptedException {
try {
Config config = new Config();
HazelcastInstance instance = Hazelcast.newHazelcastInstance(config);
IMap<String, String> map = instance.getMap("map");
map.put("key1", "value1");
map.put("key2", "value2");
map.put("key3", "value3");
ClientConfig clientConfig = new ClientConfig();
HazelcastInstance client = HazelcastClient.newHazelcastClient(clientConfig);
final AtomicInteger count = new AtomicInteger(0);
IMap<String, String> clientMap = client.getMap("map");
clientMap.addEntryListener(
new EntryListener<String, String>() {
public void entryAdded(EntryEvent<String, String> entryEvent) {
count.incrementAndGet();
}
public void entryRemoved(EntryEvent<String, String> entryEvent) {}
public void entryUpdated(EntryEvent<String, String> entryEvent) {
count.incrementAndGet();
}
public void entryEvicted(EntryEvent<String, String> entryEvent) {}
},
"key1",
true);
clientMap.addEntryListener(
new EntryListener<String, String>() {
public void entryAdded(EntryEvent<String, String> entryEvent) {
count.incrementAndGet();
}
public void entryRemoved(EntryEvent<String, String> entryEvent) {}
public void entryUpdated(EntryEvent<String, String> entryEvent) {
System.out.println("event map");
count.incrementAndGet();
}
public void entryEvicted(EntryEvent<String, String> entryEvent) {}
},
"key2",
true);
map.put("key1", "new-value1");
Thread.sleep(100);
Assert.assertEquals(count.get(), 1);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
@Test
public void testZeroResetsTTL() throws InterruptedException {
Config cfg = new Config();
MapConfig mc = cfg.getMapConfig("testZeroResetsTTL");
int ttl = 5;
mc.setTimeToLiveSeconds(ttl);
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(1);
HazelcastInstance instance = factory.newHazelcastInstance(cfg);
IMap<Object, Object> map = instance.getMap("testZeroResetsTTL");
final CountDownLatch latch = new CountDownLatch(1);
map.addEntryListener(
new EntryAdapter<Object, Object>() {
public void entryEvicted(EntryEvent event) {
latch.countDown();
}
},
false);
map.put(1, 1);
map.put(2, 2);
map.put(1, 2, 0, TimeUnit.SECONDS);
latch.await(10, TimeUnit.SECONDS);
assertNull(map.get(2));
assertEquals(2, map.get(1));
}
@Test
public void testDisablingSystemLogs() throws Exception {
Config config = new Config();
config.setProperty(GroupProperties.PROP_SYSTEM_LOG_ENABLED, "true");
config.getGroupConfig().setName("testDisablingSystemLogs");
HazelcastInstance instance = Hazelcast.newHazelcastInstance(config);
HazelcastInstance instance2 = Hazelcast.newHazelcastInstance(config);
instance.getMap("map").put("key", "value");
Node node = TestUtil.getNode(instance);
assertTrue(node.getSystemLogService().getLogBundle().size() > 0);
Hazelcast.shutdownAll();
config.setProperty(GroupProperties.PROP_SYSTEM_LOG_ENABLED, "false");
instance = Hazelcast.newHazelcastInstance(config);
instance2 = Hazelcast.newHazelcastInstance(config);
instance.getMap("map").put("key2", "value2");
assertTrue(node.getSystemLogService().getLogBundle().size() == 0);
}
@Test
public void testMapRecordIdleEvictionOnMigration() throws InterruptedException {
Config cfg = new Config();
final String name = "testMapRecordIdleEvictionOnMigration";
MapConfig mc = cfg.getMapConfig(name);
int maxIdleSeconds = 10;
int size = 100;
final int nsize = size / 5;
mc.setMaxIdleSeconds(maxIdleSeconds);
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(3);
HazelcastInstance instance1 = factory.newHazelcastInstance(cfg);
final IMap map = instance1.getMap(name);
final CountDownLatch latch = new CountDownLatch(size - nsize);
map.addEntryListener(
new EntryAdapter() {
public void entryEvicted(EntryEvent event) {
latch.countDown();
}
},
false);
for (int i = 0; i < size; i++) {
map.put(i, i);
}
final Thread thread =
new Thread(
new Runnable() {
public void run() {
while (!Thread.currentThread().isInterrupted()) {
try {
for (int i = 0; i < nsize; i++) {
map.get(i);
}
Thread.sleep(1000);
} catch (HazelcastInstanceNotActiveException e) {
return;
} catch (InterruptedException e) {
return;
}
}
}
});
thread.start();
HazelcastInstance instance2 = factory.newHazelcastInstance(cfg);
HazelcastInstance instance3 = factory.newHazelcastInstance(cfg);
assertTrue(latch.await(1, TimeUnit.MINUTES));
Assert.assertEquals(nsize, map.size());
thread.interrupt();
thread.join(5000);
}
/**
* Test for issue 614
*
* @throws InterruptedException
*/
@Test
public void testContainsKeyShouldDelayEviction() throws InterruptedException {
Config cfg = new Config();
String mapname = "testContainsKeyShouldDelayEviction";
cfg.getMapConfig(mapname).setMaxIdleSeconds(3);
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(1);
HazelcastInstance instance = factory.newHazelcastInstance(cfg);
IMap<Object, Object> map = instance.getMap(mapname);
map.put(1, 1);
for (int i = 0; i < 20; i++) {
assertTrue(map.containsKey(1));
Thread.sleep(500);
}
}
@Test
@Ignore
public void testCallState() throws Exception {
Config config = new Config();
final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(config);
final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(config);
final Node node1 = getNode(h1);
final Node node2 = getNode(h2);
Thread.sleep(100);
final CountDownLatch latch = new CountDownLatch(1);
final IMap imap1 = h1.getMap("default");
new Thread(
new Runnable() {
public void run() {
imap1.lock("1");
latch.countDown();
}
})
.start();
latch.await();
// final IMap imap2 = h2.getMap("default");
final AtomicInteger threadId = new AtomicInteger();
new Thread(
new Runnable() {
public void run() {
ThreadContext.get().setCurrentFactory(node1.factory);
threadId.set(ThreadContext.get().getThreadId());
imap1.put("1", "value1");
}
})
.start();
Thread.sleep(1000);
System.out.println(node1.getThisAddress() + " thread " + threadId.get());
CallState callState1 =
node1.getSystemLogService().getCallState(node1.getThisAddress(), threadId.get());
if (callState1 != null) {
for (Object callStateLog : callState1.getLogs()) {
System.out.println(callStateLog);
}
}
CallState callState2 =
node2.getSystemLogService().getCallState(node1.getThisAddress(), threadId.get());
System.out.println("========================");
if (callState2 != null) {
for (Object callStateLog : callState2.getLogs()) {
System.out.println(callStateLog);
}
}
}
@Test
public void testExecuteOnAllMembers() {
final IExecutorService service = client.getExecutorService(randomString());
final String mapName = randomString();
service.executeOnAllMembers(new MapPutRunnable(mapName));
final IMap map = client.getMap(mapName);
assertTrueEventually(
new AssertTask() {
public void run() throws Exception {
assertEquals(CLUSTER_SIZE, map.size());
}
});
}
@Test
public void testSqlPredicate() {
HazelcastInstance h = getHazelcastInstance();
HazelcastClient hClient = getHazelcastClient();
IMap<Integer, Employee> map = hClient.getMap("testSqlPredicate");
for (int i = 0; i < 100; i++) {
h.getMap("testSqlPredicate").put(i, new Employee("" + i, i, i % 2 == 0, i));
}
Set<Entry<Integer, Employee>> set = map.entrySet(new SqlPredicate("active AND age < 30"));
for (Entry<Integer, Employee> entry : set) {
System.out.println(entry.getValue());
assertTrue(entry.getValue().age < 30);
assertTrue(entry.getValue().active);
}
}
/*
github issue 585
*/
@Test
public void testIssue585ZeroTTLShouldPreventEvictionWithSet() throws InterruptedException {
Config config = new Config();
config.getGroupConfig().setName("testIssue585ZeroTTLShouldPreventEvictionWithSet");
NearCacheConfig nearCacheConfig = new NearCacheConfig();
config.getMapConfig("default").setNearCacheConfig(nearCacheConfig);
int n = 1;
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(n);
HazelcastInstance h = factory.newHazelcastInstance(config);
IMap<String, String> map = h.getMap("testIssue585ZeroTTLShouldPreventEvictionWithSet");
map.set("key", "value", 1, TimeUnit.SECONDS);
map.set("key", "value2", 0, TimeUnit.SECONDS);
Thread.sleep(2000);
assertEquals("value2", map.get("key"));
h.getLifecycleService().shutdown();
}
@Test
public void testExecuteOnMembers_withEmptyCollection() {
final IExecutorService service = client.getExecutorService(randomString());
final String mapName = randomString();
final Collection collection = new ArrayList();
service.executeOnMembers(new MapPutRunnable(mapName), collection);
final IMap map = client.getMap(mapName);
assertTrueEventually(
new AssertTask() {
public void run() throws Exception {
assertTrue(map.isEmpty());
}
});
}
@Test
public void testExecuteOnMember() {
final IExecutorService service = client.getExecutorService(randomString());
final String mapName = randomString();
final Member member = instance1.getCluster().getLocalMember();
final String targetUuid = member.getUuid();
service.executeOnMember(new MapPutRunnable(mapName), member);
final IMap map = client.getMap(mapName);
assertTrueEventually(
new AssertTask() {
public void run() throws Exception {
assertTrue(map.containsKey(targetUuid));
}
});
}
@Test
public void testDestroyJustAfterCreate() {
final HazelcastInstance instance = Hazelcast.newHazelcastInstance();
instance.addDistributedObjectListener(new EventCountListener());
IMap<Object, Object> map = instance.getMap(randomString());
map.destroy();
AssertTask task =
new AssertTask() {
@Override
public void run() throws Exception {
Assert.assertEquals(1, EventCountListener.createdCount.get());
Assert.assertEquals(1, EventCountListener.destroyedCount.get());
Collection<DistributedObject> distributedObjects = instance.getDistributedObjects();
Assert.assertTrue(distributedObjects.isEmpty());
}
};
assertTrueEventually(task, 5);
assertTrueAllTheTime(task, 3);
}
public static void main(String[] args) {
System.out.println("DEMO Start...");
Config cfg = new Config();
HazelcastInstance instance = Hazelcast.newHazelcastInstance(cfg);
Map<Integer, String> mapCustomers = instance.getMap("customers");
mapCustomers.put(1, "Joe");
mapCustomers.put(2, "Ali");
mapCustomers.put(3, "Avi");
System.out.println("Customer with key 1: " + mapCustomers.get(1));
System.out.println("Map Size:" + mapCustomers.size());
Queue<String> queueCustomers = instance.getQueue("customers");
queueCustomers.offer("Tom");
queueCustomers.offer("Mary");
queueCustomers.offer("Jane");
System.out.println("First customer: " + queueCustomers.poll());
System.out.println("Second customer: " + queueCustomers.peek());
System.out.println("Queue size: " + queueCustomers.size());
}
/*
github issue 304
*/
@Test
public void testIssue304EvictionDespitePut() throws InterruptedException {
Config c = new Config();
c.getGroupConfig().setName("testIssue304EvictionDespitePut");
final HashMap<String, MapConfig> mapConfigs = new HashMap<String, MapConfig>();
final MapConfig value = new MapConfig();
value.setMaxIdleSeconds(3);
mapConfigs.put("default", value);
c.setMapConfigs(mapConfigs);
final Properties properties = new Properties();
properties.setProperty("hazelcast.map.cleanup.delay.seconds", "1"); // we need faster cleanups
c.setProperties(properties);
int n = 1;
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(n);
final HazelcastInstance hazelcastInstance = factory.newHazelcastInstance(c);
IMap<String, Long> map = hazelcastInstance.getMap("testIssue304EvictionDespitePutMap");
final AtomicInteger evictCount = new AtomicInteger(0);
map.addEntryListener(
new EntryListener<String, Long>() {
public void entryAdded(EntryEvent<String, Long> event) {}
public void entryRemoved(EntryEvent<String, Long> event) {}
public void entryUpdated(EntryEvent<String, Long> event) {}
public void entryEvicted(EntryEvent<String, Long> event) {
evictCount.incrementAndGet();
}
},
true);
String key = "key";
for (int i = 0; i < 5; i++) {
map.put(key, System.currentTimeMillis());
Thread.sleep(1000);
}
assertEquals(evictCount.get(), 0);
assertNotNull(map.get(key));
hazelcastInstance.getLifecycleService().shutdown();
}
@Test
public void testExecuteOnMembers() {
final IExecutorService service = client.getExecutorService(randomString());
final String mapName = randomString();
final Collection collection = new ArrayList();
final Member member1 = instance1.getCluster().getLocalMember();
final Member member3 = instance3.getCluster().getLocalMember();
collection.add(member1);
collection.add(member3);
service.executeOnMembers(new MapPutRunnable(mapName), collection);
final IMap map = client.getMap(mapName);
assertTrueEventually(
new AssertTask() {
public void run() throws Exception {
assertTrue(map.containsKey(member1.getUuid()));
assertTrue(map.containsKey(member3.getUuid()));
}
});
}
/**
* Test for the issue 537. Eviction event is fired for an object already removed
*
* @throws Exception
*/
@Test
public void testEvictionAfterRemove() throws InterruptedException {
Config cfg = new Config();
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(1);
HazelcastInstance instance = factory.newHazelcastInstance(cfg);
IMap<Object, Object> map = instance.getMap("map");
final AtomicInteger count = new AtomicInteger(0);
map.addEntryListener(
new EntryAdapter<Object, Object>() {
@Override
public void entryEvicted(EntryEvent<Object, Object> event) {
count.incrementAndGet();
}
},
true);
map.put(1, 1, 1, TimeUnit.SECONDS);
map.put(2, 2, 1, TimeUnit.SECONDS);
map.remove(1);
Thread.sleep(2000);
assertEquals(1, count.get());
}
@Test
public void testMigrationOfTTLAndLock() throws Exception {
Config config = new Config();
final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(config);
final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(config);
ConcurrentMapManager cmm1 = getConcurrentMapManager(h1);
ConcurrentMapManager cmm2 = getConcurrentMapManager(h2);
final IMap imap1 = h1.getMap("default");
final IMap imap2 = h2.getMap("default");
final Data dKey = toData("1");
assertTrue(migrateKey("1", h1, h1, 0));
assertTrue(migrateKey("1", h1, h2, 1));
imap1.put("1", "value1", 60, TimeUnit.SECONDS);
imap1.lock("1");
Future put2 = imap2.putAsync("1", "value2");
imap2.addEntryListener(
new EntryAdapter() {
@Override
public void entryUpdated(EntryEvent entryEvent) {
System.out.println(entryEvent);
}
},
"1",
true);
if (put2 == null) fail();
Thread.sleep(1000);
CMap cmap1 = getCMap(h1, "default");
CMap cmap2 = getCMap(h2, "default");
Record record1 = cmap1.getRecord(dKey);
assertEquals(1, record1.getScheduledActionCount());
for (ScheduledAction scheduledAction : record1.getScheduledActions()) {
assertTrue(scheduledAction.isValid());
}
assertNotNull(record1.getListeners());
assertEquals(1, record1.getListeners().size());
DistributedLock lock = cmap1.getRecord(dKey).getLock();
assertTrue(cmm1.thisAddress.equals(lock.getLockAddress()));
assertTrue(lock.getLockThreadId() != -1);
assertEquals(1, lock.getLockCount());
assertEquals(1, cmap1.mapRecords.size());
assertEquals(1, cmap2.mapRecords.size());
assertEquals(0, cmap2.getMapIndexService().getOwnedRecords().size());
assertEquals(1, cmap1.getMapIndexService().getOwnedRecords().size());
assertTrue(migrateKey("1", h1, h2, 0));
assertTrue(migrateKey("1", h1, h1, 1));
assertEquals(1, cmap1.mapRecords.size());
assertEquals(1, cmap2.mapRecords.size());
assertEquals(0, cmap1.getMapIndexService().getOwnedRecords().size());
assertEquals(1, cmap2.getMapIndexService().getOwnedRecords().size());
assertTrue(cmap1.getRecord(dKey).getRemainingTTL() < 60000);
assertTrue(cmap2.getRecord(dKey).getRemainingTTL() < 60000);
Record record2 = cmap2.getRecord(dKey);
lock = record2.getLock();
assertTrue(cmm1.thisAddress.equals(lock.getLockAddress()));
assertTrue(lock.getLockThreadId() != -1);
assertEquals(1, lock.getLockCount());
lock = cmap1.getRecord(dKey).getLock();
assertTrue(cmm1.thisAddress.equals(lock.getLockAddress()));
assertTrue(lock.getLockThreadId() != -1);
assertEquals(1, lock.getLockCount());
imap1.unlock("1");
put2.get(10, TimeUnit.SECONDS);
assertEquals("value2", imap1.get("1"));
assertEquals("value2", imap2.get("1"));
assertNotNull(record2.getListeners());
assertEquals(1, record2.getListeners().size());
}
public class Node {
final HazelcastInstance hazelcast = Hazelcast.newHazelcastInstance(null);
final ITopic<StockPriceUpdate> topicFeed = hazelcast.getTopic("feed");
final IQueue<Order> qOrders = hazelcast.getQueue("orders");
final IMap<String, Position> mapPositions =
hazelcast.getMap("positions"); // <pmId,instrumentId, Position>
final IMap<Integer, Integer> mapNewOrders = hazelcast.getMap("neworders"); // <pmId, instrumentId>
final AtomicLong countReceivedStockUpdates = new AtomicLong();
final AtomicLong countOrdersProcessed = new AtomicLong();
final AtomicLong countNewOrderEvents = new AtomicLong();
final AtomicLong countPositionViews = new AtomicLong();
final Logger logger = Logger.getLogger("Node");
final ITopic<String> topicLogs = hazelcast.getTopic("logs");
final ConcurrentMap<Integer, Double> mapStockPrices =
new ConcurrentHashMap<Integer, Double>(8000);
final String memberString = hazelcast.getCluster().getLocalMember().toString();
final int threads = 100;
final ExecutorService esOrderConsumer = Executors.newFixedThreadPool(threads);
final ExecutorService esEventProcessor = Executors.newFixedThreadPool(10);
final ConcurrentMap<Integer, Portfolio> localPMPositions =
new ConcurrentHashMap<Integer, Portfolio>();
final ConcurrentMap<Integer, InstrumentInfo> mapInstrumentInfos =
new ConcurrentHashMap<Integer, InstrumentInfo>();
public static void main(String[] args) {
System.setProperty("hazelcast.mc.topic.excludes", "pm.*");
new Node().init();
}
void init() {
for (int i = 0; i < threads; i++) {
esOrderConsumer.execute(new PositionQueueSlurper());
}
topicFeed.addMessageListener(new StockStreamListener());
mapNewOrders.addLocalEntryListener(new NewOrderListener());
startStreamer();
Executors.newSingleThreadExecutor()
.execute(
new Runnable() {
public void run() {
while (true) {
try {
Thread.sleep(5000);
long feeds = countReceivedStockUpdates.getAndSet(0) / 5;
long orders = countOrdersProcessed.getAndSet(0) / 5;
long events = countNewOrderEvents.getAndSet(0) / 5;
long views = countPositionViews.getAndSet(0) / 5;
log(
"Feeds:"
+ feeds
+ ", OrdersProcessed:"
+ orders
+ ", newOrderEvents:"
+ events
+ ", Views:"
+ views);
} catch (Exception e) {
e.printStackTrace();
}
}
}
});
}
void startStreamer() {
final Timer timer = new Timer();
timer.scheduleAtFixedRate(
new TimerTask() {
@Override
public void run() {
for (int i = 0; i < 500; i++) {
double price = (int) (Math.random() * 50) + 1;
topicFeed.publish(
new StockPriceUpdate(LookupDatabase.randomPickInstrument().id, price));
}
}
},
0,
1000);
}
class StockStreamListener implements MessageListener<StockPriceUpdate> {
public void onMessage(final Message<StockPriceUpdate> stockPriceUpdate) {
esEventProcessor.execute(
new Runnable() {
public void run() {
countReceivedStockUpdates.incrementAndGet();
int instrumentId = stockPriceUpdate.getMessageObject().getInstrumentId();
mapStockPrices.put(instrumentId, stockPriceUpdate.getMessageObject().getPrice());
InstrumentInfo instrumentInfo = createOrGetInstrumentInfo(instrumentId);
Collection<Portfolio> relatedPortfolios = instrumentInfo.getPortfolios();
for (Portfolio relatedPortfolio : relatedPortfolios) {
firePositionViewChanged(createPositionView(relatedPortfolio, instrumentId));
}
}
});
}
}
private void firePositionViewChanged(PositionView positionView) {
if (positionView == null) return;
countPositionViews.incrementAndGet();
ITopic topicPM = hazelcast.getTopic("pm_" + positionView.pmId);
topicPM.publish(positionView);
}
public PositionView createPositionView(Portfolio portfolio, Integer instrumentId) {
Double lastPrice = mapStockPrices.get(instrumentId);
if (lastPrice == null) return null;
Position position = portfolio.getPosition(instrumentId);
return new PositionView(
portfolio.pmId,
instrumentId,
position.quantity,
lastPrice,
position.calculateProfitLoss(lastPrice));
}
private Portfolio createOrGetPortfolio(int pmId) {
Portfolio portfolio = localPMPositions.get(pmId);
if (portfolio == null) {
portfolio = new Portfolio(pmId);
Portfolio existing = localPMPositions.putIfAbsent(pmId, portfolio);
if (existing != null) {
portfolio = existing;
}
}
return portfolio;
}
private InstrumentInfo createOrGetInstrumentInfo(int instrumentId) {
InstrumentInfo instrumentInfo = mapInstrumentInfos.get(instrumentId);
if (instrumentInfo == null) {
instrumentInfo = new InstrumentInfo();
InstrumentInfo existing = mapInstrumentInfos.putIfAbsent(instrumentId, instrumentInfo);
if (existing != null) {
instrumentInfo = existing;
}
}
return instrumentInfo;
}
class NewOrderListener implements EntryListener<Integer, Integer> {
public void entryAdded(final EntryEvent<Integer, Integer> entryEvent) {
countNewOrderEvents.incrementAndGet();
esEventProcessor.execute(
new Runnable() {
public void run() {
Integer pmId = entryEvent.getKey();
Integer instrumentId = entryEvent.getValue();
// load all positions for this pm
// and create the Portfolio
}
});
}
public void entryRemoved(final EntryEvent<Integer, Integer> entryEvent) {}
public void entryUpdated(final EntryEvent<Integer, Integer> entryEvent) {
countNewOrderEvents.incrementAndGet();
esEventProcessor.execute(
new Runnable() {
public void run() {
try {
Integer pmId = entryEvent.getKey();
Integer instrumentId = entryEvent.getValue();
Position position = mapPositions.get(pmId + "," + instrumentId);
if (position != null) {
Portfolio portfolio = createOrGetPortfolio(pmId);
InstrumentInfo instrumentInfo = createOrGetInstrumentInfo(instrumentId);
instrumentInfo.addPortfolio(portfolio);
portfolio.update(position);
PositionView positionView = createPositionView(portfolio, instrumentId);
firePositionViewChanged(positionView);
}
} catch (Throwable e) {
e.printStackTrace();
}
}
});
}
public void entryEvicted(final EntryEvent<Integer, Integer> entryEvent) {}
}
void log(String msg) {
if (msg != null) {
logger.info(msg);
topicLogs.publish(memberString + ": " + msg);
}
}
public class PositionQueueSlurper implements Runnable {
public void run() {
while (true) {
TransactionContext context = hazelcast.newTransactionContext();
context.beginTransaction();
Set<Integer> setAllInvolvedPMs = new HashSet<Integer>(9);
try {
Order order = qOrders.take();
countOrdersProcessed.incrementAndGet();
List<Integer> lsAccounts = order.lsAccounts;
int accountQuantity = order.quantity / lsAccounts.size();
// for (Integer account : lsAccounts) {
// String key = account + "," + order.instrumentId;
// updatePosition(key, order, account, accountQuantity);
// }
String key = order.portfolioManagerId + "," + order.instrumentId;
updatePosition(key, order, order.portfolioManagerId, order.quantity);
context.commitTransaction();
// setAllInvolvedPMs.addAll(lsAccounts);
setAllInvolvedPMs.add(order.portfolioManagerId);
for (Integer involvedPM : setAllInvolvedPMs) {
mapNewOrders.put(involvedPM, order.instrumentId);
}
} catch (Throwable t) {
t.printStackTrace();
context.rollbackTransaction();
}
}
}
public void updatePosition(String key, Order order, int pmId, int quantity) {
Deal deal = new Deal(quantity, order.price);
Position position = mapPositions.get(key);
if (position == null) {
position = new Position(order.instrumentId);
} else if (position.getDealSize() > 100) {
for (int i = 0; i < 10; i++) {
position.lsDeals.remove(0);
}
}
position.addDeal(deal);
mapPositions.put(key, position);
}
}
}
public IMap<Object, Object> getMap() {
// if (map == null) {
map = hazelcast.getMap(namespace);
// }
return map;
}
public BaseHazelCastStateTracker(String connectionString, String type, int stateTrackerPort)
throws Exception {
log.info(
"Setting up hazelcast with type "
+ type
+ " connection string "
+ connectionString
+ " and port "
+ stateTrackerPort);
if (type.equals("master") && !PortTaken.portTaken(stateTrackerPort)) {
// sets up a proper connection string for reference wrt external actors needing a reference
if (connectionString.equals("master")) {
String host = InetAddress.getLocalHost().getHostName();
this.connectionString = host + ":" + stateTrackerPort;
}
this.hazelCastPort = stateTrackerPort;
config = hazelcast();
h = Hazelcast.newHazelcastInstance(config);
h.getCluster()
.addMembershipListener(
new MembershipListener() {
@Override
public void memberAdded(MembershipEvent membershipEvent) {
log.info("Member added " + membershipEvent.toString());
}
@Override
public void memberRemoved(MembershipEvent membershipEvent) {
log.info("Member removed " + membershipEvent.toString());
}
@Override
public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) {
log.info("Member changed " + memberAttributeEvent.toString());
}
});
} else if (type.equals("master") && PortTaken.portTaken(stateTrackerPort))
throw new IllegalStateException(
"Specified type was master and the port specified was taken, please specify a different port");
else {
setConnectionString(connectionString);
log.info("Connecting to hazelcast on " + connectionString);
ClientConfig client = new ClientConfig();
client.getNetworkConfig().addAddress(connectionString);
h = HazelcastClient.newHazelcastClient(client);
}
this.type = type;
jobs = h.getList(JOBS);
workers = h.getList(WORKERS);
// we can make the assumption workers isn't empty because
// the master node by default comes with a applyTransformToDestination of workers
if (!this.type.equals("master")) {
while (workers.isEmpty()) {
log.warn("Waiting for data sync...");
Thread.sleep(1000);
}
log.info("Workers is " + workers.size());
}
begunTraining = h.getAtomicReference(BEGUN);
miniBatchSize = h.getAtomicReference(INPUT_SPLIT);
workerEnabled = h.getMap(WORKER_ENABLED);
replicate = h.getList(REPLICATE_WEIGHTS);
topics = h.getList(TOPICS);
updates = h.getList(UPDATES);
heartbeat = h.getMap(HEART_BEAT);
master = h.getAtomicReference(RESULT);
isPretrain = h.getAtomicReference(IS_PRETRAIN);
numTimesPretrain = h.getAtomicReference(NUM_TIMES_RUN_PRETRAIN);
numTimesPretrainRan = h.getAtomicReference(NUM_TIMES_PRETRAIN_RAN);
done = h.getAtomicReference(DONE);
validationEpochs = h.getAtomicReference(VALIDATION_EPOCHS);
improvementThreshold = h.getAtomicReference(IMPROVEMENT_THRESHOLD);
bestLoss = h.getAtomicReference(BEST_LOSS);
earlyStop = h.getAtomicReference(EARLY_STOP);
patience = h.getAtomicReference(PATIENCE);
patienceIncrease = h.getAtomicReference(PATIENCE_INCREASE);
numBatches = h.getAtomicReference(NUM_BATCHES_SO_FAR_RAN);
// applyTransformToDestination defaults only when master, otherwise, overrides previous values
if (type.equals("master")) {
begunTraining.set(false);
saver = createUpdateSaver();
numTimesPretrainRan.set(0);
numTimesPretrain.set(1);
isPretrain.set(true);
done.set(false);
resource = new StateTrackerDropWizardResource(this);
bestLoss.set(Double.POSITIVE_INFINITY);
earlyStop.set(true);
patience.set(40.0);
patienceIncrease.set(2.0);
improvementThreshold.set(0.995);
validationEpochs.set((int) Math.min(10, patience() / 2));
numBatches.set(0);
}
}
@Test
public void testPutWithTwoMember() throws Exception {
Config config = new Config();
HazelcastInstance h1 = Hazelcast.newHazelcastInstance(config);
HazelcastInstance h2 = Hazelcast.newHazelcastInstance(config);
assertEquals(2, h1.getCluster().getMembers().size());
assertEquals(2, h2.getCluster().getMembers().size());
IMap imap1 = h1.getMap("default");
IMap imap2 = h2.getMap("default");
assertEquals(0, imap1.size());
assertEquals(0, imap2.size());
CMap cmap1 = getCMap(h1, "default");
CMap cmap2 = getCMap(h2, "default");
assertNotNull(cmap1);
assertNotNull(cmap2);
Object key = "1";
Object value = "value";
Data dKey = toData(key);
Data dValue = toData(value);
imap1.put(key, value, 5, TimeUnit.SECONDS);
assertEquals(1, cmap1.mapRecords.size());
assertEquals(1, cmap2.mapRecords.size());
assertEquals(
1,
cmap1.getMapIndexService().getOwnedRecords().size()
+ cmap2.getMapIndexService().getOwnedRecords().size());
Record record1 = cmap1.getRecord(dKey);
Record record2 = cmap2.getRecord(dKey);
long now = System.currentTimeMillis();
long millisLeft1 = record1.getExpirationTime() - now;
long millisLeft2 = record2.getExpirationTime() - now;
assertTrue(millisLeft1 <= 5000 && millisLeft1 > 0);
assertTrue(millisLeft2 <= 5000 && millisLeft2 > 0);
assertTrue(record1.isActive());
assertTrue(record2.isActive());
assertEquals(1, record1.valueCount());
assertEquals(1, record2.valueCount());
assertEquals(dValue, record1.getValueData());
assertEquals(dValue, record2.getValueData());
imap1.set("2", "value2", 5, TimeUnit.SECONDS);
assertEquals("value2", imap1.get("2"));
assertEquals("value2", imap2.get("2"));
Thread.sleep(6000);
assertNull(imap1.get("2"));
assertNull(imap2.get("2"));
now = System.currentTimeMillis();
assertFalse(record1.isValid(now));
assertFalse(record2.isValid(now));
Thread.sleep(23000);
assertEquals(0, cmap1.getMapIndexService().getOwnedRecords().size());
assertEquals(0, cmap2.getMapIndexService().getOwnedRecords().size());
assertEquals(0, cmap1.mapRecords.size());
assertEquals(0, cmap2.mapRecords.size());
imap1.put(key, value, 10, TimeUnit.SECONDS);
assertTrue(migrateKey(key, h1, h1, 0));
assertTrue(migrateKey(key, h1, h2, 1));
assertEquals(1, cmap1.mapRecords.size());
assertEquals(1, cmap2.mapRecords.size());
assertEquals(
1,
cmap1.getMapIndexService().getOwnedRecords().size()
+ cmap2.getMapIndexService().getOwnedRecords().size());
record1 = cmap1.getRecord(dKey);
record2 = cmap2.getRecord(dKey);
now = System.currentTimeMillis();
millisLeft1 = record1.getExpirationTime() - now;
millisLeft2 = record2.getExpirationTime() - now;
assertTrue(millisLeft1 <= 11000 && millisLeft1 > 0);
assertTrue(millisLeft2 <= 11000 && millisLeft2 > 0);
assertTrue(record1.isActive());
assertTrue(record2.isActive());
assertTrue(record1.isValid(now));
assertTrue(record2.isValid(now));
assertEquals(1, record1.valueCount());
assertEquals(1, record2.valueCount());
assertEquals(1, cmap1.mapRecords.size());
assertEquals(1, cmap2.mapRecords.size());
assertEquals(
1,
cmap1.getMapIndexService().getOwnedRecords().size()
+ cmap2.getMapIndexService().getOwnedRecords().size());
assertTrue(migrateKey(key, h1, h2, 0));
assertTrue(migrateKey(key, h1, h1, 1));
cmap1.startCleanup(true);
cmap2.startCleanup(true);
assertEquals(1, cmap1.mapRecords.size());
assertEquals(1, cmap2.mapRecords.size());
assertEquals(1, cmap2.getMapIndexService().getOwnedRecords().size());
assertEquals(0, cmap1.getMapIndexService().getOwnedRecords().size());
now = System.currentTimeMillis();
millisLeft1 = record1.getExpirationTime() - now;
millisLeft2 = record2.getExpirationTime() - now;
assertTrue(millisLeft1 <= 10000 && millisLeft1 > 0);
assertTrue(millisLeft2 <= 10000 && millisLeft2 > 0);
assertTrue(record1.isActive());
assertTrue(record2.isActive());
assertTrue(record1.isValid(now));
assertTrue(record2.isValid(now));
assertEquals(1, record1.valueCount());
assertEquals(1, record2.valueCount());
Thread.sleep(11000);
now = System.currentTimeMillis();
assertFalse(record1.isValid(now));
assertFalse(record2.isValid(now));
Thread.sleep(20000);
assertEquals(0, cmap1.mapRecords.size());
assertEquals(0, cmap2.mapRecords.size());
assertEquals(0, cmap1.getMapIndexService().getOwnedRecords().size());
assertEquals(0, cmap2.getMapIndexService().getOwnedRecords().size());
imap1.put("1", "value1");
record1 = cmap1.getRecord(dKey);
record2 = cmap2.getRecord(dKey);
assertEquals(1, cmap1.mapRecords.size());
assertEquals(1, cmap2.mapRecords.size());
assertEquals(0, cmap1.getMapIndexService().getOwnedRecords().size());
assertEquals(1, cmap2.getMapIndexService().getOwnedRecords().size());
now = System.currentTimeMillis();
assertEquals(Long.MAX_VALUE, record1.getExpirationTime());
assertEquals(Long.MAX_VALUE, record2.getExpirationTime());
assertTrue(record1.isActive());
assertTrue(record2.isActive());
assertTrue(record1.isValid(now));
assertTrue(record2.isValid(now));
assertEquals(1, record1.valueCount());
assertEquals(1, record2.valueCount());
imap1.remove("1");
assertEquals(0, cmap1.getMapIndexService().getOwnedRecords().size());
assertEquals(0, cmap2.getMapIndexService().getOwnedRecords().size());
Thread.sleep(20000);
assertEquals(0, cmap1.mapRecords.size());
assertEquals(0, cmap2.mapRecords.size());
assertEquals(0, cmap1.mapIndexService.size());
assertEquals(0, cmap2.mapIndexService.size());
}