@Test public void testDisablingSystemLogs() throws Exception { Config config = new Config(); config.setProperty(GroupProperties.PROP_SYSTEM_LOG_ENABLED, "true"); config.getGroupConfig().setName("testDisablingSystemLogs"); HazelcastInstance instance = Hazelcast.newHazelcastInstance(config); HazelcastInstance instance2 = Hazelcast.newHazelcastInstance(config); instance.getMap("map").put("key", "value"); Node node = TestUtil.getNode(instance); assertTrue(node.getSystemLogService().getLogBundle().size() > 0); Hazelcast.shutdownAll(); config.setProperty(GroupProperties.PROP_SYSTEM_LOG_ENABLED, "false"); instance = Hazelcast.newHazelcastInstance(config); instance2 = Hazelcast.newHazelcastInstance(config); instance.getMap("map").put("key2", "value2"); assertTrue(node.getSystemLogService().getLogBundle().size() == 0); }
@BeforeClass public static void init() { Config config = new Config(); QueueConfig queueConfig = config.getQueueConfig(queueForTestQueueWithSizeLimit); queueConfig.setMaxSize(maxSizeForQueue); queueConfig = config.getQueueConfig(queueForTestOfferPoll); queueConfig.setMaxSize(maxSizeForQueue); server = Hazelcast.newHazelcastInstance(config); client = HazelcastClient.newHazelcastClient(); }
public class Node { final HazelcastInstance hazelcast = Hazelcast.newHazelcastInstance(null); final ITopic<StockPriceUpdate> topicFeed = hazelcast.getTopic("feed"); final IQueue<Order> qOrders = hazelcast.getQueue("orders"); final IMap<String, Position> mapPositions = hazelcast.getMap("positions"); // <pmId,instrumentId, Position> final IMap<Integer, Integer> mapNewOrders = hazelcast.getMap("neworders"); // <pmId, instrumentId> final AtomicLong countReceivedStockUpdates = new AtomicLong(); final AtomicLong countOrdersProcessed = new AtomicLong(); final AtomicLong countNewOrderEvents = new AtomicLong(); final AtomicLong countPositionViews = new AtomicLong(); final Logger logger = Logger.getLogger("Node"); final ITopic<String> topicLogs = hazelcast.getTopic("logs"); final ConcurrentMap<Integer, Double> mapStockPrices = new ConcurrentHashMap<Integer, Double>(8000); final String memberString = hazelcast.getCluster().getLocalMember().toString(); final int threads = 100; final ExecutorService esOrderConsumer = Executors.newFixedThreadPool(threads); final ExecutorService esEventProcessor = Executors.newFixedThreadPool(10); final ConcurrentMap<Integer, Portfolio> localPMPositions = new ConcurrentHashMap<Integer, Portfolio>(); final ConcurrentMap<Integer, InstrumentInfo> mapInstrumentInfos = new ConcurrentHashMap<Integer, InstrumentInfo>(); public static void main(String[] args) { System.setProperty("hazelcast.mc.topic.excludes", "pm.*"); new Node().init(); } void init() { for (int i = 0; i < threads; i++) { esOrderConsumer.execute(new PositionQueueSlurper()); } topicFeed.addMessageListener(new StockStreamListener()); mapNewOrders.addLocalEntryListener(new NewOrderListener()); startStreamer(); Executors.newSingleThreadExecutor() .execute( new Runnable() { public void run() { while (true) { try { Thread.sleep(5000); long feeds = countReceivedStockUpdates.getAndSet(0) / 5; long orders = countOrdersProcessed.getAndSet(0) / 5; long events = countNewOrderEvents.getAndSet(0) / 5; long views = countPositionViews.getAndSet(0) / 5; log( "Feeds:" + feeds + ", OrdersProcessed:" + orders + ", newOrderEvents:" + events + ", Views:" + views); } catch (Exception e) { e.printStackTrace(); } } } }); } void startStreamer() { final Timer timer = new Timer(); timer.scheduleAtFixedRate( new TimerTask() { @Override public void run() { for (int i = 0; i < 500; i++) { double price = (int) (Math.random() * 50) + 1; topicFeed.publish( new StockPriceUpdate(LookupDatabase.randomPickInstrument().id, price)); } } }, 0, 1000); } class StockStreamListener implements MessageListener<StockPriceUpdate> { public void onMessage(final Message<StockPriceUpdate> stockPriceUpdate) { esEventProcessor.execute( new Runnable() { public void run() { countReceivedStockUpdates.incrementAndGet(); int instrumentId = stockPriceUpdate.getMessageObject().getInstrumentId(); mapStockPrices.put(instrumentId, stockPriceUpdate.getMessageObject().getPrice()); InstrumentInfo instrumentInfo = createOrGetInstrumentInfo(instrumentId); Collection<Portfolio> relatedPortfolios = instrumentInfo.getPortfolios(); for (Portfolio relatedPortfolio : relatedPortfolios) { firePositionViewChanged(createPositionView(relatedPortfolio, instrumentId)); } } }); } } private void firePositionViewChanged(PositionView positionView) { if (positionView == null) return; countPositionViews.incrementAndGet(); ITopic topicPM = hazelcast.getTopic("pm_" + positionView.pmId); topicPM.publish(positionView); } public PositionView createPositionView(Portfolio portfolio, Integer instrumentId) { Double lastPrice = mapStockPrices.get(instrumentId); if (lastPrice == null) return null; Position position = portfolio.getPosition(instrumentId); return new PositionView( portfolio.pmId, instrumentId, position.quantity, lastPrice, position.calculateProfitLoss(lastPrice)); } private Portfolio createOrGetPortfolio(int pmId) { Portfolio portfolio = localPMPositions.get(pmId); if (portfolio == null) { portfolio = new Portfolio(pmId); Portfolio existing = localPMPositions.putIfAbsent(pmId, portfolio); if (existing != null) { portfolio = existing; } } return portfolio; } private InstrumentInfo createOrGetInstrumentInfo(int instrumentId) { InstrumentInfo instrumentInfo = mapInstrumentInfos.get(instrumentId); if (instrumentInfo == null) { instrumentInfo = new InstrumentInfo(); InstrumentInfo existing = mapInstrumentInfos.putIfAbsent(instrumentId, instrumentInfo); if (existing != null) { instrumentInfo = existing; } } return instrumentInfo; } class NewOrderListener implements EntryListener<Integer, Integer> { public void entryAdded(final EntryEvent<Integer, Integer> entryEvent) { countNewOrderEvents.incrementAndGet(); esEventProcessor.execute( new Runnable() { public void run() { Integer pmId = entryEvent.getKey(); Integer instrumentId = entryEvent.getValue(); // load all positions for this pm // and create the Portfolio } }); } public void entryRemoved(final EntryEvent<Integer, Integer> entryEvent) {} public void entryUpdated(final EntryEvent<Integer, Integer> entryEvent) { countNewOrderEvents.incrementAndGet(); esEventProcessor.execute( new Runnable() { public void run() { try { Integer pmId = entryEvent.getKey(); Integer instrumentId = entryEvent.getValue(); Position position = mapPositions.get(pmId + "," + instrumentId); if (position != null) { Portfolio portfolio = createOrGetPortfolio(pmId); InstrumentInfo instrumentInfo = createOrGetInstrumentInfo(instrumentId); instrumentInfo.addPortfolio(portfolio); portfolio.update(position); PositionView positionView = createPositionView(portfolio, instrumentId); firePositionViewChanged(positionView); } } catch (Throwable e) { e.printStackTrace(); } } }); } public void entryEvicted(final EntryEvent<Integer, Integer> entryEvent) {} } void log(String msg) { if (msg != null) { logger.info(msg); topicLogs.publish(memberString + ": " + msg); } } public class PositionQueueSlurper implements Runnable { public void run() { while (true) { TransactionContext context = hazelcast.newTransactionContext(); context.beginTransaction(); Set<Integer> setAllInvolvedPMs = new HashSet<Integer>(9); try { Order order = qOrders.take(); countOrdersProcessed.incrementAndGet(); List<Integer> lsAccounts = order.lsAccounts; int accountQuantity = order.quantity / lsAccounts.size(); // for (Integer account : lsAccounts) { // String key = account + "," + order.instrumentId; // updatePosition(key, order, account, accountQuantity); // } String key = order.portfolioManagerId + "," + order.instrumentId; updatePosition(key, order, order.portfolioManagerId, order.quantity); context.commitTransaction(); // setAllInvolvedPMs.addAll(lsAccounts); setAllInvolvedPMs.add(order.portfolioManagerId); for (Integer involvedPM : setAllInvolvedPMs) { mapNewOrders.put(involvedPM, order.instrumentId); } } catch (Throwable t) { t.printStackTrace(); context.rollbackTransaction(); } } } public void updatePosition(String key, Order order, int pmId, int quantity) { Deal deal = new Deal(quantity, order.price); Position position = mapPositions.get(key); if (position == null) { position = new Position(order.instrumentId); } else if (position.getDealSize() > 100) { for (int i = 0; i < 10; i++) { position.lsDeals.remove(0); } } position.addDeal(deal); mapPositions.put(key, position); } } }
public static void main(String[] args) throws Exception { TestApp testApp = new TestApp(Hazelcast.newHazelcastInstance(null)); testApp.start(args); }
@BeforeClass @AfterClass public static void init() throws Exception { System.setProperty(GroupProperties.PROP_VERSION_CHECK_ENABLED, "false"); Hazelcast.shutdownAll(); }
@AfterClass public static void destroy() { HazelcastClient.shutdownAll(); Hazelcast.shutdownAll(); }
@BeforeClass @AfterClass public static void start() { Hazelcast.shutdownAll(); }
public BaseHazelCastStateTracker(String connectionString, String type, int stateTrackerPort) throws Exception { log.info( "Setting up hazelcast with type " + type + " connection string " + connectionString + " and port " + stateTrackerPort); if (type.equals("master") && !PortTaken.portTaken(stateTrackerPort)) { // sets up a proper connection string for reference wrt external actors needing a reference if (connectionString.equals("master")) { String host = InetAddress.getLocalHost().getHostName(); this.connectionString = host + ":" + stateTrackerPort; } this.hazelCastPort = stateTrackerPort; config = hazelcast(); h = Hazelcast.newHazelcastInstance(config); h.getCluster() .addMembershipListener( new MembershipListener() { @Override public void memberAdded(MembershipEvent membershipEvent) { log.info("Member added " + membershipEvent.toString()); } @Override public void memberRemoved(MembershipEvent membershipEvent) { log.info("Member removed " + membershipEvent.toString()); } @Override public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) { log.info("Member changed " + memberAttributeEvent.toString()); } }); } else if (type.equals("master") && PortTaken.portTaken(stateTrackerPort)) throw new IllegalStateException( "Specified type was master and the port specified was taken, please specify a different port"); else { setConnectionString(connectionString); log.info("Connecting to hazelcast on " + connectionString); ClientConfig client = new ClientConfig(); client.getNetworkConfig().addAddress(connectionString); h = HazelcastClient.newHazelcastClient(client); } this.type = type; jobs = h.getList(JOBS); workers = h.getList(WORKERS); // we can make the assumption workers isn't empty because // the master node by default comes with a applyTransformToDestination of workers if (!this.type.equals("master")) { while (workers.isEmpty()) { log.warn("Waiting for data sync..."); Thread.sleep(1000); } log.info("Workers is " + workers.size()); } begunTraining = h.getAtomicReference(BEGUN); miniBatchSize = h.getAtomicReference(INPUT_SPLIT); workerEnabled = h.getMap(WORKER_ENABLED); replicate = h.getList(REPLICATE_WEIGHTS); topics = h.getList(TOPICS); updates = h.getList(UPDATES); heartbeat = h.getMap(HEART_BEAT); master = h.getAtomicReference(RESULT); isPretrain = h.getAtomicReference(IS_PRETRAIN); numTimesPretrain = h.getAtomicReference(NUM_TIMES_RUN_PRETRAIN); numTimesPretrainRan = h.getAtomicReference(NUM_TIMES_PRETRAIN_RAN); done = h.getAtomicReference(DONE); validationEpochs = h.getAtomicReference(VALIDATION_EPOCHS); improvementThreshold = h.getAtomicReference(IMPROVEMENT_THRESHOLD); bestLoss = h.getAtomicReference(BEST_LOSS); earlyStop = h.getAtomicReference(EARLY_STOP); patience = h.getAtomicReference(PATIENCE); patienceIncrease = h.getAtomicReference(PATIENCE_INCREASE); numBatches = h.getAtomicReference(NUM_BATCHES_SO_FAR_RAN); // applyTransformToDestination defaults only when master, otherwise, overrides previous values if (type.equals("master")) { begunTraining.set(false); saver = createUpdateSaver(); numTimesPretrainRan.set(0); numTimesPretrain.set(1); isPretrain.set(true); done.set(false); resource = new StateTrackerDropWizardResource(this); bestLoss.set(Double.POSITIVE_INFINITY); earlyStop.set(true); patience.set(40.0); patienceIncrease.set(2.0); improvementThreshold.set(0.995); validationEpochs.set((int) Math.min(10, patience() / 2)); numBatches.set(0); } }