コード例 #1
0
 @Test
 public void testDisablingSystemLogs() throws Exception {
   Config config = new Config();
   config.setProperty(GroupProperties.PROP_SYSTEM_LOG_ENABLED, "true");
   config.getGroupConfig().setName("testDisablingSystemLogs");
   HazelcastInstance instance = Hazelcast.newHazelcastInstance(config);
   HazelcastInstance instance2 = Hazelcast.newHazelcastInstance(config);
   instance.getMap("map").put("key", "value");
   Node node = TestUtil.getNode(instance);
   assertTrue(node.getSystemLogService().getLogBundle().size() > 0);
   Hazelcast.shutdownAll();
   config.setProperty(GroupProperties.PROP_SYSTEM_LOG_ENABLED, "false");
   instance = Hazelcast.newHazelcastInstance(config);
   instance2 = Hazelcast.newHazelcastInstance(config);
   instance.getMap("map").put("key2", "value2");
   assertTrue(node.getSystemLogService().getLogBundle().size() == 0);
 }
コード例 #2
0
  @BeforeClass
  public static void init() {
    Config config = new Config();

    QueueConfig queueConfig = config.getQueueConfig(queueForTestQueueWithSizeLimit);
    queueConfig.setMaxSize(maxSizeForQueue);

    queueConfig = config.getQueueConfig(queueForTestOfferPoll);
    queueConfig.setMaxSize(maxSizeForQueue);

    server = Hazelcast.newHazelcastInstance(config);
    client = HazelcastClient.newHazelcastClient();
  }
コード例 #3
0
ファイル: Node.java プロジェクト: hussain-saif/trading-demo
public class Node {
  final HazelcastInstance hazelcast = Hazelcast.newHazelcastInstance(null);
  final ITopic<StockPriceUpdate> topicFeed = hazelcast.getTopic("feed");
  final IQueue<Order> qOrders = hazelcast.getQueue("orders");
  final IMap<String, Position> mapPositions =
      hazelcast.getMap("positions"); // <pmId,instrumentId, Position>
  final IMap<Integer, Integer> mapNewOrders = hazelcast.getMap("neworders"); // <pmId, instrumentId>
  final AtomicLong countReceivedStockUpdates = new AtomicLong();
  final AtomicLong countOrdersProcessed = new AtomicLong();
  final AtomicLong countNewOrderEvents = new AtomicLong();
  final AtomicLong countPositionViews = new AtomicLong();
  final Logger logger = Logger.getLogger("Node");
  final ITopic<String> topicLogs = hazelcast.getTopic("logs");
  final ConcurrentMap<Integer, Double> mapStockPrices =
      new ConcurrentHashMap<Integer, Double>(8000);
  final String memberString = hazelcast.getCluster().getLocalMember().toString();
  final int threads = 100;
  final ExecutorService esOrderConsumer = Executors.newFixedThreadPool(threads);
  final ExecutorService esEventProcessor = Executors.newFixedThreadPool(10);
  final ConcurrentMap<Integer, Portfolio> localPMPositions =
      new ConcurrentHashMap<Integer, Portfolio>();
  final ConcurrentMap<Integer, InstrumentInfo> mapInstrumentInfos =
      new ConcurrentHashMap<Integer, InstrumentInfo>();

  public static void main(String[] args) {
    System.setProperty("hazelcast.mc.topic.excludes", "pm.*");
    new Node().init();
  }

  void init() {
    for (int i = 0; i < threads; i++) {
      esOrderConsumer.execute(new PositionQueueSlurper());
    }
    topicFeed.addMessageListener(new StockStreamListener());
    mapNewOrders.addLocalEntryListener(new NewOrderListener());
    startStreamer();
    Executors.newSingleThreadExecutor()
        .execute(
            new Runnable() {
              public void run() {
                while (true) {
                  try {
                    Thread.sleep(5000);
                    long feeds = countReceivedStockUpdates.getAndSet(0) / 5;
                    long orders = countOrdersProcessed.getAndSet(0) / 5;
                    long events = countNewOrderEvents.getAndSet(0) / 5;
                    long views = countPositionViews.getAndSet(0) / 5;
                    log(
                        "Feeds:"
                            + feeds
                            + ", OrdersProcessed:"
                            + orders
                            + ", newOrderEvents:"
                            + events
                            + ", Views:"
                            + views);
                  } catch (Exception e) {
                    e.printStackTrace();
                  }
                }
              }
            });
  }

  void startStreamer() {
    final Timer timer = new Timer();
    timer.scheduleAtFixedRate(
        new TimerTask() {
          @Override
          public void run() {
            for (int i = 0; i < 500; i++) {
              double price = (int) (Math.random() * 50) + 1;
              topicFeed.publish(
                  new StockPriceUpdate(LookupDatabase.randomPickInstrument().id, price));
            }
          }
        },
        0,
        1000);
  }

  class StockStreamListener implements MessageListener<StockPriceUpdate> {
    public void onMessage(final Message<StockPriceUpdate> stockPriceUpdate) {
      esEventProcessor.execute(
          new Runnable() {
            public void run() {
              countReceivedStockUpdates.incrementAndGet();
              int instrumentId = stockPriceUpdate.getMessageObject().getInstrumentId();
              mapStockPrices.put(instrumentId, stockPriceUpdate.getMessageObject().getPrice());
              InstrumentInfo instrumentInfo = createOrGetInstrumentInfo(instrumentId);
              Collection<Portfolio> relatedPortfolios = instrumentInfo.getPortfolios();
              for (Portfolio relatedPortfolio : relatedPortfolios) {
                firePositionViewChanged(createPositionView(relatedPortfolio, instrumentId));
              }
            }
          });
    }
  }

  private void firePositionViewChanged(PositionView positionView) {
    if (positionView == null) return;
    countPositionViews.incrementAndGet();
    ITopic topicPM = hazelcast.getTopic("pm_" + positionView.pmId);
    topicPM.publish(positionView);
  }

  public PositionView createPositionView(Portfolio portfolio, Integer instrumentId) {
    Double lastPrice = mapStockPrices.get(instrumentId);
    if (lastPrice == null) return null;
    Position position = portfolio.getPosition(instrumentId);
    return new PositionView(
        portfolio.pmId,
        instrumentId,
        position.quantity,
        lastPrice,
        position.calculateProfitLoss(lastPrice));
  }

  private Portfolio createOrGetPortfolio(int pmId) {
    Portfolio portfolio = localPMPositions.get(pmId);
    if (portfolio == null) {
      portfolio = new Portfolio(pmId);
      Portfolio existing = localPMPositions.putIfAbsent(pmId, portfolio);
      if (existing != null) {
        portfolio = existing;
      }
    }
    return portfolio;
  }

  private InstrumentInfo createOrGetInstrumentInfo(int instrumentId) {
    InstrumentInfo instrumentInfo = mapInstrumentInfos.get(instrumentId);
    if (instrumentInfo == null) {
      instrumentInfo = new InstrumentInfo();
      InstrumentInfo existing = mapInstrumentInfos.putIfAbsent(instrumentId, instrumentInfo);
      if (existing != null) {
        instrumentInfo = existing;
      }
    }
    return instrumentInfo;
  }

  class NewOrderListener implements EntryListener<Integer, Integer> {
    public void entryAdded(final EntryEvent<Integer, Integer> entryEvent) {
      countNewOrderEvents.incrementAndGet();
      esEventProcessor.execute(
          new Runnable() {
            public void run() {
              Integer pmId = entryEvent.getKey();
              Integer instrumentId = entryEvent.getValue();
              // load all positions for this pm
              // and create the Portfolio
            }
          });
    }

    public void entryRemoved(final EntryEvent<Integer, Integer> entryEvent) {}

    public void entryUpdated(final EntryEvent<Integer, Integer> entryEvent) {
      countNewOrderEvents.incrementAndGet();
      esEventProcessor.execute(
          new Runnable() {
            public void run() {
              try {
                Integer pmId = entryEvent.getKey();
                Integer instrumentId = entryEvent.getValue();
                Position position = mapPositions.get(pmId + "," + instrumentId);
                if (position != null) {
                  Portfolio portfolio = createOrGetPortfolio(pmId);
                  InstrumentInfo instrumentInfo = createOrGetInstrumentInfo(instrumentId);
                  instrumentInfo.addPortfolio(portfolio);
                  portfolio.update(position);
                  PositionView positionView = createPositionView(portfolio, instrumentId);
                  firePositionViewChanged(positionView);
                }
              } catch (Throwable e) {
                e.printStackTrace();
              }
            }
          });
    }

    public void entryEvicted(final EntryEvent<Integer, Integer> entryEvent) {}
  }

  void log(String msg) {
    if (msg != null) {
      logger.info(msg);
      topicLogs.publish(memberString + ": " + msg);
    }
  }

  public class PositionQueueSlurper implements Runnable {

    public void run() {
      while (true) {
        TransactionContext context = hazelcast.newTransactionContext();
        context.beginTransaction();
        Set<Integer> setAllInvolvedPMs = new HashSet<Integer>(9);
        try {
          Order order = qOrders.take();
          countOrdersProcessed.incrementAndGet();
          List<Integer> lsAccounts = order.lsAccounts;
          int accountQuantity = order.quantity / lsAccounts.size();
          //                    for (Integer account : lsAccounts) {
          //                        String key = account + "," + order.instrumentId;
          //                        updatePosition(key, order, account, accountQuantity);
          //                    }
          String key = order.portfolioManagerId + "," + order.instrumentId;
          updatePosition(key, order, order.portfolioManagerId, order.quantity);
          context.commitTransaction();
          //                    setAllInvolvedPMs.addAll(lsAccounts);
          setAllInvolvedPMs.add(order.portfolioManagerId);
          for (Integer involvedPM : setAllInvolvedPMs) {
            mapNewOrders.put(involvedPM, order.instrumentId);
          }
        } catch (Throwable t) {
          t.printStackTrace();
          context.rollbackTransaction();
        }
      }
    }

    public void updatePosition(String key, Order order, int pmId, int quantity) {
      Deal deal = new Deal(quantity, order.price);
      Position position = mapPositions.get(key);
      if (position == null) {
        position = new Position(order.instrumentId);
      } else if (position.getDealSize() > 100) {
        for (int i = 0; i < 10; i++) {
          position.lsDeals.remove(0);
        }
      }
      position.addDeal(deal);
      mapPositions.put(key, position);
    }
  }
}
コード例 #4
0
ファイル: TestApp.java プロジェクト: karanbatra/hazelcast
 public static void main(String[] args) throws Exception {
   TestApp testApp = new TestApp(Hazelcast.newHazelcastInstance(null));
   testApp.start(args);
 }
コード例 #5
0
 @BeforeClass
 @AfterClass
 public static void init() throws Exception {
   System.setProperty(GroupProperties.PROP_VERSION_CHECK_ENABLED, "false");
   Hazelcast.shutdownAll();
 }
コード例 #6
0
 @AfterClass
 public static void destroy() {
   HazelcastClient.shutdownAll();
   Hazelcast.shutdownAll();
 }
コード例 #7
0
 @BeforeClass
 @AfterClass
 public static void start() {
   Hazelcast.shutdownAll();
 }
コード例 #8
0
  public BaseHazelCastStateTracker(String connectionString, String type, int stateTrackerPort)
      throws Exception {
    log.info(
        "Setting up hazelcast with type "
            + type
            + " connection string "
            + connectionString
            + " and port "
            + stateTrackerPort);

    if (type.equals("master") && !PortTaken.portTaken(stateTrackerPort)) {
      // sets up a proper connection string for reference wrt external actors needing a reference
      if (connectionString.equals("master")) {
        String host = InetAddress.getLocalHost().getHostName();
        this.connectionString = host + ":" + stateTrackerPort;
      }

      this.hazelCastPort = stateTrackerPort;
      config = hazelcast();

      h = Hazelcast.newHazelcastInstance(config);
      h.getCluster()
          .addMembershipListener(
              new MembershipListener() {

                @Override
                public void memberAdded(MembershipEvent membershipEvent) {
                  log.info("Member added " + membershipEvent.toString());
                }

                @Override
                public void memberRemoved(MembershipEvent membershipEvent) {
                  log.info("Member removed " + membershipEvent.toString());
                }

                @Override
                public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) {
                  log.info("Member changed " + memberAttributeEvent.toString());
                }
              });
    } else if (type.equals("master") && PortTaken.portTaken(stateTrackerPort))
      throw new IllegalStateException(
          "Specified type was master and the port specified was taken, please specify a different port");
    else {

      setConnectionString(connectionString);
      log.info("Connecting to hazelcast on " + connectionString);
      ClientConfig client = new ClientConfig();
      client.getNetworkConfig().addAddress(connectionString);
      h = HazelcastClient.newHazelcastClient(client);
    }

    this.type = type;

    jobs = h.getList(JOBS);
    workers = h.getList(WORKERS);

    // we can make the assumption workers isn't empty because
    // the master node by default comes with a applyTransformToDestination of workers
    if (!this.type.equals("master")) {
      while (workers.isEmpty()) {
        log.warn("Waiting for data sync...");
        Thread.sleep(1000);
      }

      log.info("Workers is " + workers.size());
    }

    begunTraining = h.getAtomicReference(BEGUN);
    miniBatchSize = h.getAtomicReference(INPUT_SPLIT);
    workerEnabled = h.getMap(WORKER_ENABLED);
    replicate = h.getList(REPLICATE_WEIGHTS);
    topics = h.getList(TOPICS);
    updates = h.getList(UPDATES);
    heartbeat = h.getMap(HEART_BEAT);
    master = h.getAtomicReference(RESULT);
    isPretrain = h.getAtomicReference(IS_PRETRAIN);
    numTimesPretrain = h.getAtomicReference(NUM_TIMES_RUN_PRETRAIN);
    numTimesPretrainRan = h.getAtomicReference(NUM_TIMES_PRETRAIN_RAN);
    done = h.getAtomicReference(DONE);
    validationEpochs = h.getAtomicReference(VALIDATION_EPOCHS);
    improvementThreshold = h.getAtomicReference(IMPROVEMENT_THRESHOLD);
    bestLoss = h.getAtomicReference(BEST_LOSS);
    earlyStop = h.getAtomicReference(EARLY_STOP);
    patience = h.getAtomicReference(PATIENCE);
    patienceIncrease = h.getAtomicReference(PATIENCE_INCREASE);
    numBatches = h.getAtomicReference(NUM_BATCHES_SO_FAR_RAN);

    // applyTransformToDestination defaults only when master, otherwise, overrides previous values
    if (type.equals("master")) {
      begunTraining.set(false);
      saver = createUpdateSaver();
      numTimesPretrainRan.set(0);
      numTimesPretrain.set(1);
      isPretrain.set(true);
      done.set(false);
      resource = new StateTrackerDropWizardResource(this);
      bestLoss.set(Double.POSITIVE_INFINITY);
      earlyStop.set(true);
      patience.set(40.0);
      patienceIncrease.set(2.0);
      improvementThreshold.set(0.995);
      validationEpochs.set((int) Math.min(10, patience() / 2));
      numBatches.set(0);
    }
  }