Exemple #1
0
  @Test
  public void testMapPutTtlWithListener() throws InterruptedException {
    Config cfg = new Config();
    TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(2);
    final HazelcastInstance[] instances = factory.newInstances(cfg);
    warmUpPartitions(instances);

    final int k = 10;
    final int putCount = 10000;
    final CountDownLatch latch = new CountDownLatch(k * putCount);
    final IMap map = instances[0].getMap("testMapEvictionTtlWithListener");

    final AtomicBoolean error = new AtomicBoolean(false);
    final Set<Long> times = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());

    map.addEntryListener(
        new EntryAdapter() {
          public void entryEvicted(final EntryEvent event) {
            final Long expectedEvictionTime = (Long) (event.getOldValue());
            long timeDifference = System.currentTimeMillis() - expectedEvictionTime;
            if (timeDifference > 5000) {
              error.set(true);
              times.add(timeDifference);
            }
            latch.countDown();
          }
        },
        true);

    for (int i = 0; i < k; i++) {
      final int threadId = i;
      int ttl = (int) (Math.random() * 5000 + 3000);
      for (int j = 0; j < putCount; j++) {
        final long expectedEvictionTime = ttl + System.currentTimeMillis();
        map.put(j + putCount * threadId, expectedEvictionTime, ttl, TimeUnit.MILLISECONDS);
      }
    }

    assertTrue(latch.await(1, TimeUnit.MINUTES));
    assertFalse("Some evictions took more than 3 seconds! -> " + times, error.get());
  }
    /*
     * (non-Javadoc)
     *
     * @see
     * org.mpilone.hazelcastmq.core.DefaultHazelcastMQConsumer.ReceiveStrategy
     * #receive(com.hazelcast.core.IQueue)
     */
    @Override
    public byte[] receive(IQueue<byte[]> queue) {
      if (closed) {
        return null;
      }

      long t = Math.min(timeout, 500L);
      timeout -= 500L;

      try {
        if (!active) {
          receiveCondition.await(t, TimeUnit.MILLISECONDS);
        } else {
          return queue.poll(t, TimeUnit.MILLISECONDS);
        }
      } catch (InterruptedException ex) {
        // Ignore for now
      }

      return null;
    }
  public BaseHazelCastStateTracker(String connectionString, String type, int stateTrackerPort)
      throws Exception {
    log.info(
        "Setting up hazelcast with type "
            + type
            + " connection string "
            + connectionString
            + " and port "
            + stateTrackerPort);

    if (type.equals("master") && !PortTaken.portTaken(stateTrackerPort)) {
      // sets up a proper connection string for reference wrt external actors needing a reference
      if (connectionString.equals("master")) {
        String host = InetAddress.getLocalHost().getHostName();
        this.connectionString = host + ":" + stateTrackerPort;
      }

      this.hazelCastPort = stateTrackerPort;
      config = hazelcast();

      h = Hazelcast.newHazelcastInstance(config);
      h.getCluster()
          .addMembershipListener(
              new MembershipListener() {

                @Override
                public void memberAdded(MembershipEvent membershipEvent) {
                  log.info("Member added " + membershipEvent.toString());
                }

                @Override
                public void memberRemoved(MembershipEvent membershipEvent) {
                  log.info("Member removed " + membershipEvent.toString());
                }

                @Override
                public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) {
                  log.info("Member changed " + memberAttributeEvent.toString());
                }
              });
    } else if (type.equals("master") && PortTaken.portTaken(stateTrackerPort))
      throw new IllegalStateException(
          "Specified type was master and the port specified was taken, please specify a different port");
    else {

      setConnectionString(connectionString);
      log.info("Connecting to hazelcast on " + connectionString);
      ClientConfig client = new ClientConfig();
      client.getNetworkConfig().addAddress(connectionString);
      h = HazelcastClient.newHazelcastClient(client);
    }

    this.type = type;

    jobs = h.getList(JOBS);
    workers = h.getList(WORKERS);

    // we can make the assumption workers isn't empty because
    // the master node by default comes with a applyTransformToDestination of workers
    if (!this.type.equals("master")) {
      while (workers.isEmpty()) {
        log.warn("Waiting for data sync...");
        Thread.sleep(1000);
      }

      log.info("Workers is " + workers.size());
    }

    begunTraining = h.getAtomicReference(BEGUN);
    miniBatchSize = h.getAtomicReference(INPUT_SPLIT);
    workerEnabled = h.getMap(WORKER_ENABLED);
    replicate = h.getList(REPLICATE_WEIGHTS);
    topics = h.getList(TOPICS);
    updates = h.getList(UPDATES);
    heartbeat = h.getMap(HEART_BEAT);
    master = h.getAtomicReference(RESULT);
    isPretrain = h.getAtomicReference(IS_PRETRAIN);
    numTimesPretrain = h.getAtomicReference(NUM_TIMES_RUN_PRETRAIN);
    numTimesPretrainRan = h.getAtomicReference(NUM_TIMES_PRETRAIN_RAN);
    done = h.getAtomicReference(DONE);
    validationEpochs = h.getAtomicReference(VALIDATION_EPOCHS);
    improvementThreshold = h.getAtomicReference(IMPROVEMENT_THRESHOLD);
    bestLoss = h.getAtomicReference(BEST_LOSS);
    earlyStop = h.getAtomicReference(EARLY_STOP);
    patience = h.getAtomicReference(PATIENCE);
    patienceIncrease = h.getAtomicReference(PATIENCE_INCREASE);
    numBatches = h.getAtomicReference(NUM_BATCHES_SO_FAR_RAN);

    // applyTransformToDestination defaults only when master, otherwise, overrides previous values
    if (type.equals("master")) {
      begunTraining.set(false);
      saver = createUpdateSaver();
      numTimesPretrainRan.set(0);
      numTimesPretrain.set(1);
      isPretrain.set(true);
      done.set(false);
      resource = new StateTrackerDropWizardResource(this);
      bestLoss.set(Double.POSITIVE_INFINITY);
      earlyStop.set(true);
      patience.set(40.0);
      patienceIncrease.set(2.0);
      improvementThreshold.set(0.995);
      validationEpochs.set((int) Math.min(10, patience() / 2));
      numBatches.set(0);
    }
  }
Exemple #4
0
 public long findDelayMillis(ScheduledEntry entry) {
   long diffMillis = (System.nanoTime() - entry.getScheduleTimeNanos()) / 1000000;
   return Math.max(0, entry.getScheduledDelayMillis() - diffMillis);
 }