@Verify(global = false)
  public void verify() throws Exception {
    LockCounter total = new LockCounter();
    for (LockCounter counter : globalCounter) {
      total.add(counter);
    }
    LOGGER.info(basename + ": " + total + " from " + globalCounter.size() + " worker threads");

    long[] expected = new long[keyCount];
    for (long[] increments : globalIncrements) {
      for (int i = 0; i < increments.length; i++) {
        expected[i] += increments[i];
      }
    }

    int failures = 0;
    for (int key = 0; key < keyCount; key++) {
      if (expected[key] != list.get(key)) {
        failures++;
        LOGGER.info(
            basename
                + ": key="
                + key
                + " expected "
                + expected[key]
                + " != "
                + "actual "
                + list.get(key));
      }
    }
    assertEquals(
        basename + ": " + failures + " key=>values have been incremented unexpected", 0, failures);
  }
 @Override
 public void addWorker(String worker) {
   heartbeat.put(worker, System.currentTimeMillis());
   if (!workers.contains(worker)) {
     log.info("Adding worker " + worker);
     workers.add(worker);
     log.info("Number of workers is now " + workers.size());
   }
 }
  @Test
  public void testAddRemove() throws Exception {
    String listName = randomString();
    final IList l = client.getList(listName);
    l.add("item1");

    final TransactionContext context = client.newTransactionContext();
    context.beginTransaction();
    final TransactionalList<Object> list = context.getList(listName);
    assertTrue(list.add("item2"));
    assertEquals(2, list.size());
    assertEquals(1, l.size());
    assertFalse(list.remove("item3"));
    assertTrue(list.remove("item1"));

    context.commitTransaction();

    assertEquals(1, l.size());
  }
Example #4
0
 protected void handleListAddMany(String[] args) {
   int count = 1;
   if (args.length > 1) count = Integer.parseInt(args[1]);
   int successCount = 0;
   long t0 = Clock.currentTimeMillis();
   for (int i = 0; i < count; i++) {
     boolean success = getList().add("obj" + i);
     if (success) successCount++;
   }
   long t1 = Clock.currentTimeMillis();
   println("Added " + successCount + " objects.");
   println("size = " + list.size() + ", " + successCount * 1000 / (t1 - t0) + " evt/s");
 }
  @Test
  public void testAddAndRoleBack() throws Exception {
    final String listName = randomString();
    final IList l = client.getList(listName);
    l.add("item1");

    final TransactionContext context = client.newTransactionContext();
    context.beginTransaction();
    final TransactionalList<Object> list = context.getList(listName);
    list.add("item2");
    context.rollbackTransaction();

    assertEquals(1, l.size());
  }
  @Verify
  public void globalVerify() {
    Counter totalCounter = new Counter();
    for (Counter counter : results) {
      totalCounter.add(counter);
    }
    LOGGER.info(basename + " " + totalCounter + " from " + results.size() + " Worker threads");

    ICacheEntryListener totalEvents = new ICacheEntryListener();
    for (ICacheEntryListener entryListener : listeners) {
      totalEvents.add(entryListener);
    }
    LOGGER.info(basename + " totalEvents: " + totalEvents);
    assertEquals(basename + " unexpected events found", 0, totalEvents.getUnexpected());
  }
 @Override
 public int numWorkers() {
   int num = workers.size();
   return num;
 }
  public BaseHazelCastStateTracker(String connectionString, String type, int stateTrackerPort)
      throws Exception {
    log.info(
        "Setting up hazelcast with type "
            + type
            + " connection string "
            + connectionString
            + " and port "
            + stateTrackerPort);

    if (type.equals("master") && !PortTaken.portTaken(stateTrackerPort)) {
      // sets up a proper connection string for reference wrt external actors needing a reference
      if (connectionString.equals("master")) {
        String host = InetAddress.getLocalHost().getHostName();
        this.connectionString = host + ":" + stateTrackerPort;
      }

      this.hazelCastPort = stateTrackerPort;
      config = hazelcast();

      h = Hazelcast.newHazelcastInstance(config);
      h.getCluster()
          .addMembershipListener(
              new MembershipListener() {

                @Override
                public void memberAdded(MembershipEvent membershipEvent) {
                  log.info("Member added " + membershipEvent.toString());
                }

                @Override
                public void memberRemoved(MembershipEvent membershipEvent) {
                  log.info("Member removed " + membershipEvent.toString());
                }

                @Override
                public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) {
                  log.info("Member changed " + memberAttributeEvent.toString());
                }
              });
    } else if (type.equals("master") && PortTaken.portTaken(stateTrackerPort))
      throw new IllegalStateException(
          "Specified type was master and the port specified was taken, please specify a different port");
    else {

      setConnectionString(connectionString);
      log.info("Connecting to hazelcast on " + connectionString);
      ClientConfig client = new ClientConfig();
      client.getNetworkConfig().addAddress(connectionString);
      h = HazelcastClient.newHazelcastClient(client);
    }

    this.type = type;

    jobs = h.getList(JOBS);
    workers = h.getList(WORKERS);

    // we can make the assumption workers isn't empty because
    // the master node by default comes with a applyTransformToDestination of workers
    if (!this.type.equals("master")) {
      while (workers.isEmpty()) {
        log.warn("Waiting for data sync...");
        Thread.sleep(1000);
      }

      log.info("Workers is " + workers.size());
    }

    begunTraining = h.getAtomicReference(BEGUN);
    miniBatchSize = h.getAtomicReference(INPUT_SPLIT);
    workerEnabled = h.getMap(WORKER_ENABLED);
    replicate = h.getList(REPLICATE_WEIGHTS);
    topics = h.getList(TOPICS);
    updates = h.getList(UPDATES);
    heartbeat = h.getMap(HEART_BEAT);
    master = h.getAtomicReference(RESULT);
    isPretrain = h.getAtomicReference(IS_PRETRAIN);
    numTimesPretrain = h.getAtomicReference(NUM_TIMES_RUN_PRETRAIN);
    numTimesPretrainRan = h.getAtomicReference(NUM_TIMES_PRETRAIN_RAN);
    done = h.getAtomicReference(DONE);
    validationEpochs = h.getAtomicReference(VALIDATION_EPOCHS);
    improvementThreshold = h.getAtomicReference(IMPROVEMENT_THRESHOLD);
    bestLoss = h.getAtomicReference(BEST_LOSS);
    earlyStop = h.getAtomicReference(EARLY_STOP);
    patience = h.getAtomicReference(PATIENCE);
    patienceIncrease = h.getAtomicReference(PATIENCE_INCREASE);
    numBatches = h.getAtomicReference(NUM_BATCHES_SO_FAR_RAN);

    // applyTransformToDestination defaults only when master, otherwise, overrides previous values
    if (type.equals("master")) {
      begunTraining.set(false);
      saver = createUpdateSaver();
      numTimesPretrainRan.set(0);
      numTimesPretrain.set(1);
      isPretrain.set(true);
      done.set(false);
      resource = new StateTrackerDropWizardResource(this);
      bestLoss.set(Double.POSITIVE_INFINITY);
      earlyStop.set(true);
      patience.set(40.0);
      patienceIncrease.set(2.0);
      improvementThreshold.set(0.995);
      validationEpochs.set((int) Math.min(10, patience() / 2));
      numBatches.set(0);
    }
  }